{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 15284,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0001308579373517625,
      "grad_norm": 4.019994735717773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11609930545091629,
      "logits/rejected": 0.13315832614898682,
      "logps/chosen": -349.37884521484375,
      "logps/rejected": -377.8258056640625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.000261715874703525,
      "grad_norm": 5.019124984741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21686424314975739,
      "logits/rejected": 0.45246145129203796,
      "logps/chosen": -730.12060546875,
      "logps/rejected": -401.0517883300781,
      "loss": 0.6851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0023597716353833675,
      "rewards/margins": 0.017063520848751068,
      "rewards/rejected": -0.014703752472996712,
      "step": 2
    },
    {
      "epoch": 0.0003925738120552875,
      "grad_norm": 4.27782678604126,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3172861337661743,
      "logits/rejected": 0.4784848690032959,
      "logps/chosen": -422.0670471191406,
      "logps/rejected": -471.250244140625,
      "loss": 0.6859,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0024726868141442537,
      "rewards/margins": 0.015834424644708633,
      "rewards/rejected": -0.018307112157344818,
      "step": 3
    },
    {
      "epoch": 0.00052343174940705,
      "grad_norm": 4.303768157958984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06437239050865173,
      "logits/rejected": 0.04038151726126671,
      "logps/chosen": -374.6029968261719,
      "logps/rejected": -325.31207275390625,
      "loss": 0.7119,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.05454196780920029,
      "rewards/margins": -0.036675069481134415,
      "rewards/rejected": -0.017866898328065872,
      "step": 4
    },
    {
      "epoch": 0.0006542896867588124,
      "grad_norm": 4.421416759490967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11105303466320038,
      "logits/rejected": 0.22600476443767548,
      "logps/chosen": -547.6106567382812,
      "logps/rejected": -394.646240234375,
      "loss": 0.7059,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01652851328253746,
      "rewards/margins": -0.024637414142489433,
      "rewards/rejected": 0.008108902722597122,
      "step": 5
    },
    {
      "epoch": 0.000785147624110575,
      "grad_norm": 4.543910026550293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07156988233327866,
      "logits/rejected": 0.05400428920984268,
      "logps/chosen": -441.0995788574219,
      "logps/rejected": -372.9764709472656,
      "loss": 0.7373,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0630287230014801,
      "rewards/margins": -0.08390311896800995,
      "rewards/rejected": 0.02087440714240074,
      "step": 6
    },
    {
      "epoch": 0.0009160055614623375,
      "grad_norm": 5.023899555206299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1298614740371704,
      "logits/rejected": 0.15890483558177948,
      "logps/chosen": -535.0745239257812,
      "logps/rejected": -555.55517578125,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019374465569853783,
      "rewards/margins": 0.005030821543186903,
      "rewards/rejected": -0.024405287578701973,
      "step": 7
    },
    {
      "epoch": 0.0010468634988141,
      "grad_norm": 3.7505042552948,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055877067148685455,
      "logits/rejected": 0.22418925166130066,
      "logps/chosen": -312.0386962890625,
      "logps/rejected": -394.556396484375,
      "loss": 0.6913,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007229042239487171,
      "rewards/margins": 0.0057485587894916534,
      "rewards/rejected": -0.01297760009765625,
      "step": 8
    },
    {
      "epoch": 0.0011777214361658624,
      "grad_norm": 4.253511428833008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2374730259180069,
      "logits/rejected": 0.18007442355155945,
      "logps/chosen": -458.3515625,
      "logps/rejected": -549.8235473632812,
      "loss": 0.6891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006969263311475515,
      "rewards/margins": 0.009076502174139023,
      "rewards/rejected": -0.002107238629832864,
      "step": 9
    },
    {
      "epoch": 0.0013085793735176249,
      "grad_norm": 4.711312294006348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08431166410446167,
      "logits/rejected": 0.27150270342826843,
      "logps/chosen": -456.0757141113281,
      "logps/rejected": -333.4166259765625,
      "loss": 0.6715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04525776207447052,
      "rewards/margins": 0.043826863169670105,
      "rewards/rejected": 0.0014308933168649673,
      "step": 10
    },
    {
      "epoch": 0.0014394373108693873,
      "grad_norm": 4.1482768058776855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3909122347831726,
      "logits/rejected": 0.44533202052116394,
      "logps/chosen": -403.6810607910156,
      "logps/rejected": -414.2901306152344,
      "loss": 0.6869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004221724346280098,
      "rewards/margins": 0.013609122484922409,
      "rewards/rejected": -0.009387396275997162,
      "step": 11
    },
    {
      "epoch": 0.00157029524822115,
      "grad_norm": 5.328396797180176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15476898849010468,
      "logits/rejected": 0.15321169793605804,
      "logps/chosen": -589.0208740234375,
      "logps/rejected": -426.51593017578125,
      "loss": 0.7181,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03008747100830078,
      "rewards/margins": -0.046694185584783554,
      "rewards/rejected": 0.016606714576482773,
      "step": 12
    },
    {
      "epoch": 0.0017011531855729125,
      "grad_norm": 3.2820634841918945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2813072204589844,
      "logits/rejected": 0.4703616201877594,
      "logps/chosen": -239.05091857910156,
      "logps/rejected": -332.4923095703125,
      "loss": 0.6806,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0052759163081645966,
      "rewards/margins": 0.025765802711248398,
      "rewards/rejected": -0.031041719019412994,
      "step": 13
    },
    {
      "epoch": 0.001832011122924675,
      "grad_norm": 4.906239986419678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30537691712379456,
      "logits/rejected": 0.10829132795333862,
      "logps/chosen": -543.1334228515625,
      "logps/rejected": -602.1573486328125,
      "loss": 0.6763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021294213831424713,
      "rewards/margins": 0.035765647888183594,
      "rewards/rejected": -0.01447143405675888,
      "step": 14
    },
    {
      "epoch": 0.0019628690602764374,
      "grad_norm": 4.4502692222595215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09916817396879196,
      "logits/rejected": -0.02464183047413826,
      "logps/chosen": -446.0805358886719,
      "logps/rejected": -374.2259216308594,
      "loss": 0.7053,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0021236413158476353,
      "rewards/margins": -0.0223909392952919,
      "rewards/rejected": 0.02451458014547825,
      "step": 15
    },
    {
      "epoch": 0.0020937269976282,
      "grad_norm": 3.5941359996795654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41770705580711365,
      "logits/rejected": 0.5505073070526123,
      "logps/chosen": -325.699462890625,
      "logps/rejected": -422.1822814941406,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007407760713249445,
      "rewards/margins": -0.0037878043949604034,
      "rewards/rejected": 0.011195564642548561,
      "step": 16
    },
    {
      "epoch": 0.0022245849349799623,
      "grad_norm": 3.5838825702667236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07911428809165955,
      "logits/rejected": 0.22571799159049988,
      "logps/chosen": -229.21578979492188,
      "logps/rejected": -264.6595764160156,
      "loss": 0.7024,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.005604267120361328,
      "rewards/margins": -0.016668125987052917,
      "rewards/rejected": 0.011063861660659313,
      "step": 17
    },
    {
      "epoch": 0.0023554428723317248,
      "grad_norm": 4.213283061981201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21488501131534576,
      "logits/rejected": 0.31080183386802673,
      "logps/chosen": -527.5712890625,
      "logps/rejected": -493.2065734863281,
      "loss": 0.6694,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01785736158490181,
      "rewards/margins": 0.04994926601648331,
      "rewards/rejected": -0.032091908156871796,
      "step": 18
    },
    {
      "epoch": 0.0024863008096834872,
      "grad_norm": 4.5876784324646,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28333523869514465,
      "logits/rejected": 0.2319900095462799,
      "logps/chosen": -633.0523681640625,
      "logps/rejected": -463.620849609375,
      "loss": 0.688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010850907303392887,
      "rewards/margins": 0.012945365160703659,
      "rewards/rejected": -0.002094459719955921,
      "step": 19
    },
    {
      "epoch": 0.0026171587470352497,
      "grad_norm": 4.435912132263184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13234278559684753,
      "logits/rejected": 0.29447507858276367,
      "logps/chosen": -429.19146728515625,
      "logps/rejected": -397.6658935546875,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00601043738424778,
      "rewards/margins": 0.014446067623794079,
      "rewards/rejected": -0.008435631170868874,
      "step": 20
    },
    {
      "epoch": 0.002748016684387012,
      "grad_norm": 4.771241188049316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6212067604064941,
      "logits/rejected": 0.3850190043449402,
      "logps/chosen": -457.8097839355469,
      "logps/rejected": -441.323974609375,
      "loss": 0.6982,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.029157066717743874,
      "rewards/margins": -0.008925436064600945,
      "rewards/rejected": -0.02023162692785263,
      "step": 21
    },
    {
      "epoch": 0.0028788746217387746,
      "grad_norm": 3.68045711517334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27745360136032104,
      "logits/rejected": 0.27240481972694397,
      "logps/chosen": -378.6035461425781,
      "logps/rejected": -335.9482421875,
      "loss": 0.6968,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.006327246781438589,
      "rewards/margins": -0.006311226636171341,
      "rewards/rejected": -1.6022473573684692e-05,
      "step": 22
    },
    {
      "epoch": 0.0030097325590905375,
      "grad_norm": 9.181048393249512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2762887179851532,
      "logits/rejected": 0.08594018220901489,
      "logps/chosen": -397.0447082519531,
      "logps/rejected": -436.3529357910156,
      "loss": 0.6835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.037613485008478165,
      "rewards/margins": 0.02069568634033203,
      "rewards/rejected": 0.016917802393436432,
      "step": 23
    },
    {
      "epoch": 0.0031405904964423,
      "grad_norm": 4.4781999588012695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19166429340839386,
      "logits/rejected": 0.14189594984054565,
      "logps/chosen": -541.4094848632812,
      "logps/rejected": -411.98260498046875,
      "loss": 0.6858,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023554230108857155,
      "rewards/margins": 0.018877316266298294,
      "rewards/rejected": 0.004676914773881435,
      "step": 24
    },
    {
      "epoch": 0.0032714484337940625,
      "grad_norm": 4.239091396331787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14996689558029175,
      "logits/rejected": 0.3212306499481201,
      "logps/chosen": -274.8697509765625,
      "logps/rejected": -388.9626770019531,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02066974714398384,
      "rewards/margins": 0.0023355456069111824,
      "rewards/rejected": 0.018334200605750084,
      "step": 25
    },
    {
      "epoch": 0.003402306371145825,
      "grad_norm": 4.362470626831055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49947598576545715,
      "logits/rejected": 0.23864203691482544,
      "logps/chosen": -507.2669677734375,
      "logps/rejected": -427.9361877441406,
      "loss": 0.7141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013267136178910732,
      "rewards/margins": -0.03732491284608841,
      "rewards/rejected": 0.05059204623103142,
      "step": 26
    },
    {
      "epoch": 0.0035331643084975874,
      "grad_norm": 5.165990829467773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15346910059452057,
      "logits/rejected": 0.37145742774009705,
      "logps/chosen": -494.79144287109375,
      "logps/rejected": -493.7478942871094,
      "loss": 0.6882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021247675642371178,
      "rewards/margins": 0.010299110785126686,
      "rewards/rejected": -0.031546782702207565,
      "step": 27
    },
    {
      "epoch": 0.00366402224584935,
      "grad_norm": 3.577435255050659,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44122225046157837,
      "logits/rejected": 0.35830238461494446,
      "logps/chosen": -297.5520935058594,
      "logps/rejected": -299.5623779296875,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0021184924989938736,
      "rewards/margins": 0.0032888432033360004,
      "rewards/rejected": -0.005407335236668587,
      "step": 28
    },
    {
      "epoch": 0.0037948801832011123,
      "grad_norm": 3.939171314239502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09026060253381729,
      "logits/rejected": -0.02577439695596695,
      "logps/chosen": -366.0101318359375,
      "logps/rejected": -391.14874267578125,
      "loss": 0.7011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009260749444365501,
      "rewards/margins": -0.015500640496611595,
      "rewards/rejected": 0.024761388078331947,
      "step": 29
    },
    {
      "epoch": 0.003925738120552875,
      "grad_norm": 4.80866813659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09240834414958954,
      "logits/rejected": -0.2480899542570114,
      "logps/chosen": -541.4590454101562,
      "logps/rejected": -502.7213134765625,
      "loss": 0.678,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.050672151148319244,
      "rewards/margins": 0.03107643499970436,
      "rewards/rejected": 0.019595719873905182,
      "step": 30
    },
    {
      "epoch": 0.004056596057904638,
      "grad_norm": 4.567424297332764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3814801573753357,
      "logits/rejected": 0.5344337821006775,
      "logps/chosen": -363.1371154785156,
      "logps/rejected": -466.00054931640625,
      "loss": 0.721,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009642220102250576,
      "rewards/margins": -0.052023887634277344,
      "rewards/rejected": 0.061666108667850494,
      "step": 31
    },
    {
      "epoch": 0.0041874539952564,
      "grad_norm": 4.460197925567627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4063534736633301,
      "logits/rejected": 0.4990983307361603,
      "logps/chosen": -365.4987487792969,
      "logps/rejected": -417.1683044433594,
      "loss": 0.6882,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.019475746899843216,
      "rewards/margins": 0.01232614554464817,
      "rewards/rejected": -0.031801894307136536,
      "step": 32
    },
    {
      "epoch": 0.004318311932608163,
      "grad_norm": 4.323782444000244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08003051578998566,
      "logits/rejected": 0.3539099097251892,
      "logps/chosen": -524.8880615234375,
      "logps/rejected": -486.9082336425781,
      "loss": 0.6776,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04752159118652344,
      "rewards/margins": 0.03257618099451065,
      "rewards/rejected": -0.08009777963161469,
      "step": 33
    },
    {
      "epoch": 0.004449169869959925,
      "grad_norm": 4.891851902008057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2396443635225296,
      "logits/rejected": 0.08699733018875122,
      "logps/chosen": -631.5333251953125,
      "logps/rejected": -548.5568237304688,
      "loss": 0.6565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07434120029211044,
      "rewards/margins": 0.076258085668087,
      "rewards/rejected": -0.0019168853759765625,
      "step": 34
    },
    {
      "epoch": 0.0045800278073116875,
      "grad_norm": 5.813004016876221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1057858094573021,
      "logits/rejected": 0.2161017656326294,
      "logps/chosen": -353.63250732421875,
      "logps/rejected": -361.7607116699219,
      "loss": 0.702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04904938116669655,
      "rewards/margins": -0.01602916792035103,
      "rewards/rejected": -0.03302021324634552,
      "step": 35
    },
    {
      "epoch": 0.0047108857446634496,
      "grad_norm": 4.474618434906006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3007143437862396,
      "logits/rejected": 0.39814460277557373,
      "logps/chosen": -382.59063720703125,
      "logps/rejected": -402.1508483886719,
      "loss": 0.6868,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012992098927497864,
      "rewards/margins": 0.01556834764778614,
      "rewards/rejected": -0.028560448437929153,
      "step": 36
    },
    {
      "epoch": 0.0048417436820152124,
      "grad_norm": 4.74944543838501,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35419362783432007,
      "logits/rejected": 0.27809494733810425,
      "logps/chosen": -510.1699523925781,
      "logps/rejected": -618.57421875,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025824928656220436,
      "rewards/margins": 0.006722829304635525,
      "rewards/rejected": 0.019102098420262337,
      "step": 37
    },
    {
      "epoch": 0.0049726016193669745,
      "grad_norm": 3.760690689086914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3597000241279602,
      "logits/rejected": 0.17115631699562073,
      "logps/chosen": -332.5480041503906,
      "logps/rejected": -280.7835998535156,
      "loss": 0.6581,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.003469275077804923,
      "rewards/margins": 0.07294673472642899,
      "rewards/rejected": -0.06947746872901917,
      "step": 38
    },
    {
      "epoch": 0.005103459556718737,
      "grad_norm": 4.815457820892334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2917459309101105,
      "logits/rejected": 0.12186439335346222,
      "logps/chosen": -513.7770385742188,
      "logps/rejected": -631.90380859375,
      "loss": 0.6992,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010833359323441982,
      "rewards/margins": -0.010678481310606003,
      "rewards/rejected": -0.00015487847849726677,
      "step": 39
    },
    {
      "epoch": 0.005234317494070499,
      "grad_norm": 4.445349216461182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2906126081943512,
      "logits/rejected": 0.40336301922798157,
      "logps/chosen": -582.8955688476562,
      "logps/rejected": -335.1579895019531,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05740528181195259,
      "rewards/margins": 0.01310109905898571,
      "rewards/rejected": -0.07050637900829315,
      "step": 40
    },
    {
      "epoch": 0.005365175431422262,
      "grad_norm": 5.319154739379883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08672136068344116,
      "logits/rejected": 0.14785397052764893,
      "logps/chosen": -390.6336669921875,
      "logps/rejected": -612.679443359375,
      "loss": 0.6883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.060314565896987915,
      "rewards/margins": 0.01097259670495987,
      "rewards/rejected": -0.07128715515136719,
      "step": 41
    },
    {
      "epoch": 0.005496033368774024,
      "grad_norm": 3.558258295059204,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15699131786823273,
      "logits/rejected": -0.05092620849609375,
      "logps/chosen": -267.23388671875,
      "logps/rejected": -377.9810485839844,
      "loss": 0.6701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01801891438663006,
      "rewards/margins": 0.04872627183794975,
      "rewards/rejected": -0.06674518436193466,
      "step": 42
    },
    {
      "epoch": 0.005626891306125787,
      "grad_norm": 4.721142768859863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1228008046746254,
      "logits/rejected": 0.04874851182103157,
      "logps/chosen": -425.9426574707031,
      "logps/rejected": -515.6366577148438,
      "loss": 0.683,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031795501708984375,
      "rewards/margins": 0.022620774805545807,
      "rewards/rejected": -0.05441627278923988,
      "step": 43
    },
    {
      "epoch": 0.005757749243477549,
      "grad_norm": 5.007089138031006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06280704587697983,
      "logits/rejected": -0.069880411028862,
      "logps/chosen": -441.1565246582031,
      "logps/rejected": -432.1254577636719,
      "loss": 0.7245,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08812656253576279,
      "rewards/margins": -0.05685272067785263,
      "rewards/rejected": -0.031273841857910156,
      "step": 44
    },
    {
      "epoch": 0.005888607180829312,
      "grad_norm": 4.012905597686768,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3671085238456726,
      "logits/rejected": 0.430165559053421,
      "logps/chosen": -275.6766357421875,
      "logps/rejected": -357.93328857421875,
      "loss": 0.6742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.058805085718631744,
      "rewards/margins": 0.040199846029281616,
      "rewards/rejected": -0.09900493919849396,
      "step": 45
    },
    {
      "epoch": 0.006019465118181075,
      "grad_norm": 4.2856059074401855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40006381273269653,
      "logits/rejected": 0.3669899106025696,
      "logps/chosen": -522.2506103515625,
      "logps/rejected": -386.48077392578125,
      "loss": 0.6437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.002447510603815317,
      "rewards/margins": 0.10364608466625214,
      "rewards/rejected": -0.10609360039234161,
      "step": 46
    },
    {
      "epoch": 0.006150323055532837,
      "grad_norm": 4.742467403411865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33155661821365356,
      "logits/rejected": 0.18379348516464233,
      "logps/chosen": -711.1583251953125,
      "logps/rejected": -677.4453735351562,
      "loss": 0.6977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06031780317425728,
      "rewards/margins": -0.007233620621263981,
      "rewards/rejected": -0.053084179759025574,
      "step": 47
    },
    {
      "epoch": 0.0062811809928846,
      "grad_norm": 4.326780796051025,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23393186926841736,
      "logits/rejected": 0.4307714104652405,
      "logps/chosen": -327.5823669433594,
      "logps/rejected": -387.9642639160156,
      "loss": 0.6509,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004347324836999178,
      "rewards/margins": 0.09040365368127823,
      "rewards/rejected": -0.08605632930994034,
      "step": 48
    },
    {
      "epoch": 0.006412038930236362,
      "grad_norm": 4.050255298614502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18488770723342896,
      "logits/rejected": 0.35120636224746704,
      "logps/chosen": -429.7603759765625,
      "logps/rejected": -397.48907470703125,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02003498002886772,
      "rewards/margins": 0.0004627211019396782,
      "rewards/rejected": -0.020497702062129974,
      "step": 49
    },
    {
      "epoch": 0.006542896867588125,
      "grad_norm": 4.495725631713867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46305277943611145,
      "logits/rejected": 0.38554102182388306,
      "logps/chosen": -448.8554992675781,
      "logps/rejected": -395.7950744628906,
      "loss": 0.6629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0468175895512104,
      "rewards/margins": 0.06481209397315979,
      "rewards/rejected": -0.1116296723484993,
      "step": 50
    },
    {
      "epoch": 0.006673754804939887,
      "grad_norm": 4.668083667755127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25148117542266846,
      "logits/rejected": 0.39018550515174866,
      "logps/chosen": -543.184326171875,
      "logps/rejected": -505.0086669921875,
      "loss": 0.6885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07143783569335938,
      "rewards/margins": 0.015693094581365585,
      "rewards/rejected": -0.08713092654943466,
      "step": 51
    },
    {
      "epoch": 0.00680461274229165,
      "grad_norm": 4.272920608520508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38455212116241455,
      "logits/rejected": 0.3207390308380127,
      "logps/chosen": -508.409423828125,
      "logps/rejected": -371.3665771484375,
      "loss": 0.674,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017963597550988197,
      "rewards/margins": 0.040291979908943176,
      "rewards/rejected": -0.058255575597286224,
      "step": 52
    },
    {
      "epoch": 0.006935470679643412,
      "grad_norm": 5.1063127517700195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012591050937771797,
      "logits/rejected": 0.20418870449066162,
      "logps/chosen": -450.5303039550781,
      "logps/rejected": -580.2407836914062,
      "loss": 0.707,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04129056632518768,
      "rewards/margins": -0.017147347331047058,
      "rewards/rejected": -0.024143218994140625,
      "step": 53
    },
    {
      "epoch": 0.007066328616995175,
      "grad_norm": 4.915628910064697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07999907433986664,
      "logits/rejected": 0.32775402069091797,
      "logps/chosen": -595.6427001953125,
      "logps/rejected": -442.6011962890625,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07125435769557953,
      "rewards/margins": -0.005262184888124466,
      "rewards/rejected": -0.06599216163158417,
      "step": 54
    },
    {
      "epoch": 0.007197186554346937,
      "grad_norm": 9.419629096984863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24950432777404785,
      "logits/rejected": 0.20741130411624908,
      "logps/chosen": -407.393798828125,
      "logps/rejected": -554.8449096679688,
      "loss": 0.7059,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10764827579259872,
      "rewards/margins": -0.0239015594124794,
      "rewards/rejected": -0.08374671638011932,
      "step": 55
    },
    {
      "epoch": 0.0073280444916987,
      "grad_norm": 3.9790608882904053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5687007308006287,
      "logits/rejected": 0.5302609801292419,
      "logps/chosen": -565.6495361328125,
      "logps/rejected": -293.4624328613281,
      "loss": 0.6208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.022330282256007195,
      "rewards/margins": 0.15978986024856567,
      "rewards/rejected": -0.18212014436721802,
      "step": 56
    },
    {
      "epoch": 0.007458902429050462,
      "grad_norm": 4.128306865692139,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34895992279052734,
      "logits/rejected": 0.14184966683387756,
      "logps/chosen": -264.2990417480469,
      "logps/rejected": -279.65899658203125,
      "loss": 0.6838,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0043823253363370895,
      "rewards/margins": 0.02052479237318039,
      "rewards/rejected": -0.01614246517419815,
      "step": 57
    },
    {
      "epoch": 0.007589760366402225,
      "grad_norm": 5.910367488861084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32420119643211365,
      "logits/rejected": 0.0802418515086174,
      "logps/chosen": -756.112548828125,
      "logps/rejected": -613.457763671875,
      "loss": 0.7212,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20174658298492432,
      "rewards/margins": -0.04545840620994568,
      "rewards/rejected": -0.15628814697265625,
      "step": 58
    },
    {
      "epoch": 0.007720618303753987,
      "grad_norm": 5.326003551483154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3374771773815155,
      "logits/rejected": 0.24339434504508972,
      "logps/chosen": -770.984375,
      "logps/rejected": -556.7044067382812,
      "loss": 0.6079,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02362842485308647,
      "rewards/margins": 0.20520401000976562,
      "rewards/rejected": -0.18157559633255005,
      "step": 59
    },
    {
      "epoch": 0.00785147624110575,
      "grad_norm": 5.055182933807373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2797171473503113,
      "logits/rejected": 0.33567163348197937,
      "logps/chosen": -596.4898071289062,
      "logps/rejected": -626.381103515625,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1541953980922699,
      "rewards/margins": 0.024938207119703293,
      "rewards/rejected": -0.1791336089372635,
      "step": 60
    },
    {
      "epoch": 0.007982334178457512,
      "grad_norm": 4.177269458770752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30341944098472595,
      "logits/rejected": 0.13601791858673096,
      "logps/chosen": -471.1578369140625,
      "logps/rejected": -417.69427490234375,
      "loss": 0.6845,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03839416429400444,
      "rewards/margins": 0.022845840081572533,
      "rewards/rejected": -0.06124000996351242,
      "step": 61
    },
    {
      "epoch": 0.008113192115809275,
      "grad_norm": 3.912862539291382,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3048654794692993,
      "logits/rejected": 0.2515895664691925,
      "logps/chosen": -361.2912902832031,
      "logps/rejected": -235.46414184570312,
      "loss": 0.6882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.037020113319158554,
      "rewards/margins": 0.011501695029437542,
      "rewards/rejected": -0.04852180927991867,
      "step": 62
    },
    {
      "epoch": 0.008244050053161037,
      "grad_norm": 5.125492572784424,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5390836596488953,
      "logits/rejected": 0.410571813583374,
      "logps/chosen": -539.4088134765625,
      "logps/rejected": -518.030517578125,
      "loss": 0.659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04832287132740021,
      "rewards/margins": 0.07187405228614807,
      "rewards/rejected": -0.12019691616296768,
      "step": 63
    },
    {
      "epoch": 0.0083749079905128,
      "grad_norm": 4.907171249389648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02198125794529915,
      "logits/rejected": 0.08584921807050705,
      "logps/chosen": -485.9099426269531,
      "logps/rejected": -493.1305236816406,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08617401123046875,
      "rewards/margins": 0.004416652023792267,
      "rewards/rejected": -0.09059067070484161,
      "step": 64
    },
    {
      "epoch": 0.008505765927864561,
      "grad_norm": 3.762979507446289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3425368666648865,
      "logits/rejected": 0.15109968185424805,
      "logps/chosen": -349.31573486328125,
      "logps/rejected": -318.93328857421875,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12296333909034729,
      "rewards/margins": 0.0244230218231678,
      "rewards/rejected": -0.1473863571882248,
      "step": 65
    },
    {
      "epoch": 0.008636623865216325,
      "grad_norm": 5.415464878082275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17358481884002686,
      "logits/rejected": -0.1355375051498413,
      "logps/chosen": -608.5418090820312,
      "logps/rejected": -524.892578125,
      "loss": 0.6845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.043917085975408554,
      "rewards/margins": 0.021864894777536392,
      "rewards/rejected": 0.022052191197872162,
      "step": 66
    },
    {
      "epoch": 0.008767481802568087,
      "grad_norm": 5.462266445159912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030437655746936798,
      "logits/rejected": 0.2971053421497345,
      "logps/chosen": -375.10137939453125,
      "logps/rejected": -552.265380859375,
      "loss": 0.7073,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.076696015894413,
      "rewards/margins": -0.023814018815755844,
      "rewards/rejected": -0.052882008254528046,
      "step": 67
    },
    {
      "epoch": 0.00889833973991985,
      "grad_norm": 4.239383697509766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3074442148208618,
      "logits/rejected": 0.353371798992157,
      "logps/chosen": -339.1686096191406,
      "logps/rejected": -336.4491882324219,
      "loss": 0.6853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04357089847326279,
      "rewards/margins": 0.020442677661776543,
      "rewards/rejected": -0.06401357799768448,
      "step": 68
    },
    {
      "epoch": 0.009029197677271611,
      "grad_norm": 5.574032783508301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5594172477722168,
      "logits/rejected": 0.4374110698699951,
      "logps/chosen": -581.3301391601562,
      "logps/rejected": -491.4036560058594,
      "loss": 0.7006,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1336912214756012,
      "rewards/margins": -0.010267635807394981,
      "rewards/rejected": -0.12342357635498047,
      "step": 69
    },
    {
      "epoch": 0.009160055614623375,
      "grad_norm": 5.443347930908203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4404604434967041,
      "logits/rejected": 0.19518578052520752,
      "logps/chosen": -685.9243774414062,
      "logps/rejected": -719.7042236328125,
      "loss": 0.6926,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.029776960611343384,
      "rewards/margins": 0.023143954575061798,
      "rewards/rejected": -0.05292091518640518,
      "step": 70
    },
    {
      "epoch": 0.009290913551975137,
      "grad_norm": 4.8397417068481445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021576061844825745,
      "logits/rejected": -0.10298702865839005,
      "logps/chosen": -578.3907470703125,
      "logps/rejected": -429.1872253417969,
      "loss": 0.6606,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10595112293958664,
      "rewards/margins": 0.07816428691148758,
      "rewards/rejected": -0.18411540985107422,
      "step": 71
    },
    {
      "epoch": 0.009421771489326899,
      "grad_norm": 4.325377464294434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3032209873199463,
      "logits/rejected": 0.2591268718242645,
      "logps/chosen": -386.39910888671875,
      "logps/rejected": -419.31439208984375,
      "loss": 0.6217,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0005350112915039062,
      "rewards/margins": 0.15474966168403625,
      "rewards/rejected": -0.15528470277786255,
      "step": 72
    },
    {
      "epoch": 0.009552629426678661,
      "grad_norm": 4.438838481903076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31216245889663696,
      "logits/rejected": 0.4660393297672272,
      "logps/chosen": -416.496826171875,
      "logps/rejected": -476.2098693847656,
      "loss": 0.681,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12011031806468964,
      "rewards/margins": 0.02787799760699272,
      "rewards/rejected": -0.14798831939697266,
      "step": 73
    },
    {
      "epoch": 0.009683487364030425,
      "grad_norm": 5.023260593414307,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3395363688468933,
      "logits/rejected": 0.3543981611728668,
      "logps/chosen": -486.50128173828125,
      "logps/rejected": -481.7796325683594,
      "loss": 0.6583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11466598510742188,
      "rewards/margins": 0.07474327832460403,
      "rewards/rejected": -0.1894092708826065,
      "step": 74
    },
    {
      "epoch": 0.009814345301382187,
      "grad_norm": 4.7433013916015625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3733667731285095,
      "logits/rejected": 0.2447236329317093,
      "logps/chosen": -460.9827575683594,
      "logps/rejected": -526.0267333984375,
      "loss": 0.7012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0918310135602951,
      "rewards/margins": -0.008136169984936714,
      "rewards/rejected": -0.08369484543800354,
      "step": 75
    },
    {
      "epoch": 0.009945203238733949,
      "grad_norm": 4.486622333526611,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16644200682640076,
      "logits/rejected": 0.30536043643951416,
      "logps/chosen": -570.5712890625,
      "logps/rejected": -366.0877685546875,
      "loss": 0.6794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.042233657091856,
      "rewards/margins": 0.029487989842891693,
      "rewards/rejected": -0.07172165811061859,
      "step": 76
    },
    {
      "epoch": 0.010076061176085713,
      "grad_norm": 5.493346214294434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3364584743976593,
      "logits/rejected": 0.3392626941204071,
      "logps/chosen": -522.8375244140625,
      "logps/rejected": -579.5823974609375,
      "loss": 0.7501,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.11192568391561508,
      "rewards/margins": -0.09972935914993286,
      "rewards/rejected": -0.012196349911391735,
      "step": 77
    },
    {
      "epoch": 0.010206919113437475,
      "grad_norm": 5.078700065612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.285831481218338,
      "logits/rejected": 0.12670032680034637,
      "logps/chosen": -583.9414672851562,
      "logps/rejected": -376.6417541503906,
      "loss": 0.7402,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1975250393152237,
      "rewards/margins": -0.07462826371192932,
      "rewards/rejected": -0.12289677560329437,
      "step": 78
    },
    {
      "epoch": 0.010337777050789237,
      "grad_norm": 4.987334251403809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4369807243347168,
      "logits/rejected": 0.5604466795921326,
      "logps/chosen": -357.1953125,
      "logps/rejected": -401.21905517578125,
      "loss": 0.7379,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11760462820529938,
      "rewards/margins": -0.06767559051513672,
      "rewards/rejected": -0.049929048866033554,
      "step": 79
    },
    {
      "epoch": 0.010468634988140999,
      "grad_norm": 5.883652687072754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3595871925354004,
      "logits/rejected": 0.1665538251399994,
      "logps/chosen": -457.03424072265625,
      "logps/rejected": -336.2619934082031,
      "loss": 0.7409,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.10561923682689667,
      "rewards/margins": -0.0830957442522049,
      "rewards/rejected": -0.02252349816262722,
      "step": 80
    },
    {
      "epoch": 0.010599492925492763,
      "grad_norm": 5.38502311706543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.606387734413147,
      "logits/rejected": 0.17944875359535217,
      "logps/chosen": -522.596435546875,
      "logps/rejected": -621.0318603515625,
      "loss": 0.7451,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.22202548384666443,
      "rewards/margins": -0.0863979384303093,
      "rewards/rejected": -0.13562755286693573,
      "step": 81
    },
    {
      "epoch": 0.010730350862844525,
      "grad_norm": 4.658065319061279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32372403144836426,
      "logits/rejected": 0.43408557772636414,
      "logps/chosen": -403.7944030761719,
      "logps/rejected": -444.5059509277344,
      "loss": 0.6704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1739627867937088,
      "rewards/margins": 0.04829730838537216,
      "rewards/rejected": -0.22226008772850037,
      "step": 82
    },
    {
      "epoch": 0.010861208800196287,
      "grad_norm": 4.333441734313965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2980445623397827,
      "logits/rejected": 0.2827475666999817,
      "logps/chosen": -363.2199401855469,
      "logps/rejected": -432.2488708496094,
      "loss": 0.7174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07552775740623474,
      "rewards/margins": -0.034290313720703125,
      "rewards/rejected": -0.04123745113611221,
      "step": 83
    },
    {
      "epoch": 0.010992066737548049,
      "grad_norm": 4.515619277954102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.589389979839325,
      "logits/rejected": 0.43855684995651245,
      "logps/chosen": -481.41021728515625,
      "logps/rejected": -301.5770263671875,
      "loss": 0.6615,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06796703487634659,
      "rewards/margins": 0.07240256667137146,
      "rewards/rejected": -0.14036960899829865,
      "step": 84
    },
    {
      "epoch": 0.011122924674899812,
      "grad_norm": 5.1704511642456055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2997265160083771,
      "logits/rejected": -0.03348863124847412,
      "logps/chosen": -567.7769775390625,
      "logps/rejected": -377.5173645019531,
      "loss": 0.6313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07113400101661682,
      "rewards/margins": 0.13725757598876953,
      "rewards/rejected": -0.20839157700538635,
      "step": 85
    },
    {
      "epoch": 0.011253782612251574,
      "grad_norm": 3.99615740776062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10160573571920395,
      "logits/rejected": 0.1969631165266037,
      "logps/chosen": -373.70709228515625,
      "logps/rejected": -400.31451416015625,
      "loss": 0.6689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10770435631275177,
      "rewards/margins": 0.05608825013041496,
      "rewards/rejected": -0.16379261016845703,
      "step": 86
    },
    {
      "epoch": 0.011384640549603336,
      "grad_norm": 4.161011219024658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14412669837474823,
      "logits/rejected": 0.13424117863178253,
      "logps/chosen": -526.6570434570312,
      "logps/rejected": -425.50262451171875,
      "loss": 0.6457,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21888485550880432,
      "rewards/margins": 0.11455173045396805,
      "rewards/rejected": -0.3334365487098694,
      "step": 87
    },
    {
      "epoch": 0.011515498486955099,
      "grad_norm": 3.8250722885131836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03835221752524376,
      "logits/rejected": -0.04153617471456528,
      "logps/chosen": -351.63714599609375,
      "logps/rejected": -368.1717224121094,
      "loss": 0.6699,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.052980996668338776,
      "rewards/margins": 0.051076892763376236,
      "rewards/rejected": -0.10405788570642471,
      "step": 88
    },
    {
      "epoch": 0.011646356424306862,
      "grad_norm": 4.2553582191467285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25609421730041504,
      "logits/rejected": 0.48623472452163696,
      "logps/chosen": -425.5670471191406,
      "logps/rejected": -359.0299072265625,
      "loss": 0.6851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20239582657814026,
      "rewards/margins": 0.017786595970392227,
      "rewards/rejected": -0.2201824188232422,
      "step": 89
    },
    {
      "epoch": 0.011777214361658624,
      "grad_norm": 4.798654079437256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4355543851852417,
      "logits/rejected": 0.4555695950984955,
      "logps/chosen": -486.6609191894531,
      "logps/rejected": -479.5500183105469,
      "loss": 0.6514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13459300994873047,
      "rewards/margins": 0.09480535238981247,
      "rewards/rejected": -0.22939833998680115,
      "step": 90
    },
    {
      "epoch": 0.011908072299010386,
      "grad_norm": 4.127087116241455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24924181401729584,
      "logits/rejected": 0.08054346591234207,
      "logps/chosen": -437.6323547363281,
      "logps/rejected": -422.84393310546875,
      "loss": 0.644,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05938110500574112,
      "rewards/margins": 0.10245761275291443,
      "rewards/rejected": -0.16183872520923615,
      "step": 91
    },
    {
      "epoch": 0.01203893023636215,
      "grad_norm": 4.919739723205566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15634651482105255,
      "logits/rejected": 0.2655249536037445,
      "logps/chosen": -458.50836181640625,
      "logps/rejected": -647.3480834960938,
      "loss": 0.6999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1005500853061676,
      "rewards/margins": -0.007173351012170315,
      "rewards/rejected": -0.09337673336267471,
      "step": 92
    },
    {
      "epoch": 0.012169788173713912,
      "grad_norm": 4.4528422355651855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1069018617272377,
      "logits/rejected": 0.11509416252374649,
      "logps/chosen": -431.93463134765625,
      "logps/rejected": -324.4208068847656,
      "loss": 0.6379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15376168489456177,
      "rewards/margins": 0.12826117873191833,
      "rewards/rejected": -0.2820228636264801,
      "step": 93
    },
    {
      "epoch": 0.012300646111065674,
      "grad_norm": 4.54075288772583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2113036811351776,
      "logits/rejected": 0.12645356357097626,
      "logps/chosen": -458.4559020996094,
      "logps/rejected": -423.3515625,
      "loss": 0.643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03538799285888672,
      "rewards/margins": 0.13521462678909302,
      "rewards/rejected": -0.17060260474681854,
      "step": 94
    },
    {
      "epoch": 0.012431504048417436,
      "grad_norm": 5.51557731628418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4115981161594391,
      "logits/rejected": 0.3917279541492462,
      "logps/chosen": -494.1802062988281,
      "logps/rejected": -817.0531005859375,
      "loss": 0.6873,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13647803664207458,
      "rewards/margins": 0.02303161844611168,
      "rewards/rejected": -0.15950965881347656,
      "step": 95
    },
    {
      "epoch": 0.0125623619857692,
      "grad_norm": 3.4913198947906494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32294055819511414,
      "logits/rejected": 0.22433960437774658,
      "logps/chosen": -342.2097473144531,
      "logps/rejected": -326.3746643066406,
      "loss": 0.6413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05781078338623047,
      "rewards/margins": 0.11050014197826385,
      "rewards/rejected": -0.16831094026565552,
      "step": 96
    },
    {
      "epoch": 0.012693219923120962,
      "grad_norm": 4.488138675689697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07737385481595993,
      "logits/rejected": 0.04746689647436142,
      "logps/chosen": -432.6883544921875,
      "logps/rejected": -460.5835266113281,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15423735976219177,
      "rewards/margins": 0.013614274561405182,
      "rewards/rejected": -0.16785165667533875,
      "step": 97
    },
    {
      "epoch": 0.012824077860472724,
      "grad_norm": 3.8368000984191895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31634533405303955,
      "logits/rejected": 0.2283225655555725,
      "logps/chosen": -349.7532653808594,
      "logps/rejected": -332.5409851074219,
      "loss": 0.689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25433140993118286,
      "rewards/margins": 0.016295433044433594,
      "rewards/rejected": -0.27062684297561646,
      "step": 98
    },
    {
      "epoch": 0.012954935797824486,
      "grad_norm": 4.502670764923096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009375087916851044,
      "logits/rejected": 0.22318363189697266,
      "logps/chosen": -454.9289245605469,
      "logps/rejected": -386.88970947265625,
      "loss": 0.6308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05866088718175888,
      "rewards/margins": 0.13194961845874786,
      "rewards/rejected": -0.19061049818992615,
      "step": 99
    },
    {
      "epoch": 0.01308579373517625,
      "grad_norm": 4.768619060516357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4013669490814209,
      "logits/rejected": 0.3905637264251709,
      "logps/chosen": -694.21923828125,
      "logps/rejected": -619.2630615234375,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05304031819105148,
      "rewards/margins": 0.015190888196229935,
      "rewards/rejected": -0.06823120266199112,
      "step": 100
    },
    {
      "epoch": 0.013216651672528012,
      "grad_norm": 4.6215362548828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25840306282043457,
      "logits/rejected": 0.2513318359851837,
      "logps/chosen": -437.29852294921875,
      "logps/rejected": -495.24505615234375,
      "loss": 0.6633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1398511826992035,
      "rewards/margins": 0.07111856341362,
      "rewards/rejected": -0.2109697461128235,
      "step": 101
    },
    {
      "epoch": 0.013347509609879774,
      "grad_norm": 4.321023464202881,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37326139211654663,
      "logits/rejected": 0.31426966190338135,
      "logps/chosen": -432.6902770996094,
      "logps/rejected": -358.0008544921875,
      "loss": 0.6262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.127574160695076,
      "rewards/margins": 0.16196900606155396,
      "rewards/rejected": -0.28954315185546875,
      "step": 102
    },
    {
      "epoch": 0.013478367547231536,
      "grad_norm": 5.0519700050354,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45357653498649597,
      "logits/rejected": 0.6501546502113342,
      "logps/chosen": -521.2735595703125,
      "logps/rejected": -447.5411376953125,
      "loss": 0.6395,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1259765625,
      "rewards/margins": 0.12041854858398438,
      "rewards/rejected": -0.24639511108398438,
      "step": 103
    },
    {
      "epoch": 0.0136092254845833,
      "grad_norm": 4.254510879516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2071772664785385,
      "logits/rejected": 0.259492427110672,
      "logps/chosen": -392.37139892578125,
      "logps/rejected": -334.9525451660156,
      "loss": 0.6836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17174969613552094,
      "rewards/margins": 0.0235111303627491,
      "rewards/rejected": -0.19526082277297974,
      "step": 104
    },
    {
      "epoch": 0.013740083421935062,
      "grad_norm": 4.735138416290283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30530259013175964,
      "logits/rejected": 0.5116562247276306,
      "logps/chosen": -499.4810485839844,
      "logps/rejected": -437.1482238769531,
      "loss": 0.7197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13914966583251953,
      "rewards/margins": -0.040247149765491486,
      "rewards/rejected": -0.09890253096818924,
      "step": 105
    },
    {
      "epoch": 0.013870941359286824,
      "grad_norm": 4.330573558807373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4398939311504364,
      "logits/rejected": 0.22020024061203003,
      "logps/chosen": -470.0017395019531,
      "logps/rejected": -351.70721435546875,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13443279266357422,
      "rewards/margins": 0.018519973382353783,
      "rewards/rejected": -0.15295277535915375,
      "step": 106
    },
    {
      "epoch": 0.014001799296638587,
      "grad_norm": 4.453364372253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4848306477069855,
      "logits/rejected": 0.5989286303520203,
      "logps/chosen": -622.597412109375,
      "logps/rejected": -520.619140625,
      "loss": 0.5894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.025253865867853165,
      "rewards/margins": 0.23152771592140198,
      "rewards/rejected": -0.2062738537788391,
      "step": 107
    },
    {
      "epoch": 0.01413265723399035,
      "grad_norm": 5.029712677001953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3881509304046631,
      "logits/rejected": 0.30735906958580017,
      "logps/chosen": -515.3121337890625,
      "logps/rejected": -523.190673828125,
      "loss": 0.7027,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1461540162563324,
      "rewards/margins": -0.008754543028771877,
      "rewards/rejected": -0.13739949464797974,
      "step": 108
    },
    {
      "epoch": 0.014263515171342112,
      "grad_norm": 4.564441680908203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04395703971385956,
      "logits/rejected": 0.30571216344833374,
      "logps/chosen": -693.8758544921875,
      "logps/rejected": -370.2099609375,
      "loss": 0.6421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04758262634277344,
      "rewards/margins": 0.10832138359546661,
      "rewards/rejected": -0.15590400993824005,
      "step": 109
    },
    {
      "epoch": 0.014394373108693874,
      "grad_norm": 4.710188865661621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29507941007614136,
      "logits/rejected": 0.323211133480072,
      "logps/chosen": -382.6880798339844,
      "logps/rejected": -530.119140625,
      "loss": 0.7326,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.315052330493927,
      "rewards/margins": -0.06914015114307404,
      "rewards/rejected": -0.24591217935085297,
      "step": 110
    },
    {
      "epoch": 0.014525231046045637,
      "grad_norm": 4.3353095054626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008349008858203888,
      "logits/rejected": 0.06668437272310257,
      "logps/chosen": -554.504638671875,
      "logps/rejected": -412.4930725097656,
      "loss": 0.6621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05069980025291443,
      "rewards/margins": 0.07473811507225037,
      "rewards/rejected": -0.125437930226326,
      "step": 111
    },
    {
      "epoch": 0.0146560889833974,
      "grad_norm": 5.361558437347412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29917585849761963,
      "logits/rejected": 0.16190744936466217,
      "logps/chosen": -385.6678771972656,
      "logps/rejected": -530.8809204101562,
      "loss": 0.6841,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08654594421386719,
      "rewards/margins": 0.03433895856142044,
      "rewards/rejected": -0.12088489532470703,
      "step": 112
    },
    {
      "epoch": 0.014786946920749161,
      "grad_norm": 5.0383453369140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2982660233974457,
      "logits/rejected": 0.3400033116340637,
      "logps/chosen": -528.0137939453125,
      "logps/rejected": -453.9364318847656,
      "loss": 0.6884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25337716937065125,
      "rewards/margins": 0.018196485936641693,
      "rewards/rejected": -0.27157366275787354,
      "step": 113
    },
    {
      "epoch": 0.014917804858100923,
      "grad_norm": 4.373133659362793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3518742024898529,
      "logits/rejected": 0.1662548929452896,
      "logps/chosen": -422.1090087890625,
      "logps/rejected": -423.24871826171875,
      "loss": 0.6615,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21317215263843536,
      "rewards/margins": 0.07732105255126953,
      "rewards/rejected": -0.2904932200908661,
      "step": 114
    },
    {
      "epoch": 0.015048662795452687,
      "grad_norm": 4.676416397094727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5084983110427856,
      "logits/rejected": 0.3342590034008026,
      "logps/chosen": -440.2375793457031,
      "logps/rejected": -452.7659912109375,
      "loss": 0.6543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18372783064842224,
      "rewards/margins": 0.08166980743408203,
      "rewards/rejected": -0.26539766788482666,
      "step": 115
    },
    {
      "epoch": 0.01517952073280445,
      "grad_norm": 4.536140441894531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.056345902383327484,
      "logits/rejected": 0.33735308051109314,
      "logps/chosen": -317.6902160644531,
      "logps/rejected": -414.8197021484375,
      "loss": 0.6284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2484910935163498,
      "rewards/margins": 0.14324168860912323,
      "rewards/rejected": -0.3917328119277954,
      "step": 116
    },
    {
      "epoch": 0.015310378670156211,
      "grad_norm": 5.756826400756836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42474547028541565,
      "logits/rejected": 0.43394461274147034,
      "logps/chosen": -420.3377685546875,
      "logps/rejected": -498.755615234375,
      "loss": 0.6799,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.062026023864746094,
      "rewards/margins": 0.03619404137134552,
      "rewards/rejected": -0.09822007268667221,
      "step": 117
    },
    {
      "epoch": 0.015441236607507973,
      "grad_norm": 4.391515254974365,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11992591619491577,
      "logits/rejected": 0.41974812746047974,
      "logps/chosen": -389.3403015136719,
      "logps/rejected": -354.1685485839844,
      "loss": 0.6299,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16332580149173737,
      "rewards/margins": 0.13657522201538086,
      "rewards/rejected": -0.2999010384082794,
      "step": 118
    },
    {
      "epoch": 0.015572094544859737,
      "grad_norm": 5.509573459625244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2876776456832886,
      "logits/rejected": 0.5793942809104919,
      "logps/chosen": -603.17724609375,
      "logps/rejected": -493.7462158203125,
      "loss": 0.7655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14991284906864166,
      "rewards/margins": -0.1112319827079773,
      "rewards/rejected": -0.03868084028363228,
      "step": 119
    },
    {
      "epoch": 0.0157029524822115,
      "grad_norm": 5.5172200202941895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33876729011535645,
      "logits/rejected": 0.16234789788722992,
      "logps/chosen": -646.662841796875,
      "logps/rejected": -753.0335693359375,
      "loss": 0.6513,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2992347776889801,
      "rewards/margins": 0.10627860575914383,
      "rewards/rejected": -0.4055134057998657,
      "step": 120
    },
    {
      "epoch": 0.015833810419563263,
      "grad_norm": 4.3665690422058105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2466317117214203,
      "logits/rejected": 0.197279691696167,
      "logps/chosen": -439.50225830078125,
      "logps/rejected": -399.4263610839844,
      "loss": 0.609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04798393324017525,
      "rewards/margins": 0.18731346726417542,
      "rewards/rejected": -0.23529739677906036,
      "step": 121
    },
    {
      "epoch": 0.015964668356915023,
      "grad_norm": 3.2235231399536133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3947415053844452,
      "logits/rejected": 0.40341511368751526,
      "logps/chosen": -339.8936767578125,
      "logps/rejected": -299.99169921875,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1183418408036232,
      "rewards/margins": 0.018645666539669037,
      "rewards/rejected": -0.13698749244213104,
      "step": 122
    },
    {
      "epoch": 0.016095526294266787,
      "grad_norm": 4.027682304382324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25956085324287415,
      "logits/rejected": 0.30468761920928955,
      "logps/chosen": -478.691650390625,
      "logps/rejected": -290.4677429199219,
      "loss": 0.6532,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2097984254360199,
      "rewards/margins": 0.10971049219369888,
      "rewards/rejected": -0.31950893998146057,
      "step": 123
    },
    {
      "epoch": 0.01622638423161855,
      "grad_norm": 3.503437042236328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3351380228996277,
      "logits/rejected": 0.6112049221992493,
      "logps/chosen": -280.9819641113281,
      "logps/rejected": -281.4586486816406,
      "loss": 0.6683,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11548519134521484,
      "rewards/margins": 0.05448436737060547,
      "rewards/rejected": -0.1699695587158203,
      "step": 124
    },
    {
      "epoch": 0.01635724216897031,
      "grad_norm": 4.377984046936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3188759982585907,
      "logits/rejected": 0.26932746171951294,
      "logps/chosen": -542.79345703125,
      "logps/rejected": -475.9574890136719,
      "loss": 0.5886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0016113296151161194,
      "rewards/margins": 0.2585090696811676,
      "rewards/rejected": -0.26012036204338074,
      "step": 125
    },
    {
      "epoch": 0.016488100106322075,
      "grad_norm": 3.776169776916504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5250689387321472,
      "logits/rejected": 0.5339680910110474,
      "logps/chosen": -335.7232666015625,
      "logps/rejected": -363.1050109863281,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1255718320608139,
      "rewards/margins": 0.018306730315089226,
      "rewards/rejected": -0.14387856423854828,
      "step": 126
    },
    {
      "epoch": 0.016618958043673835,
      "grad_norm": 4.861093044281006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27994781732559204,
      "logits/rejected": 0.4008841812610626,
      "logps/chosen": -393.5257568359375,
      "logps/rejected": -367.7913513183594,
      "loss": 0.7035,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.08557643741369247,
      "rewards/margins": -0.007211778312921524,
      "rewards/rejected": -0.07836465537548065,
      "step": 127
    },
    {
      "epoch": 0.0167498159810256,
      "grad_norm": 4.76942777633667,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27817386388778687,
      "logits/rejected": 0.15012601017951965,
      "logps/chosen": -562.1429443359375,
      "logps/rejected": -356.7657165527344,
      "loss": 0.6419,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20396631956100464,
      "rewards/margins": 0.12629450857639313,
      "rewards/rejected": -0.33026087284088135,
      "step": 128
    },
    {
      "epoch": 0.016880673918377363,
      "grad_norm": 3.6870884895324707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18304334580898285,
      "logits/rejected": 0.08761835843324661,
      "logps/chosen": -365.57568359375,
      "logps/rejected": -318.6695556640625,
      "loss": 0.6233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11976069957017899,
      "rewards/margins": 0.1560390293598175,
      "rewards/rejected": -0.2757997512817383,
      "step": 129
    },
    {
      "epoch": 0.017011531855729123,
      "grad_norm": 4.303879737854004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26387786865234375,
      "logits/rejected": 0.07367696613073349,
      "logps/chosen": -548.4412841796875,
      "logps/rejected": -465.06890869140625,
      "loss": 0.6347,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2646644413471222,
      "rewards/margins": 0.13517969846725464,
      "rewards/rejected": -0.3998441994190216,
      "step": 130
    },
    {
      "epoch": 0.017142389793080887,
      "grad_norm": 4.680978298187256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12531563639640808,
      "logits/rejected": -0.14538174867630005,
      "logps/chosen": -393.6585693359375,
      "logps/rejected": -466.1787414550781,
      "loss": 0.6199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05934924632310867,
      "rewards/margins": 0.1703590601682663,
      "rewards/rejected": -0.22970829904079437,
      "step": 131
    },
    {
      "epoch": 0.01727324773043265,
      "grad_norm": 4.205098628997803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1968822181224823,
      "logits/rejected": 0.15980933606624603,
      "logps/chosen": -343.87054443359375,
      "logps/rejected": -415.7183532714844,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1448826789855957,
      "rewards/margins": 0.004851631820201874,
      "rewards/rejected": -0.14973430335521698,
      "step": 132
    },
    {
      "epoch": 0.01740410566778441,
      "grad_norm": 4.641057014465332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5824930667877197,
      "logits/rejected": 0.6103401184082031,
      "logps/chosen": -529.7232666015625,
      "logps/rejected": -450.36322021484375,
      "loss": 0.6038,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09688682854175568,
      "rewards/margins": 0.20408879220485687,
      "rewards/rejected": -0.30097565054893494,
      "step": 133
    },
    {
      "epoch": 0.017534963605136174,
      "grad_norm": 4.477089881896973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010284360498189926,
      "logits/rejected": 0.11901645362377167,
      "logps/chosen": -367.2096252441406,
      "logps/rejected": -417.5522766113281,
      "loss": 0.6627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21781732141971588,
      "rewards/margins": 0.0713052749633789,
      "rewards/rejected": -0.2891225814819336,
      "step": 134
    },
    {
      "epoch": 0.017665821542487938,
      "grad_norm": 4.874619007110596,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1603892594575882,
      "logits/rejected": 0.15432295203208923,
      "logps/chosen": -454.2311096191406,
      "logps/rejected": -639.231201171875,
      "loss": 0.6477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04801731929183006,
      "rewards/margins": 0.10443553328514099,
      "rewards/rejected": -0.15245285630226135,
      "step": 135
    },
    {
      "epoch": 0.0177966794798397,
      "grad_norm": 3.797924041748047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3109673857688904,
      "logits/rejected": 0.3258686363697052,
      "logps/chosen": -305.68621826171875,
      "logps/rejected": -408.81591796875,
      "loss": 0.5735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1734982579946518,
      "rewards/margins": 0.2661334276199341,
      "rewards/rejected": -0.4396316409111023,
      "step": 136
    },
    {
      "epoch": 0.017927537417191462,
      "grad_norm": 5.397445201873779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5551742911338806,
      "logits/rejected": 0.3222101926803589,
      "logps/chosen": -638.4053955078125,
      "logps/rejected": -615.6397094726562,
      "loss": 0.6429,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.031333550810813904,
      "rewards/margins": 0.15820808708667755,
      "rewards/rejected": -0.18954163789749146,
      "step": 137
    },
    {
      "epoch": 0.018058395354543223,
      "grad_norm": 4.27779483795166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2054029107093811,
      "logits/rejected": 0.1514713317155838,
      "logps/chosen": -363.4287414550781,
      "logps/rejected": -381.16229248046875,
      "loss": 0.583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06309471279382706,
      "rewards/margins": 0.25868991017341614,
      "rewards/rejected": -0.3217846155166626,
      "step": 138
    },
    {
      "epoch": 0.018189253291894986,
      "grad_norm": 4.352468013763428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3680392801761627,
      "logits/rejected": 0.3730838894844055,
      "logps/chosen": -401.5879821777344,
      "logps/rejected": -465.5873107910156,
      "loss": 0.6333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.246980682015419,
      "rewards/margins": 0.14114324748516083,
      "rewards/rejected": -0.38812389969825745,
      "step": 139
    },
    {
      "epoch": 0.01832011122924675,
      "grad_norm": 3.7825253009796143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24083444476127625,
      "logits/rejected": 0.24821962416172028,
      "logps/chosen": -306.3134765625,
      "logps/rejected": -348.40374755859375,
      "loss": 0.6372,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12554949522018433,
      "rewards/margins": 0.121587373316288,
      "rewards/rejected": -0.2471368908882141,
      "step": 140
    },
    {
      "epoch": 0.01845096916659851,
      "grad_norm": 4.280195713043213,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3185957372188568,
      "logits/rejected": 0.38192611932754517,
      "logps/chosen": -404.2518005371094,
      "logps/rejected": -515.8758544921875,
      "loss": 0.5948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28023090958595276,
      "rewards/margins": 0.2289915233850479,
      "rewards/rejected": -0.5092223882675171,
      "step": 141
    },
    {
      "epoch": 0.018581827103950274,
      "grad_norm": 4.103994369506836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09828405827283859,
      "logits/rejected": 0.117824986577034,
      "logps/chosen": -421.6966552734375,
      "logps/rejected": -448.71337890625,
      "loss": 0.6185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19207020103931427,
      "rewards/margins": 0.20737743377685547,
      "rewards/rejected": -0.39944761991500854,
      "step": 142
    },
    {
      "epoch": 0.018712685041302038,
      "grad_norm": 4.487863063812256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13464827835559845,
      "logits/rejected": -0.14612653851509094,
      "logps/chosen": -357.89935302734375,
      "logps/rejected": -318.7261962890625,
      "loss": 0.646,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23168601095676422,
      "rewards/margins": 0.12120667099952698,
      "rewards/rejected": -0.3528926968574524,
      "step": 143
    },
    {
      "epoch": 0.018843542978653798,
      "grad_norm": 3.931802988052368,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030716702342033386,
      "logits/rejected": 0.36117467284202576,
      "logps/chosen": -413.164794921875,
      "logps/rejected": -399.4678649902344,
      "loss": 0.6502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3943733274936676,
      "rewards/margins": 0.10597744584083557,
      "rewards/rejected": -0.5003507733345032,
      "step": 144
    },
    {
      "epoch": 0.018974400916005562,
      "grad_norm": 3.8967788219451904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4190385639667511,
      "logits/rejected": 0.333528995513916,
      "logps/chosen": -419.34466552734375,
      "logps/rejected": -379.7839050292969,
      "loss": 0.6018,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21035003662109375,
      "rewards/margins": 0.22840023040771484,
      "rewards/rejected": -0.4387502670288086,
      "step": 145
    },
    {
      "epoch": 0.019105258853357322,
      "grad_norm": 5.097753047943115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24579669535160065,
      "logits/rejected": 0.258829802274704,
      "logps/chosen": -558.97998046875,
      "logps/rejected": -480.5152282714844,
      "loss": 0.652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25470906496047974,
      "rewards/margins": 0.09112205356359482,
      "rewards/rejected": -0.34583112597465515,
      "step": 146
    },
    {
      "epoch": 0.019236116790709086,
      "grad_norm": 4.88690710067749,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37902548909187317,
      "logits/rejected": 0.42776525020599365,
      "logps/chosen": -622.9922485351562,
      "logps/rejected": -530.8080444335938,
      "loss": 0.6965,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4148237705230713,
      "rewards/margins": 0.008830074220895767,
      "rewards/rejected": -0.423653781414032,
      "step": 147
    },
    {
      "epoch": 0.01936697472806085,
      "grad_norm": 5.199206829071045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37549468874931335,
      "logits/rejected": 0.5445336103439331,
      "logps/chosen": -500.26727294921875,
      "logps/rejected": -530.4534912109375,
      "loss": 0.6725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2209787368774414,
      "rewards/margins": 0.05012302100658417,
      "rewards/rejected": -0.27110177278518677,
      "step": 148
    },
    {
      "epoch": 0.01949783266541261,
      "grad_norm": 4.342934608459473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3014261722564697,
      "logits/rejected": 0.3096834719181061,
      "logps/chosen": -561.7203369140625,
      "logps/rejected": -489.80450439453125,
      "loss": 0.5072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11361847817897797,
      "rewards/margins": 0.4724365472793579,
      "rewards/rejected": -0.5860550403594971,
      "step": 149
    },
    {
      "epoch": 0.019628690602764374,
      "grad_norm": 4.930890083312988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.351177453994751,
      "logits/rejected": 0.5526539087295532,
      "logps/chosen": -491.90789794921875,
      "logps/rejected": -345.3132629394531,
      "loss": 0.7639,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.44764214754104614,
      "rewards/margins": -0.1131402999162674,
      "rewards/rejected": -0.33450183272361755,
      "step": 150
    },
    {
      "epoch": 0.019759548540116138,
      "grad_norm": 4.002737045288086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5438966155052185,
      "logits/rejected": 0.4170554578304291,
      "logps/chosen": -364.44281005859375,
      "logps/rejected": -310.381591796875,
      "loss": 0.6543,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.393520712852478,
      "rewards/margins": 0.10366934537887573,
      "rewards/rejected": -0.49719011783599854,
      "step": 151
    },
    {
      "epoch": 0.019890406477467898,
      "grad_norm": 4.061173915863037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3149647116661072,
      "logits/rejected": 0.26889222860336304,
      "logps/chosen": -479.8333435058594,
      "logps/rejected": -376.6438293457031,
      "loss": 0.5292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1949256807565689,
      "rewards/margins": 0.3941513001918793,
      "rewards/rejected": -0.5890770554542542,
      "step": 152
    },
    {
      "epoch": 0.02002126441481966,
      "grad_norm": 5.747546672821045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39181745052337646,
      "logits/rejected": 0.38663673400878906,
      "logps/chosen": -353.9447326660156,
      "logps/rejected": -482.2281494140625,
      "loss": 0.7947,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5360145568847656,
      "rewards/margins": -0.18095572292804718,
      "rewards/rejected": -0.35505884885787964,
      "step": 153
    },
    {
      "epoch": 0.020152122352171425,
      "grad_norm": 4.2551703453063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008274108171463013,
      "logits/rejected": 0.1951521337032318,
      "logps/chosen": -339.6717224121094,
      "logps/rejected": -399.87347412109375,
      "loss": 0.6746,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2509637773036957,
      "rewards/margins": 0.06723099201917648,
      "rewards/rejected": -0.31819480657577515,
      "step": 154
    },
    {
      "epoch": 0.020282980289523186,
      "grad_norm": 4.775103569030762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3226171135902405,
      "logits/rejected": 0.33752521872520447,
      "logps/chosen": -461.00738525390625,
      "logps/rejected": -452.8443603515625,
      "loss": 0.6577,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4942568242549896,
      "rewards/margins": 0.10790690779685974,
      "rewards/rejected": -0.6021637320518494,
      "step": 155
    },
    {
      "epoch": 0.02041383822687495,
      "grad_norm": 4.2550177574157715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38043317198753357,
      "logits/rejected": 0.2213030606508255,
      "logps/chosen": -403.4322204589844,
      "logps/rejected": -369.64959716796875,
      "loss": 0.6318,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23151513934135437,
      "rewards/margins": 0.18641987442970276,
      "rewards/rejected": -0.41793501377105713,
      "step": 156
    },
    {
      "epoch": 0.02054469616422671,
      "grad_norm": 4.5998358726501465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3355603814125061,
      "logits/rejected": 0.3553331196308136,
      "logps/chosen": -537.4739990234375,
      "logps/rejected": -375.305419921875,
      "loss": 0.5891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4053596556186676,
      "rewards/margins": 0.28004664182662964,
      "rewards/rejected": -0.6854063272476196,
      "step": 157
    },
    {
      "epoch": 0.020675554101578474,
      "grad_norm": 4.176523208618164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12852773070335388,
      "logits/rejected": 0.16214142739772797,
      "logps/chosen": -372.90081787109375,
      "logps/rejected": -345.57647705078125,
      "loss": 0.586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27432650327682495,
      "rewards/margins": 0.33534032106399536,
      "rewards/rejected": -0.6096668243408203,
      "step": 158
    },
    {
      "epoch": 0.020806412038930237,
      "grad_norm": 4.4667134284973145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7416837811470032,
      "logits/rejected": 0.45329681038856506,
      "logps/chosen": -610.9322509765625,
      "logps/rejected": -540.7864379882812,
      "loss": 0.5988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13643458485603333,
      "rewards/margins": 0.2439945489168167,
      "rewards/rejected": -0.38042905926704407,
      "step": 159
    },
    {
      "epoch": 0.020937269976281998,
      "grad_norm": 7.167546272277832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29737237095832825,
      "logits/rejected": 0.044885315001010895,
      "logps/chosen": -572.6336669921875,
      "logps/rejected": -538.8963623046875,
      "loss": 0.8825,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.49811726808547974,
      "rewards/margins": -0.32470396161079407,
      "rewards/rejected": -0.17341327667236328,
      "step": 160
    },
    {
      "epoch": 0.02106812791363376,
      "grad_norm": 4.854349613189697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45725739002227783,
      "logits/rejected": 0.21693937480449677,
      "logps/chosen": -466.7396240234375,
      "logps/rejected": -444.7483215332031,
      "loss": 0.7831,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6049213409423828,
      "rewards/margins": -0.0797908753156662,
      "rewards/rejected": -0.5251304507255554,
      "step": 161
    },
    {
      "epoch": 0.021198985850985525,
      "grad_norm": 4.109683990478516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13964059948921204,
      "logits/rejected": 0.19173458218574524,
      "logps/chosen": -409.7204284667969,
      "logps/rejected": -307.6723937988281,
      "loss": 0.6894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40552881360054016,
      "rewards/margins": 0.014929957687854767,
      "rewards/rejected": -0.4204587936401367,
      "step": 162
    },
    {
      "epoch": 0.021329843788337285,
      "grad_norm": 6.37981653213501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06771308183670044,
      "logits/rejected": -0.15896235406398773,
      "logps/chosen": -562.2279052734375,
      "logps/rejected": -473.4747009277344,
      "loss": 0.6685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18625450134277344,
      "rewards/margins": 0.09655095636844635,
      "rewards/rejected": -0.282805472612381,
      "step": 163
    },
    {
      "epoch": 0.02146070172568905,
      "grad_norm": 4.102161407470703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18235641717910767,
      "logits/rejected": 0.1067160964012146,
      "logps/chosen": -580.4844360351562,
      "logps/rejected": -442.1970520019531,
      "loss": 0.5095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16444815695285797,
      "rewards/margins": 0.45964956283569336,
      "rewards/rejected": -0.6240977644920349,
      "step": 164
    },
    {
      "epoch": 0.021591559663040813,
      "grad_norm": 4.660390377044678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27718621492385864,
      "logits/rejected": 0.1380298137664795,
      "logps/chosen": -426.76251220703125,
      "logps/rejected": -368.2798156738281,
      "loss": 0.6649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.33095550537109375,
      "rewards/margins": 0.06959132105112076,
      "rewards/rejected": -0.4005468189716339,
      "step": 165
    },
    {
      "epoch": 0.021722417600392573,
      "grad_norm": 5.940152645111084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2520817518234253,
      "logits/rejected": 0.2787063419818878,
      "logps/chosen": -413.8662414550781,
      "logps/rejected": -492.918701171875,
      "loss": 0.741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33886754512786865,
      "rewards/margins": -0.05935489013791084,
      "rewards/rejected": -0.2795127034187317,
      "step": 166
    },
    {
      "epoch": 0.021853275537744337,
      "grad_norm": 4.540476322174072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3534674346446991,
      "logits/rejected": 0.01582895964384079,
      "logps/chosen": -396.9669189453125,
      "logps/rejected": -460.0363464355469,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3050724267959595,
      "rewards/margins": 0.0374092236161232,
      "rewards/rejected": -0.3424816429615021,
      "step": 167
    },
    {
      "epoch": 0.021984133475096097,
      "grad_norm": 4.670223712921143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40670135617256165,
      "logits/rejected": 0.45693114399909973,
      "logps/chosen": -432.2122802734375,
      "logps/rejected": -478.7832336425781,
      "loss": 0.6084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4358133375644684,
      "rewards/margins": 0.2656700015068054,
      "rewards/rejected": -0.7014833688735962,
      "step": 168
    },
    {
      "epoch": 0.02211499141244786,
      "grad_norm": 4.088492393493652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38478508591651917,
      "logits/rejected": 0.4750421643257141,
      "logps/chosen": -434.5316162109375,
      "logps/rejected": -338.9808654785156,
      "loss": 0.5995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24372577667236328,
      "rewards/margins": 0.2635577917098999,
      "rewards/rejected": -0.5072835683822632,
      "step": 169
    },
    {
      "epoch": 0.022245849349799625,
      "grad_norm": 4.678393363952637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14312335848808289,
      "logits/rejected": 0.43470385670661926,
      "logps/chosen": -557.234130859375,
      "logps/rejected": -486.3250427246094,
      "loss": 0.6243,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5456637144088745,
      "rewards/margins": 0.19832345843315125,
      "rewards/rejected": -0.7439871430397034,
      "step": 170
    },
    {
      "epoch": 0.022376707287151385,
      "grad_norm": 5.657143592834473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41024452447891235,
      "logits/rejected": 0.3971763551235199,
      "logps/chosen": -579.9823608398438,
      "logps/rejected": -590.6130981445312,
      "loss": 0.7343,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4386390745639801,
      "rewards/margins": -0.04683971405029297,
      "rewards/rejected": -0.39179936051368713,
      "step": 171
    },
    {
      "epoch": 0.02250756522450315,
      "grad_norm": 6.840025901794434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08769319951534271,
      "logits/rejected": 0.33124208450317383,
      "logps/chosen": -612.6814575195312,
      "logps/rejected": -630.3541259765625,
      "loss": 0.7585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28855934739112854,
      "rewards/margins": -0.10931127518415451,
      "rewards/rejected": -0.17924803495407104,
      "step": 172
    },
    {
      "epoch": 0.022638423161854913,
      "grad_norm": 4.550375461578369,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36389613151550293,
      "logits/rejected": 0.2036552131175995,
      "logps/chosen": -286.83441162109375,
      "logps/rejected": -392.8669738769531,
      "loss": 0.6058,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15256957709789276,
      "rewards/margins": 0.21931380033493042,
      "rewards/rejected": -0.3718833923339844,
      "step": 173
    },
    {
      "epoch": 0.022769281099206673,
      "grad_norm": 4.324200630187988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5021041631698608,
      "logits/rejected": 0.4905813932418823,
      "logps/chosen": -517.7786865234375,
      "logps/rejected": -472.2088928222656,
      "loss": 0.5319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16157132387161255,
      "rewards/margins": 0.4144302308559418,
      "rewards/rejected": -0.5760015845298767,
      "step": 174
    },
    {
      "epoch": 0.022900139036558437,
      "grad_norm": 4.4189534187316895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48920971155166626,
      "logits/rejected": 0.4439666271209717,
      "logps/chosen": -573.994873046875,
      "logps/rejected": -472.3438720703125,
      "loss": 0.5653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.038542553782463074,
      "rewards/margins": 0.28830185532569885,
      "rewards/rejected": -0.32684439420700073,
      "step": 175
    },
    {
      "epoch": 0.023030996973910197,
      "grad_norm": 4.49552059173584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09582729637622833,
      "logits/rejected": -0.0732363611459732,
      "logps/chosen": -409.4186706542969,
      "logps/rejected": -303.7612609863281,
      "loss": 0.7185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24825327098369598,
      "rewards/margins": 0.007320594973862171,
      "rewards/rejected": -0.2555738389492035,
      "step": 176
    },
    {
      "epoch": 0.02316185491126196,
      "grad_norm": 3.7690250873565674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4702295660972595,
      "logits/rejected": 0.38508060574531555,
      "logps/chosen": -390.5576171875,
      "logps/rejected": -285.8189392089844,
      "loss": 0.6147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22591380774974823,
      "rewards/margins": 0.17587889730930328,
      "rewards/rejected": -0.4017927050590515,
      "step": 177
    },
    {
      "epoch": 0.023292712848613725,
      "grad_norm": 4.985249042510986,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4997263252735138,
      "logits/rejected": 0.4669240117073059,
      "logps/chosen": -236.7034912109375,
      "logps/rejected": -349.5284729003906,
      "loss": 0.832,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.30929702520370483,
      "rewards/margins": -0.2364915907382965,
      "rewards/rejected": -0.07280540466308594,
      "step": 178
    },
    {
      "epoch": 0.023423570785965485,
      "grad_norm": 5.788545608520508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15217304229736328,
      "logits/rejected": 0.3029392957687378,
      "logps/chosen": -334.40411376953125,
      "logps/rejected": -399.2774963378906,
      "loss": 0.7657,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5010617971420288,
      "rewards/margins": -0.015802189707756042,
      "rewards/rejected": -0.48525965213775635,
      "step": 179
    },
    {
      "epoch": 0.02355442872331725,
      "grad_norm": 4.538683891296387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5906680226325989,
      "logits/rejected": 0.45106104016304016,
      "logps/chosen": -370.2652893066406,
      "logps/rejected": -409.30413818359375,
      "loss": 0.739,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39007094502449036,
      "rewards/margins": -0.058854296803474426,
      "rewards/rejected": -0.33121660351753235,
      "step": 180
    },
    {
      "epoch": 0.023685286660669012,
      "grad_norm": 4.343271255493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14346995949745178,
      "logits/rejected": 0.10486015677452087,
      "logps/chosen": -520.9603271484375,
      "logps/rejected": -464.1452331542969,
      "loss": 0.5686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2809251844882965,
      "rewards/margins": 0.3260757625102997,
      "rewards/rejected": -0.6070009469985962,
      "step": 181
    },
    {
      "epoch": 0.023816144598020773,
      "grad_norm": 3.967813014984131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20700018107891083,
      "logits/rejected": 0.24991920590400696,
      "logps/chosen": -481.04827880859375,
      "logps/rejected": -473.79608154296875,
      "loss": 0.5682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17825336754322052,
      "rewards/margins": 0.31470242142677307,
      "rewards/rejected": -0.4929557740688324,
      "step": 182
    },
    {
      "epoch": 0.023947002535372536,
      "grad_norm": 6.1477203369140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45322614908218384,
      "logits/rejected": 0.33041298389434814,
      "logps/chosen": -431.3497009277344,
      "logps/rejected": -427.322265625,
      "loss": 0.8035,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6945570111274719,
      "rewards/margins": -0.1713750809431076,
      "rewards/rejected": -0.5231819748878479,
      "step": 183
    },
    {
      "epoch": 0.0240778604727243,
      "grad_norm": 4.5795087814331055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21808168292045593,
      "logits/rejected": -0.009059444069862366,
      "logps/chosen": -300.82525634765625,
      "logps/rejected": -452.6053466796875,
      "loss": 0.6481,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20764315128326416,
      "rewards/margins": 0.1296379119157791,
      "rewards/rejected": -0.3372810482978821,
      "step": 184
    },
    {
      "epoch": 0.02420871841007606,
      "grad_norm": 3.8404338359832764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09028559923171997,
      "logits/rejected": 0.18707653880119324,
      "logps/chosen": -243.42161560058594,
      "logps/rejected": -304.4251708984375,
      "loss": 0.5967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13862267136573792,
      "rewards/margins": 0.20915165543556213,
      "rewards/rejected": -0.34777429699897766,
      "step": 185
    },
    {
      "epoch": 0.024339576347427824,
      "grad_norm": 4.899326324462891,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06995011866092682,
      "logits/rejected": -0.0013511478900909424,
      "logps/chosen": -557.5728759765625,
      "logps/rejected": -555.2491455078125,
      "loss": 0.7416,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.386557400226593,
      "rewards/margins": -0.04723682999610901,
      "rewards/rejected": -0.339320570230484,
      "step": 186
    },
    {
      "epoch": 0.024470434284779585,
      "grad_norm": 4.737020492553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.577307403087616,
      "logits/rejected": 0.4129565358161926,
      "logps/chosen": -486.23284912109375,
      "logps/rejected": -314.2021179199219,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2851276397705078,
      "rewards/margins": 0.018186956644058228,
      "rewards/rejected": -0.30331456661224365,
      "step": 187
    },
    {
      "epoch": 0.02460129222213135,
      "grad_norm": 5.075306415557861,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23826920986175537,
      "logits/rejected": 0.4811863899230957,
      "logps/chosen": -399.005126953125,
      "logps/rejected": -364.0255432128906,
      "loss": 0.7644,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5459148287773132,
      "rewards/margins": -0.07910691201686859,
      "rewards/rejected": -0.4668079614639282,
      "step": 188
    },
    {
      "epoch": 0.024732150159483112,
      "grad_norm": 4.505218029022217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32787996530532837,
      "logits/rejected": 0.18277984857559204,
      "logps/chosen": -471.8974304199219,
      "logps/rejected": -369.908935546875,
      "loss": 0.5897,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24737340211868286,
      "rewards/margins": 0.23117810487747192,
      "rewards/rejected": -0.4785515069961548,
      "step": 189
    },
    {
      "epoch": 0.024863008096834872,
      "grad_norm": 4.768984794616699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14809370040893555,
      "logits/rejected": 0.3054432272911072,
      "logps/chosen": -487.30670166015625,
      "logps/rejected": -513.2891235351562,
      "loss": 0.6761,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.42352354526519775,
      "rewards/margins": 0.11089382320642471,
      "rewards/rejected": -0.5344173312187195,
      "step": 190
    },
    {
      "epoch": 0.024993866034186636,
      "grad_norm": 6.205466270446777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4690900444984436,
      "logits/rejected": 0.2809460759162903,
      "logps/chosen": -659.1843872070312,
      "logps/rejected": -569.6343383789062,
      "loss": 0.7615,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6790894865989685,
      "rewards/margins": -0.10388296842575073,
      "rewards/rejected": -0.5752065777778625,
      "step": 191
    },
    {
      "epoch": 0.0251247239715384,
      "grad_norm": 6.874771595001221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33669862151145935,
      "logits/rejected": 0.2356777787208557,
      "logps/chosen": -420.4542236328125,
      "logps/rejected": -439.314208984375,
      "loss": 0.612,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2822223901748657,
      "rewards/margins": 0.22920836508274078,
      "rewards/rejected": -0.5114307403564453,
      "step": 192
    },
    {
      "epoch": 0.02525558190889016,
      "grad_norm": 5.293374538421631,
      "learning_rate": 5e-06,
      "logits/chosen": 0.691996157169342,
      "logits/rejected": 0.38998734951019287,
      "logps/chosen": -530.881103515625,
      "logps/rejected": -386.7685546875,
      "loss": 0.6682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.590018630027771,
      "rewards/margins": 0.1737205535173416,
      "rewards/rejected": -0.763739287853241,
      "step": 193
    },
    {
      "epoch": 0.025386439846241924,
      "grad_norm": 4.734703540802002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34037840366363525,
      "logits/rejected": 0.16774477064609528,
      "logps/chosen": -588.3373413085938,
      "logps/rejected": -448.3605651855469,
      "loss": 0.578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16043415665626526,
      "rewards/margins": 0.2971602976322174,
      "rewards/rejected": -0.45759448409080505,
      "step": 194
    },
    {
      "epoch": 0.025517297783593688,
      "grad_norm": 3.7737298011779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48436233401298523,
      "logits/rejected": 0.40310969948768616,
      "logps/chosen": -471.5328369140625,
      "logps/rejected": -243.65245056152344,
      "loss": 0.6588,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1145046204328537,
      "rewards/margins": 0.09801311790943146,
      "rewards/rejected": -0.21251773834228516,
      "step": 195
    },
    {
      "epoch": 0.025648155720945448,
      "grad_norm": 6.425372123718262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06532195210456848,
      "logits/rejected": 0.056748166680336,
      "logps/chosen": -350.28460693359375,
      "logps/rejected": -613.7586059570312,
      "loss": 0.8618,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5256309509277344,
      "rewards/margins": -0.27303048968315125,
      "rewards/rejected": -0.2526004910469055,
      "step": 196
    },
    {
      "epoch": 0.025779013658297212,
      "grad_norm": 4.644787311553955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2977415919303894,
      "logits/rejected": 0.2535744905471802,
      "logps/chosen": -340.7049255371094,
      "logps/rejected": -326.43280029296875,
      "loss": 0.7001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4886302947998047,
      "rewards/margins": 0.03582077845931053,
      "rewards/rejected": -0.5244510769844055,
      "step": 197
    },
    {
      "epoch": 0.025909871595648972,
      "grad_norm": 4.253194332122803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2792966663837433,
      "logits/rejected": 0.4114265739917755,
      "logps/chosen": -308.98785400390625,
      "logps/rejected": -309.3365478515625,
      "loss": 0.6931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39419594407081604,
      "rewards/margins": 0.046662718057632446,
      "rewards/rejected": -0.4408586323261261,
      "step": 198
    },
    {
      "epoch": 0.026040729533000736,
      "grad_norm": 4.043999671936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03585371375083923,
      "logits/rejected": 0.13707533478736877,
      "logps/chosen": -460.0248107910156,
      "logps/rejected": -413.6667175292969,
      "loss": 0.6121,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18645095825195312,
      "rewards/margins": 0.1889801323413849,
      "rewards/rejected": -0.3754310607910156,
      "step": 199
    },
    {
      "epoch": 0.0261715874703525,
      "grad_norm": 4.638398170471191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5694870352745056,
      "logits/rejected": 0.486000657081604,
      "logps/chosen": -477.98779296875,
      "logps/rejected": -451.8931884765625,
      "loss": 0.5754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22930680215358734,
      "rewards/margins": 0.30895155668258667,
      "rewards/rejected": -0.5382583737373352,
      "step": 200
    },
    {
      "epoch": 0.02630244540770426,
      "grad_norm": 4.127016067504883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6928343772888184,
      "logits/rejected": 0.5306254625320435,
      "logps/chosen": -343.92303466796875,
      "logps/rejected": -322.4310302734375,
      "loss": 0.6501,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30643710494041443,
      "rewards/margins": 0.12324753403663635,
      "rewards/rejected": -0.4296846389770508,
      "step": 201
    },
    {
      "epoch": 0.026433303345056024,
      "grad_norm": 3.8462352752685547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21823939681053162,
      "logits/rejected": 0.28639984130859375,
      "logps/chosen": -398.90496826171875,
      "logps/rejected": -582.783203125,
      "loss": 0.5809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2684665620326996,
      "rewards/margins": 0.2725957930088043,
      "rewards/rejected": -0.5410623550415039,
      "step": 202
    },
    {
      "epoch": 0.026564161282407787,
      "grad_norm": 4.961702823638916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21966515481472015,
      "logits/rejected": -0.038262829184532166,
      "logps/chosen": -413.1593322753906,
      "logps/rejected": -351.8193359375,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3814886212348938,
      "rewards/margins": 0.008020404726266861,
      "rewards/rejected": -0.38950902223587036,
      "step": 203
    },
    {
      "epoch": 0.026695019219759548,
      "grad_norm": 5.029583930969238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07812164723873138,
      "logits/rejected": -0.025303799659013748,
      "logps/chosen": -430.9508056640625,
      "logps/rejected": -511.5746154785156,
      "loss": 0.6253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20233286917209625,
      "rewards/margins": 0.15169428288936615,
      "rewards/rejected": -0.35402712225914,
      "step": 204
    },
    {
      "epoch": 0.02682587715711131,
      "grad_norm": 7.206608295440674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4083921015262604,
      "logits/rejected": 0.54448002576828,
      "logps/chosen": -740.3980102539062,
      "logps/rejected": -749.5164794921875,
      "loss": 0.7167,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5318048596382141,
      "rewards/margins": 0.06694623082876205,
      "rewards/rejected": -0.5987510085105896,
      "step": 205
    },
    {
      "epoch": 0.026956735094463072,
      "grad_norm": 6.448324203491211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28921204805374146,
      "logits/rejected": 0.3082549571990967,
      "logps/chosen": -442.365234375,
      "logps/rejected": -311.89581298828125,
      "loss": 0.7582,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4661531448364258,
      "rewards/margins": -0.07867659628391266,
      "rewards/rejected": -0.3874765634536743,
      "step": 206
    },
    {
      "epoch": 0.027087593031814836,
      "grad_norm": 4.420883655548096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21344688534736633,
      "logits/rejected": 0.23658795654773712,
      "logps/chosen": -389.0162048339844,
      "logps/rejected": -361.32037353515625,
      "loss": 0.7216,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39737415313720703,
      "rewards/margins": -0.03957042470574379,
      "rewards/rejected": -0.35780373215675354,
      "step": 207
    },
    {
      "epoch": 0.0272184509691666,
      "grad_norm": 6.267385482788086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22139513492584229,
      "logits/rejected": -0.03910814970731735,
      "logps/chosen": -396.99285888671875,
      "logps/rejected": -465.7701110839844,
      "loss": 0.7357,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4419476091861725,
      "rewards/margins": -0.040269091725349426,
      "rewards/rejected": -0.4016784727573395,
      "step": 208
    },
    {
      "epoch": 0.02734930890651836,
      "grad_norm": 5.023506164550781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20848163962364197,
      "logits/rejected": 0.042167216539382935,
      "logps/chosen": -480.2857360839844,
      "logps/rejected": -372.70367431640625,
      "loss": 0.7229,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3992391526699066,
      "rewards/margins": -0.04467450827360153,
      "rewards/rejected": -0.3545646369457245,
      "step": 209
    },
    {
      "epoch": 0.027480166843870123,
      "grad_norm": 4.539527416229248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10650742053985596,
      "logits/rejected": 0.16458413004875183,
      "logps/chosen": -418.8702087402344,
      "logps/rejected": -397.39801025390625,
      "loss": 0.6354,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20349407196044922,
      "rewards/margins": 0.17007483541965485,
      "rewards/rejected": -0.37356895208358765,
      "step": 210
    },
    {
      "epoch": 0.027611024781221887,
      "grad_norm": 5.786539554595947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41276395320892334,
      "logits/rejected": 0.3890700340270996,
      "logps/chosen": -464.38275146484375,
      "logps/rejected": -494.0610656738281,
      "loss": 0.7457,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2699834704399109,
      "rewards/margins": -0.05371074005961418,
      "rewards/rejected": -0.2162727266550064,
      "step": 211
    },
    {
      "epoch": 0.027741882718573647,
      "grad_norm": 4.6607279777526855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31353628635406494,
      "logits/rejected": 0.270368754863739,
      "logps/chosen": -492.9356994628906,
      "logps/rejected": -428.8128662109375,
      "loss": 0.6467,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2986127734184265,
      "rewards/margins": 0.16273097693920135,
      "rewards/rejected": -0.46134376525878906,
      "step": 212
    },
    {
      "epoch": 0.02787274065592541,
      "grad_norm": 4.141776084899902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1530095636844635,
      "logits/rejected": -0.050540417432785034,
      "logps/chosen": -375.4052429199219,
      "logps/rejected": -435.4034423828125,
      "loss": 0.6343,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23413638770580292,
      "rewards/margins": 0.14850616455078125,
      "rewards/rejected": -0.382642537355423,
      "step": 213
    },
    {
      "epoch": 0.028003598593277175,
      "grad_norm": 7.599163055419922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20117472112178802,
      "logits/rejected": -0.13265500962734222,
      "logps/chosen": -509.4815979003906,
      "logps/rejected": -412.42364501953125,
      "loss": 0.6868,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21920983493328094,
      "rewards/margins": 0.0728733092546463,
      "rewards/rejected": -0.292083203792572,
      "step": 214
    },
    {
      "epoch": 0.028134456530628935,
      "grad_norm": 5.328446865081787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28650736808776855,
      "logits/rejected": 0.42924219369888306,
      "logps/chosen": -483.92962646484375,
      "logps/rejected": -428.20172119140625,
      "loss": 0.8774,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4586017429828644,
      "rewards/margins": -0.30247077345848083,
      "rewards/rejected": -0.15613096952438354,
      "step": 215
    },
    {
      "epoch": 0.0282653144679807,
      "grad_norm": 5.2452592849731445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3791770935058594,
      "logits/rejected": 0.4132555425167084,
      "logps/chosen": -653.3497314453125,
      "logps/rejected": -447.86846923828125,
      "loss": 0.6988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37818413972854614,
      "rewards/margins": 0.06512565165758133,
      "rewards/rejected": -0.4433097541332245,
      "step": 216
    },
    {
      "epoch": 0.02839617240533246,
      "grad_norm": 5.4132819175720215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42884713411331177,
      "logits/rejected": 0.34799033403396606,
      "logps/chosen": -463.8648376464844,
      "logps/rejected": -371.42889404296875,
      "loss": 0.6691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27982065081596375,
      "rewards/margins": 0.07908649742603302,
      "rewards/rejected": -0.35890713334083557,
      "step": 217
    },
    {
      "epoch": 0.028527030342684223,
      "grad_norm": 3.883805274963379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5109996795654297,
      "logits/rejected": 0.40731653571128845,
      "logps/chosen": -463.73309326171875,
      "logps/rejected": -334.94012451171875,
      "loss": 0.5693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19004708528518677,
      "rewards/margins": 0.31719666719436646,
      "rewards/rejected": -0.5072437524795532,
      "step": 218
    },
    {
      "epoch": 0.028657888280035987,
      "grad_norm": 4.320803642272949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.61614590883255,
      "logits/rejected": 0.6027024388313293,
      "logps/chosen": -398.6131591796875,
      "logps/rejected": -355.75140380859375,
      "loss": 0.6242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46738091111183167,
      "rewards/margins": 0.20040056109428406,
      "rewards/rejected": -0.6677814722061157,
      "step": 219
    },
    {
      "epoch": 0.028788746217387747,
      "grad_norm": 4.339902877807617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32147231698036194,
      "logits/rejected": 0.18178454041481018,
      "logps/chosen": -473.01806640625,
      "logps/rejected": -465.18341064453125,
      "loss": 0.5644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2962349057197571,
      "rewards/margins": 0.3416624069213867,
      "rewards/rejected": -0.6378973126411438,
      "step": 220
    },
    {
      "epoch": 0.02891960415473951,
      "grad_norm": 4.271994113922119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43184083700180054,
      "logits/rejected": 0.451326847076416,
      "logps/chosen": -389.7900085449219,
      "logps/rejected": -417.35760498046875,
      "loss": 0.6048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1222354918718338,
      "rewards/margins": 0.20778904855251312,
      "rewards/rejected": -0.3300245404243469,
      "step": 221
    },
    {
      "epoch": 0.029050462092091275,
      "grad_norm": 4.524006366729736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17696726322174072,
      "logits/rejected": 0.040786415338516235,
      "logps/chosen": -509.2662048339844,
      "logps/rejected": -383.02325439453125,
      "loss": 0.5953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18220385909080505,
      "rewards/margins": 0.23823848366737366,
      "rewards/rejected": -0.4204424023628235,
      "step": 222
    },
    {
      "epoch": 0.029181320029443035,
      "grad_norm": 4.7507500648498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4431719183921814,
      "logits/rejected": 0.21186678111553192,
      "logps/chosen": -477.43890380859375,
      "logps/rejected": -419.01593017578125,
      "loss": 0.5852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2024528682231903,
      "rewards/margins": 0.25508004426956177,
      "rewards/rejected": -0.4575328826904297,
      "step": 223
    },
    {
      "epoch": 0.0293121779667948,
      "grad_norm": 4.893976211547852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17634879052639008,
      "logits/rejected": 0.2666419446468353,
      "logps/chosen": -456.28057861328125,
      "logps/rejected": -403.60943603515625,
      "loss": 0.5996,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3106778860092163,
      "rewards/margins": 0.24800509214401245,
      "rewards/rejected": -0.5586830377578735,
      "step": 224
    },
    {
      "epoch": 0.029443035904146563,
      "grad_norm": 4.5718278884887695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24466057121753693,
      "logits/rejected": 0.01825624331831932,
      "logps/chosen": -600.2244262695312,
      "logps/rejected": -386.3888244628906,
      "loss": 0.6179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31738191843032837,
      "rewards/margins": 0.17732325196266174,
      "rewards/rejected": -0.4947052299976349,
      "step": 225
    },
    {
      "epoch": 0.029573893841498323,
      "grad_norm": 3.6438655853271484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6028978228569031,
      "logits/rejected": 0.41285163164138794,
      "logps/chosen": -418.2997741699219,
      "logps/rejected": -340.80670166015625,
      "loss": 0.5686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11413155496120453,
      "rewards/margins": 0.2868839204311371,
      "rewards/rejected": -0.4010154604911804,
      "step": 226
    },
    {
      "epoch": 0.029704751778850087,
      "grad_norm": 4.326143264770508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3127782642841339,
      "logits/rejected": 0.2895694971084595,
      "logps/chosen": -412.4805908203125,
      "logps/rejected": -474.3132019042969,
      "loss": 0.5452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2761133313179016,
      "rewards/margins": 0.3726499676704407,
      "rewards/rejected": -0.6487632989883423,
      "step": 227
    },
    {
      "epoch": 0.029835609716201847,
      "grad_norm": 4.8972015380859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3514506220817566,
      "logits/rejected": 0.4404231309890747,
      "logps/chosen": -363.7027282714844,
      "logps/rejected": -399.4655456542969,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3080919682979584,
      "rewards/margins": 0.08750361204147339,
      "rewards/rejected": -0.3955955505371094,
      "step": 228
    },
    {
      "epoch": 0.02996646765355361,
      "grad_norm": 4.737015724182129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21634912490844727,
      "logits/rejected": 0.09646666795015335,
      "logps/chosen": -387.8190002441406,
      "logps/rejected": -508.74468994140625,
      "loss": 0.5452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0022029951214790344,
      "rewards/margins": 0.34748518466949463,
      "rewards/rejected": -0.3452821671962738,
      "step": 229
    },
    {
      "epoch": 0.030097325590905374,
      "grad_norm": 4.191524505615234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2831606864929199,
      "logits/rejected": 0.43964287638664246,
      "logps/chosen": -376.3763427734375,
      "logps/rejected": -362.8734130859375,
      "loss": 0.6371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39840373396873474,
      "rewards/margins": 0.12805023789405823,
      "rewards/rejected": -0.526453971862793,
      "step": 230
    },
    {
      "epoch": 0.030228183528257135,
      "grad_norm": 4.09756326675415,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4619649350643158,
      "logits/rejected": 0.668535590171814,
      "logps/chosen": -341.9045104980469,
      "logps/rejected": -521.0686645507812,
      "loss": 0.5209,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2880021929740906,
      "rewards/margins": 0.44754141569137573,
      "rewards/rejected": -0.7355436086654663,
      "step": 231
    },
    {
      "epoch": 0.0303590414656089,
      "grad_norm": 6.55304479598999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3239702582359314,
      "logits/rejected": 0.3456684947013855,
      "logps/chosen": -604.8966064453125,
      "logps/rejected": -523.162353515625,
      "loss": 0.6969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3518296480178833,
      "rewards/margins": 0.12608745694160461,
      "rewards/rejected": -0.4779171049594879,
      "step": 232
    },
    {
      "epoch": 0.030489899402960662,
      "grad_norm": 3.828798532485962,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35633426904678345,
      "logits/rejected": 0.35634005069732666,
      "logps/chosen": -322.52581787109375,
      "logps/rejected": -385.0384521484375,
      "loss": 0.6085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31049084663391113,
      "rewards/margins": 0.2483169585466385,
      "rewards/rejected": -0.5588077306747437,
      "step": 233
    },
    {
      "epoch": 0.030620757340312423,
      "grad_norm": 5.506887435913086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40429890155792236,
      "logits/rejected": 0.5025910139083862,
      "logps/chosen": -641.557861328125,
      "logps/rejected": -528.5175170898438,
      "loss": 0.6242,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37856560945510864,
      "rewards/margins": 0.18557414412498474,
      "rewards/rejected": -0.564139723777771,
      "step": 234
    },
    {
      "epoch": 0.030751615277664186,
      "grad_norm": 6.027918815612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0019418522715568542,
      "logits/rejected": 0.021937035024166107,
      "logps/chosen": -510.8000793457031,
      "logps/rejected": -453.41387939453125,
      "loss": 0.7973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45063361525535583,
      "rewards/margins": -0.019496358931064606,
      "rewards/rejected": -0.4311372637748718,
      "step": 235
    },
    {
      "epoch": 0.030882473215015947,
      "grad_norm": 6.653013706207275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19273491203784943,
      "logits/rejected": 0.1572134643793106,
      "logps/chosen": -615.7701416015625,
      "logps/rejected": -484.6575622558594,
      "loss": 0.8345,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6363191604614258,
      "rewards/margins": -0.19727648794651031,
      "rewards/rejected": -0.43904271721839905,
      "step": 236
    },
    {
      "epoch": 0.03101333115236771,
      "grad_norm": 4.438225746154785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4508000910282135,
      "logits/rejected": 0.15140831470489502,
      "logps/chosen": -507.39678955078125,
      "logps/rejected": -442.0443420410156,
      "loss": 0.4842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16764965653419495,
      "rewards/margins": 0.5310306549072266,
      "rewards/rejected": -0.6986802816390991,
      "step": 237
    },
    {
      "epoch": 0.031144189089719474,
      "grad_norm": 4.781407833099365,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09329070150852203,
      "logits/rejected": -0.022862911224365234,
      "logps/chosen": -430.43115234375,
      "logps/rejected": -460.0069580078125,
      "loss": 0.4917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17703494429588318,
      "rewards/margins": 0.47930392622947693,
      "rewards/rejected": -0.6563388705253601,
      "step": 238
    },
    {
      "epoch": 0.031275047027071234,
      "grad_norm": 7.10589075088501,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1791592538356781,
      "logits/rejected": 0.24797841906547546,
      "logps/chosen": -410.3333740234375,
      "logps/rejected": -500.1128234863281,
      "loss": 0.8481,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8723381757736206,
      "rewards/margins": -0.20679381489753723,
      "rewards/rejected": -0.6655443906784058,
      "step": 239
    },
    {
      "epoch": 0.031405904964423,
      "grad_norm": 5.512359619140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5155239105224609,
      "logits/rejected": 0.38045233488082886,
      "logps/chosen": -493.917724609375,
      "logps/rejected": -447.11297607421875,
      "loss": 0.8098,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.7369743585586548,
      "rewards/margins": -0.1914258897304535,
      "rewards/rejected": -0.5455484390258789,
      "step": 240
    },
    {
      "epoch": 0.03153676290177476,
      "grad_norm": 4.384058475494385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26516208052635193,
      "logits/rejected": 0.29982855916023254,
      "logps/chosen": -337.6627197265625,
      "logps/rejected": -448.90789794921875,
      "loss": 0.5394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23584330081939697,
      "rewards/margins": 0.360574334859848,
      "rewards/rejected": -0.5964176654815674,
      "step": 241
    },
    {
      "epoch": 0.031667620839126526,
      "grad_norm": 4.586907386779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22326433658599854,
      "logits/rejected": 0.23821702599525452,
      "logps/chosen": -429.98590087890625,
      "logps/rejected": -345.3206481933594,
      "loss": 0.5964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.486237496137619,
      "rewards/margins": 0.24504870176315308,
      "rewards/rejected": -0.7312862277030945,
      "step": 242
    },
    {
      "epoch": 0.03179847877647828,
      "grad_norm": 5.443427562713623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5499683618545532,
      "logits/rejected": 0.6707589626312256,
      "logps/chosen": -313.1552734375,
      "logps/rejected": -472.08343505859375,
      "loss": 0.7309,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6050956845283508,
      "rewards/margins": -0.051337435841560364,
      "rewards/rejected": -0.5537583231925964,
      "step": 243
    },
    {
      "epoch": 0.031929336713830046,
      "grad_norm": 4.371643543243408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3841451406478882,
      "logits/rejected": 0.3242434561252594,
      "logps/chosen": -302.9096374511719,
      "logps/rejected": -327.9336853027344,
      "loss": 0.6871,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2832355499267578,
      "rewards/margins": 0.04111023619771004,
      "rewards/rejected": -0.32434576749801636,
      "step": 244
    },
    {
      "epoch": 0.03206019465118181,
      "grad_norm": 4.97109842300415,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4776446223258972,
      "logits/rejected": 0.1792898029088974,
      "logps/chosen": -502.1082763671875,
      "logps/rejected": -405.71539306640625,
      "loss": 0.5069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2773929834365845,
      "rewards/margins": 0.5073764324188232,
      "rewards/rejected": -0.7847695350646973,
      "step": 245
    },
    {
      "epoch": 0.032191052588533574,
      "grad_norm": 5.060812950134277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26775306463241577,
      "logits/rejected": 0.6423653960227966,
      "logps/chosen": -284.58123779296875,
      "logps/rejected": -277.360107421875,
      "loss": 0.8222,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5210809707641602,
      "rewards/margins": -0.21312713623046875,
      "rewards/rejected": -0.3079538345336914,
      "step": 246
    },
    {
      "epoch": 0.03232191052588534,
      "grad_norm": 5.92545747756958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3193414509296417,
      "logits/rejected": 0.29941967129707336,
      "logps/chosen": -599.7767333984375,
      "logps/rejected": -764.8754272460938,
      "loss": 0.6147,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5615814328193665,
      "rewards/margins": 0.264364629983902,
      "rewards/rejected": -0.8259460926055908,
      "step": 247
    },
    {
      "epoch": 0.0324527684632371,
      "grad_norm": 5.56595516204834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5611213445663452,
      "logits/rejected": 0.6147809028625488,
      "logps/chosen": -583.2975463867188,
      "logps/rejected": -578.548828125,
      "loss": 0.6993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5807011127471924,
      "rewards/margins": 0.21057051420211792,
      "rewards/rejected": -0.7912716269493103,
      "step": 248
    },
    {
      "epoch": 0.03258362640058886,
      "grad_norm": 4.617050647735596,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0767006129026413,
      "logits/rejected": 0.1036556139588356,
      "logps/chosen": -359.3365478515625,
      "logps/rejected": -392.1581726074219,
      "loss": 0.6454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.22908058762550354,
      "rewards/margins": 0.15728341042995453,
      "rewards/rejected": -0.38636401295661926,
      "step": 249
    },
    {
      "epoch": 0.03271448433794062,
      "grad_norm": 5.229171276092529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6496633291244507,
      "logits/rejected": 0.5952814817428589,
      "logps/chosen": -604.9987182617188,
      "logps/rejected": -706.2393188476562,
      "loss": 0.7003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17217275500297546,
      "rewards/margins": 0.05216693878173828,
      "rewards/rejected": -0.22433967888355255,
      "step": 250
    },
    {
      "epoch": 0.032845342275292386,
      "grad_norm": 4.919482707977295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1166970431804657,
      "logits/rejected": -0.07415224611759186,
      "logps/chosen": -476.57513427734375,
      "logps/rejected": -474.1702880859375,
      "loss": 0.7234,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28642523288726807,
      "rewards/margins": -0.011326611042022705,
      "rewards/rejected": -0.27509862184524536,
      "step": 251
    },
    {
      "epoch": 0.03297620021264415,
      "grad_norm": 4.839311122894287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07771134376525879,
      "logits/rejected": 0.31097325682640076,
      "logps/chosen": -451.8902893066406,
      "logps/rejected": -337.67230224609375,
      "loss": 0.6199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28054332733154297,
      "rewards/margins": 0.17063942551612854,
      "rewards/rejected": -0.4511827528476715,
      "step": 252
    },
    {
      "epoch": 0.03310705814999591,
      "grad_norm": 4.33993673324585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4398200511932373,
      "logits/rejected": 0.5722271203994751,
      "logps/chosen": -512.3703002929688,
      "logps/rejected": -523.0138549804688,
      "loss": 0.5165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35039517283439636,
      "rewards/margins": 0.4518989324569702,
      "rewards/rejected": -0.8022941946983337,
      "step": 253
    },
    {
      "epoch": 0.03323791608734767,
      "grad_norm": 4.1630401611328125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5952610969543457,
      "logits/rejected": 0.3444269597530365,
      "logps/chosen": -565.7742919921875,
      "logps/rejected": -516.3785400390625,
      "loss": 0.4811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2667558789253235,
      "rewards/margins": 0.6816452145576477,
      "rewards/rejected": -0.9484010934829712,
      "step": 254
    },
    {
      "epoch": 0.033368774024699434,
      "grad_norm": 5.665619850158691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7256820797920227,
      "logits/rejected": 0.5449804663658142,
      "logps/chosen": -605.892333984375,
      "logps/rejected": -538.336181640625,
      "loss": 0.6298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4888923764228821,
      "rewards/margins": 0.19073832035064697,
      "rewards/rejected": -0.679630696773529,
      "step": 255
    },
    {
      "epoch": 0.0334996319620512,
      "grad_norm": 5.029339790344238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3458904027938843,
      "logits/rejected": 0.2278502881526947,
      "logps/chosen": -476.0877685546875,
      "logps/rejected": -425.63037109375,
      "loss": 0.6241,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47982996702194214,
      "rewards/margins": 0.2158498913049698,
      "rewards/rejected": -0.6956798434257507,
      "step": 256
    },
    {
      "epoch": 0.03363048989940296,
      "grad_norm": 3.539057493209839,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14332528412342072,
      "logits/rejected": -0.017828477546572685,
      "logps/chosen": -285.2547912597656,
      "logps/rejected": -294.3752136230469,
      "loss": 0.5603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13848458230495453,
      "rewards/margins": 0.35573044419288635,
      "rewards/rejected": -0.4942150413990021,
      "step": 257
    },
    {
      "epoch": 0.033761347836754725,
      "grad_norm": 4.219119548797607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4322986602783203,
      "logits/rejected": 0.31669899821281433,
      "logps/chosen": -354.9842224121094,
      "logps/rejected": -288.5286865234375,
      "loss": 0.6852,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34230920672416687,
      "rewards/margins": 0.031638529151678085,
      "rewards/rejected": -0.37394773960113525,
      "step": 258
    },
    {
      "epoch": 0.03389220577410649,
      "grad_norm": 5.107654571533203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23614534735679626,
      "logits/rejected": 0.21490907669067383,
      "logps/chosen": -339.9158935546875,
      "logps/rejected": -325.318115234375,
      "loss": 0.7577,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5852544903755188,
      "rewards/margins": -0.1035182997584343,
      "rewards/rejected": -0.4817361831665039,
      "step": 259
    },
    {
      "epoch": 0.034023063711458246,
      "grad_norm": 4.72564697265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2315375804901123,
      "logits/rejected": 0.4161727726459503,
      "logps/chosen": -281.0024719238281,
      "logps/rejected": -433.94049072265625,
      "loss": 0.63,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.238981232047081,
      "rewards/margins": 0.16289137303829193,
      "rewards/rejected": -0.4018726050853729,
      "step": 260
    },
    {
      "epoch": 0.03415392164881001,
      "grad_norm": 5.317502021789551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29074206948280334,
      "logits/rejected": 0.4434715509414673,
      "logps/chosen": -406.83966064453125,
      "logps/rejected": -442.87310791015625,
      "loss": 0.6174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4821489453315735,
      "rewards/margins": 0.2942405343055725,
      "rewards/rejected": -0.776389479637146,
      "step": 261
    },
    {
      "epoch": 0.03428477958616177,
      "grad_norm": 4.761331558227539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3028906583786011,
      "logits/rejected": 0.12237473577260971,
      "logps/chosen": -277.29998779296875,
      "logps/rejected": -376.3660583496094,
      "loss": 0.6259,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43400612473487854,
      "rewards/margins": 0.16516359150409698,
      "rewards/rejected": -0.5991697311401367,
      "step": 262
    },
    {
      "epoch": 0.03441563752351354,
      "grad_norm": 4.974634170532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07767312228679657,
      "logits/rejected": 0.004098616540431976,
      "logps/chosen": -280.974853515625,
      "logps/rejected": -387.29290771484375,
      "loss": 0.5834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39999598264694214,
      "rewards/margins": 0.255113422870636,
      "rewards/rejected": -0.6551094055175781,
      "step": 263
    },
    {
      "epoch": 0.0345464954608653,
      "grad_norm": 4.314610481262207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.361573725938797,
      "logits/rejected": 0.36622923612594604,
      "logps/chosen": -341.58563232421875,
      "logps/rejected": -361.1927490234375,
      "loss": 0.6689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49075984954833984,
      "rewards/margins": 0.0800117552280426,
      "rewards/rejected": -0.5707716345787048,
      "step": 264
    },
    {
      "epoch": 0.03467735339821706,
      "grad_norm": 5.281688690185547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1688232123851776,
      "logits/rejected": 0.28671225905418396,
      "logps/chosen": -342.1351013183594,
      "logps/rejected": -437.8913269042969,
      "loss": 0.7513,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5262924432754517,
      "rewards/margins": -0.08640548586845398,
      "rewards/rejected": -0.4398869276046753,
      "step": 265
    },
    {
      "epoch": 0.03480821133556882,
      "grad_norm": 6.572450637817383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02591582015156746,
      "logits/rejected": 0.16835147142410278,
      "logps/chosen": -392.13775634765625,
      "logps/rejected": -293.2984313964844,
      "loss": 0.7501,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5094913840293884,
      "rewards/margins": -0.06421814858913422,
      "rewards/rejected": -0.44527319073677063,
      "step": 266
    },
    {
      "epoch": 0.034939069272920585,
      "grad_norm": 5.259457111358643,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4099973738193512,
      "logits/rejected": 0.3838554918766022,
      "logps/chosen": -537.3209838867188,
      "logps/rejected": -429.2388610839844,
      "loss": 0.6063,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45241624116897583,
      "rewards/margins": 0.2517494261264801,
      "rewards/rejected": -0.7041656374931335,
      "step": 267
    },
    {
      "epoch": 0.03506992721027235,
      "grad_norm": 7.166561126708984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.681746244430542,
      "logits/rejected": 0.4092065989971161,
      "logps/chosen": -513.4437866210938,
      "logps/rejected": -565.7958984375,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5706033706665039,
      "rewards/margins": 0.1934516727924347,
      "rewards/rejected": -0.7640550136566162,
      "step": 268
    },
    {
      "epoch": 0.03520078514762411,
      "grad_norm": 5.150920867919922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40322309732437134,
      "logits/rejected": 0.43389829993247986,
      "logps/chosen": -850.2542114257812,
      "logps/rejected": -626.0519409179688,
      "loss": 0.4645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3355392515659332,
      "rewards/margins": 0.6290706396102905,
      "rewards/rejected": -0.9646099209785461,
      "step": 269
    },
    {
      "epoch": 0.035331643084975876,
      "grad_norm": 4.5504045486450195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004349663853645325,
      "logits/rejected": 0.32856205105781555,
      "logps/chosen": -514.7254638671875,
      "logps/rejected": -510.634765625,
      "loss": 0.4751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3028150796890259,
      "rewards/margins": 0.5464197397232056,
      "rewards/rejected": -0.8492348194122314,
      "step": 270
    },
    {
      "epoch": 0.03546250102232763,
      "grad_norm": 5.034389972686768,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1503106653690338,
      "logits/rejected": 0.35181668400764465,
      "logps/chosen": -447.4947509765625,
      "logps/rejected": -338.6711730957031,
      "loss": 0.6551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5316963791847229,
      "rewards/margins": 0.16221961379051208,
      "rewards/rejected": -0.6939159035682678,
      "step": 271
    },
    {
      "epoch": 0.0355933589596794,
      "grad_norm": 4.0445876121521,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23128508031368256,
      "logits/rejected": -0.1261998414993286,
      "logps/chosen": -355.063720703125,
      "logps/rejected": -472.8136901855469,
      "loss": 0.5558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40828627347946167,
      "rewards/margins": 0.3800518214702606,
      "rewards/rejected": -0.7883381247520447,
      "step": 272
    },
    {
      "epoch": 0.03572421689703116,
      "grad_norm": 4.8137712478637695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2445928156375885,
      "logits/rejected": 0.411453515291214,
      "logps/chosen": -395.67047119140625,
      "logps/rejected": -591.0692138671875,
      "loss": 0.5065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4109247028827667,
      "rewards/margins": 0.46897321939468384,
      "rewards/rejected": -0.8798978924751282,
      "step": 273
    },
    {
      "epoch": 0.035855074834382925,
      "grad_norm": 7.93860387802124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047065041959285736,
      "logits/rejected": 0.10105772316455841,
      "logps/chosen": -790.2926025390625,
      "logps/rejected": -599.2424926757812,
      "loss": 0.9381,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6135272979736328,
      "rewards/margins": -0.38621407747268677,
      "rewards/rejected": -0.22731323540210724,
      "step": 274
    },
    {
      "epoch": 0.03598593277173469,
      "grad_norm": 4.841496467590332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4693457782268524,
      "logits/rejected": 0.6083489656448364,
      "logps/chosen": -398.3099670410156,
      "logps/rejected": -550.173095703125,
      "loss": 0.4267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3916477560997009,
      "rewards/margins": 0.9104536175727844,
      "rewards/rejected": -1.3021013736724854,
      "step": 275
    },
    {
      "epoch": 0.036116790709086445,
      "grad_norm": 4.734603404998779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1908791959285736,
      "logits/rejected": 0.07280156761407852,
      "logps/chosen": -559.9161376953125,
      "logps/rejected": -421.79327392578125,
      "loss": 0.6004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5800946950912476,
      "rewards/margins": 0.27557599544525146,
      "rewards/rejected": -0.8556707501411438,
      "step": 276
    },
    {
      "epoch": 0.03624764864643821,
      "grad_norm": 5.377972602844238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45938193798065186,
      "logits/rejected": 0.3211559057235718,
      "logps/chosen": -515.2651977539062,
      "logps/rejected": -389.06903076171875,
      "loss": 0.6324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6258628964424133,
      "rewards/margins": 0.26844120025634766,
      "rewards/rejected": -0.8943041563034058,
      "step": 277
    },
    {
      "epoch": 0.03637850658378997,
      "grad_norm": 4.815244197845459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3201729655265808,
      "logits/rejected": 0.30892401933670044,
      "logps/chosen": -266.2329406738281,
      "logps/rejected": -460.513427734375,
      "loss": 0.4821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6343601942062378,
      "rewards/margins": 0.6002616286277771,
      "rewards/rejected": -1.2346218824386597,
      "step": 278
    },
    {
      "epoch": 0.036509364521141736,
      "grad_norm": 4.775508403778076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3283112645149231,
      "logits/rejected": 0.3223389685153961,
      "logps/chosen": -540.3406982421875,
      "logps/rejected": -385.1600646972656,
      "loss": 0.6026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34304457902908325,
      "rewards/margins": 0.2639945149421692,
      "rewards/rejected": -0.6070390939712524,
      "step": 279
    },
    {
      "epoch": 0.0366402224584935,
      "grad_norm": 7.134719371795654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5158048272132874,
      "logits/rejected": 0.611487090587616,
      "logps/chosen": -397.5939636230469,
      "logps/rejected": -475.3860778808594,
      "loss": 0.9385,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7209994792938232,
      "rewards/margins": -0.35530224442481995,
      "rewards/rejected": -0.36569729447364807,
      "step": 280
    },
    {
      "epoch": 0.036771080395845264,
      "grad_norm": 5.900284767150879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.353054940700531,
      "logits/rejected": 0.2425847202539444,
      "logps/chosen": -453.2152099609375,
      "logps/rejected": -548.1212768554688,
      "loss": 0.7242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6023125052452087,
      "rewards/margins": 0.08151113986968994,
      "rewards/rejected": -0.6838235855102539,
      "step": 281
    },
    {
      "epoch": 0.03690193833319702,
      "grad_norm": 7.460247039794922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47798338532447815,
      "logits/rejected": 0.47057798504829407,
      "logps/chosen": -552.0781860351562,
      "logps/rejected": -516.76953125,
      "loss": 0.7432,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5551136136054993,
      "rewards/margins": 0.20190852880477905,
      "rewards/rejected": -0.7570221424102783,
      "step": 282
    },
    {
      "epoch": 0.037032796270548785,
      "grad_norm": 6.72042989730835,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23980209231376648,
      "logits/rejected": -0.06390000879764557,
      "logps/chosen": -462.2620544433594,
      "logps/rejected": -379.8782958984375,
      "loss": 0.8073,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.590399980545044,
      "rewards/margins": -0.042486369609832764,
      "rewards/rejected": -0.5479135513305664,
      "step": 283
    },
    {
      "epoch": 0.03716365420790055,
      "grad_norm": 5.491424083709717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40121084451675415,
      "logits/rejected": 0.41930150985717773,
      "logps/chosen": -483.2684326171875,
      "logps/rejected": -354.09564208984375,
      "loss": 0.7158,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5621427297592163,
      "rewards/margins": -0.02331332117319107,
      "rewards/rejected": -0.5388294458389282,
      "step": 284
    },
    {
      "epoch": 0.03729451214525231,
      "grad_norm": 4.565275192260742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28759777545928955,
      "logits/rejected": 0.19992703199386597,
      "logps/chosen": -519.1260375976562,
      "logps/rejected": -427.9156799316406,
      "loss": 0.46,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5210173726081848,
      "rewards/margins": 0.6386467814445496,
      "rewards/rejected": -1.1596641540527344,
      "step": 285
    },
    {
      "epoch": 0.037425370082604076,
      "grad_norm": 5.271563529968262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4885726869106293,
      "logits/rejected": 0.30724936723709106,
      "logps/chosen": -451.2267761230469,
      "logps/rejected": -450.1126403808594,
      "loss": 0.6418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6036813259124756,
      "rewards/margins": 0.3155519664287567,
      "rewards/rejected": -0.9192333221435547,
      "step": 286
    },
    {
      "epoch": 0.03755622801995583,
      "grad_norm": 6.068844795227051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2804253101348877,
      "logits/rejected": 0.30493780970573425,
      "logps/chosen": -348.52423095703125,
      "logps/rejected": -505.77581787109375,
      "loss": 0.6927,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5486434698104858,
      "rewards/margins": 0.09225980937480927,
      "rewards/rejected": -0.6409032940864563,
      "step": 287
    },
    {
      "epoch": 0.037687085957307596,
      "grad_norm": 6.282651901245117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4408447742462158,
      "logits/rejected": 0.1424785852432251,
      "logps/chosen": -452.39007568359375,
      "logps/rejected": -451.2476501464844,
      "loss": 0.7989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6111915707588196,
      "rewards/margins": -0.09340724349021912,
      "rewards/rejected": -0.5177842974662781,
      "step": 288
    },
    {
      "epoch": 0.03781794389465936,
      "grad_norm": 5.021039962768555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32054686546325684,
      "logits/rejected": 0.43401938676834106,
      "logps/chosen": -511.9537048339844,
      "logps/rejected": -350.9972229003906,
      "loss": 0.6858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7363338470458984,
      "rewards/margins": 0.09709013998508453,
      "rewards/rejected": -0.8334239721298218,
      "step": 289
    },
    {
      "epoch": 0.037948801832011124,
      "grad_norm": 3.8324100971221924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1578797847032547,
      "logits/rejected": 0.20861241221427917,
      "logps/chosen": -369.402587890625,
      "logps/rejected": -356.80340576171875,
      "loss": 0.6351,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5648474097251892,
      "rewards/margins": 0.12471179664134979,
      "rewards/rejected": -0.6895592212677002,
      "step": 290
    },
    {
      "epoch": 0.03807965976936289,
      "grad_norm": 4.250813961029053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.224760502576828,
      "logits/rejected": 0.521228551864624,
      "logps/chosen": -438.7769470214844,
      "logps/rejected": -398.7730712890625,
      "loss": 0.4673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47422200441360474,
      "rewards/margins": 0.5638433694839478,
      "rewards/rejected": -1.0380654335021973,
      "step": 291
    },
    {
      "epoch": 0.038210517706714645,
      "grad_norm": 3.653061628341675,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5658254027366638,
      "logits/rejected": 0.2994332015514374,
      "logps/chosen": -379.86865234375,
      "logps/rejected": -345.53741455078125,
      "loss": 0.5768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3664526045322418,
      "rewards/margins": 0.32985442876815796,
      "rewards/rejected": -0.6963070034980774,
      "step": 292
    },
    {
      "epoch": 0.03834137564406641,
      "grad_norm": 6.024176120758057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4183366894721985,
      "logits/rejected": 0.3342351019382477,
      "logps/chosen": -498.51361083984375,
      "logps/rejected": -552.7994384765625,
      "loss": 0.5966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4533214569091797,
      "rewards/margins": 0.33934521675109863,
      "rewards/rejected": -0.7926666736602783,
      "step": 293
    },
    {
      "epoch": 0.03847223358141817,
      "grad_norm": 6.038728713989258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20387035608291626,
      "logits/rejected": 0.3595498204231262,
      "logps/chosen": -387.42138671875,
      "logps/rejected": -351.7195739746094,
      "loss": 0.7575,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48936864733695984,
      "rewards/margins": -0.050462543964385986,
      "rewards/rejected": -0.43890607357025146,
      "step": 294
    },
    {
      "epoch": 0.038603091518769936,
      "grad_norm": 3.6693100929260254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5108063817024231,
      "logits/rejected": 0.41421231627464294,
      "logps/chosen": -350.9541015625,
      "logps/rejected": -333.2620849609375,
      "loss": 0.5828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4867361783981323,
      "rewards/margins": 0.32821357250213623,
      "rewards/rejected": -0.8149498105049133,
      "step": 295
    },
    {
      "epoch": 0.0387339494561217,
      "grad_norm": 5.21578311920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6088750958442688,
      "logits/rejected": 0.5543259978294373,
      "logps/chosen": -550.9296875,
      "logps/rejected": -449.0247497558594,
      "loss": 0.5501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30563944578170776,
      "rewards/margins": 0.49800169467926025,
      "rewards/rejected": -0.8036412000656128,
      "step": 296
    },
    {
      "epoch": 0.03886480739347346,
      "grad_norm": 5.4187445640563965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24516254663467407,
      "logits/rejected": 0.18276332318782806,
      "logps/chosen": -380.3341369628906,
      "logps/rejected": -460.50433349609375,
      "loss": 0.7308,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5361271500587463,
      "rewards/margins": -0.03961925953626633,
      "rewards/rejected": -0.496507853269577,
      "step": 297
    },
    {
      "epoch": 0.03899566533082522,
      "grad_norm": 6.311671733856201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33820897340774536,
      "logits/rejected": 0.24920403957366943,
      "logps/chosen": -383.7987365722656,
      "logps/rejected": -354.95159912109375,
      "loss": 0.7336,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7322078943252563,
      "rewards/margins": -0.005370713770389557,
      "rewards/rejected": -0.726837158203125,
      "step": 298
    },
    {
      "epoch": 0.039126523268176984,
      "grad_norm": 5.353372573852539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43976080417633057,
      "logits/rejected": 0.593671977519989,
      "logps/chosen": -418.5880126953125,
      "logps/rejected": -355.54925537109375,
      "loss": 0.4904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1879587173461914,
      "rewards/margins": 0.631213366985321,
      "rewards/rejected": -0.8191721439361572,
      "step": 299
    },
    {
      "epoch": 0.03925738120552875,
      "grad_norm": 4.53757905960083,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25185030698776245,
      "logits/rejected": 0.1561872959136963,
      "logps/chosen": -301.6915588378906,
      "logps/rejected": -332.2089538574219,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7823549509048462,
      "rewards/margins": 0.08416368067264557,
      "rewards/rejected": -0.8665186166763306,
      "step": 300
    },
    {
      "epoch": 0.03938823914288051,
      "grad_norm": 5.0436177253723145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04782915487885475,
      "logits/rejected": 0.0799856185913086,
      "logps/chosen": -445.507568359375,
      "logps/rejected": -492.820068359375,
      "loss": 0.5981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4023193418979645,
      "rewards/margins": 0.26884955167770386,
      "rewards/rejected": -0.6711689233779907,
      "step": 301
    },
    {
      "epoch": 0.039519097080232275,
      "grad_norm": 4.838476657867432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.550308108329773,
      "logits/rejected": 0.5241241455078125,
      "logps/chosen": -547.9292602539062,
      "logps/rejected": -385.43548583984375,
      "loss": 0.6147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4084964692592621,
      "rewards/margins": 0.1778959482908249,
      "rewards/rejected": -0.5863924026489258,
      "step": 302
    },
    {
      "epoch": 0.03964995501758403,
      "grad_norm": 4.214456081390381,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3120103180408478,
      "logits/rejected": 0.10156922042369843,
      "logps/chosen": -328.4974060058594,
      "logps/rejected": -281.1789855957031,
      "loss": 0.6991,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4928235709667206,
      "rewards/margins": 0.046898264437913895,
      "rewards/rejected": -0.5397219061851501,
      "step": 303
    },
    {
      "epoch": 0.039780812954935796,
      "grad_norm": 4.956816673278809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08824694901704788,
      "logits/rejected": 0.09921348094940186,
      "logps/chosen": -371.0183410644531,
      "logps/rejected": -363.83856201171875,
      "loss": 0.6962,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5950672030448914,
      "rewards/margins": 0.0351167656481266,
      "rewards/rejected": -0.63018399477005,
      "step": 304
    },
    {
      "epoch": 0.03991167089228756,
      "grad_norm": 4.740318775177002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28808528184890747,
      "logits/rejected": 0.26636648178100586,
      "logps/chosen": -478.00140380859375,
      "logps/rejected": -318.99261474609375,
      "loss": 0.5425,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3506176173686981,
      "rewards/margins": 0.3966890275478363,
      "rewards/rejected": -0.7473065853118896,
      "step": 305
    },
    {
      "epoch": 0.04004252882963932,
      "grad_norm": 3.9553425312042236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03157849237322807,
      "logits/rejected": 0.20655997097492218,
      "logps/chosen": -455.2530517578125,
      "logps/rejected": -460.4598388671875,
      "loss": 0.485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3496580123901367,
      "rewards/margins": 0.5539106726646423,
      "rewards/rejected": -0.903568685054779,
      "step": 306
    },
    {
      "epoch": 0.04017338676699109,
      "grad_norm": 3.6928765773773193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4299944043159485,
      "logits/rejected": 0.3186611533164978,
      "logps/chosen": -361.4515686035156,
      "logps/rejected": -545.6176147460938,
      "loss": 0.3619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16996480524539948,
      "rewards/margins": 0.9505263566970825,
      "rewards/rejected": -1.1204911470413208,
      "step": 307
    },
    {
      "epoch": 0.04030424470434285,
      "grad_norm": 9.013162612915039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.438626229763031,
      "logits/rejected": 0.36867934465408325,
      "logps/chosen": -448.2912902832031,
      "logps/rejected": -283.66583251953125,
      "loss": 0.5429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35202112793922424,
      "rewards/margins": 0.43146181106567383,
      "rewards/rejected": -0.7834829688072205,
      "step": 308
    },
    {
      "epoch": 0.04043510264169461,
      "grad_norm": 4.354415416717529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32914093136787415,
      "logits/rejected": 0.23772060871124268,
      "logps/chosen": -470.47528076171875,
      "logps/rejected": -474.840576171875,
      "loss": 0.5962,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3391716182231903,
      "rewards/margins": 0.24231739342212677,
      "rewards/rejected": -0.5814890265464783,
      "step": 309
    },
    {
      "epoch": 0.04056596057904637,
      "grad_norm": 4.024397373199463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32210808992385864,
      "logits/rejected": 0.14882758259773254,
      "logps/chosen": -534.8065795898438,
      "logps/rejected": -490.72265625,
      "loss": 0.4629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23892749845981598,
      "rewards/margins": 0.8015075325965881,
      "rewards/rejected": -1.0404350757598877,
      "step": 310
    },
    {
      "epoch": 0.040696818516398135,
      "grad_norm": 5.587780475616455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29880091547966003,
      "logits/rejected": 0.30154693126678467,
      "logps/chosen": -375.53900146484375,
      "logps/rejected": -441.373291015625,
      "loss": 0.6141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4403972923755646,
      "rewards/margins": 0.2969587445259094,
      "rewards/rejected": -0.7373560667037964,
      "step": 311
    },
    {
      "epoch": 0.0408276764537499,
      "grad_norm": 5.606086254119873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024718886241316795,
      "logits/rejected": -0.0018633045256137848,
      "logps/chosen": -630.2090454101562,
      "logps/rejected": -420.26690673828125,
      "loss": 0.6056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.44801580905914307,
      "rewards/margins": 0.2416282594203949,
      "rewards/rejected": -0.6896440982818604,
      "step": 312
    },
    {
      "epoch": 0.04095853439110166,
      "grad_norm": 5.750139236450195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13645616173744202,
      "logits/rejected": 0.054842397570610046,
      "logps/chosen": -454.8558349609375,
      "logps/rejected": -505.8895568847656,
      "loss": 0.6314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5927807092666626,
      "rewards/margins": 0.46341443061828613,
      "rewards/rejected": -1.0561951398849487,
      "step": 313
    },
    {
      "epoch": 0.04108939232845342,
      "grad_norm": 5.223240375518799,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07074694335460663,
      "logits/rejected": 0.15303143858909607,
      "logps/chosen": -594.8034057617188,
      "logps/rejected": -510.03582763671875,
      "loss": 0.6888,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4095819890499115,
      "rewards/margins": 0.06620997190475464,
      "rewards/rejected": -0.47579193115234375,
      "step": 314
    },
    {
      "epoch": 0.04122025026580518,
      "grad_norm": 5.881261348724365,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17713895440101624,
      "logits/rejected": 0.1563115119934082,
      "logps/chosen": -555.895263671875,
      "logps/rejected": -646.1314086914062,
      "loss": 0.6527,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6984707117080688,
      "rewards/margins": 0.1379789412021637,
      "rewards/rejected": -0.8364496231079102,
      "step": 315
    },
    {
      "epoch": 0.04135110820315695,
      "grad_norm": 4.348703861236572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2707357108592987,
      "logits/rejected": 0.3168409466743469,
      "logps/chosen": -414.7586975097656,
      "logps/rejected": -390.9122009277344,
      "loss": 0.4658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4073253870010376,
      "rewards/margins": 0.6054094433784485,
      "rewards/rejected": -1.0127347707748413,
      "step": 316
    },
    {
      "epoch": 0.04148196614050871,
      "grad_norm": 5.977775573730469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13429640233516693,
      "logits/rejected": 0.32549285888671875,
      "logps/chosen": -359.7897644042969,
      "logps/rejected": -494.740234375,
      "loss": 0.687,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6300893425941467,
      "rewards/margins": 0.19057047367095947,
      "rewards/rejected": -0.820659875869751,
      "step": 317
    },
    {
      "epoch": 0.041612824077860475,
      "grad_norm": 5.767075538635254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1763259768486023,
      "logits/rejected": 0.2899007201194763,
      "logps/chosen": -608.9200439453125,
      "logps/rejected": -450.07379150390625,
      "loss": 0.7239,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29400119185447693,
      "rewards/margins": 0.15814456343650818,
      "rewards/rejected": -0.4521457552909851,
      "step": 318
    },
    {
      "epoch": 0.04174368201521224,
      "grad_norm": 4.394071578979492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4884413778781891,
      "logits/rejected": 0.40911269187927246,
      "logps/chosen": -396.3707580566406,
      "logps/rejected": -395.6192321777344,
      "loss": 0.6105,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3694683313369751,
      "rewards/margins": 0.40723443031311035,
      "rewards/rejected": -0.7767027616500854,
      "step": 319
    },
    {
      "epoch": 0.041874539952563995,
      "grad_norm": 3.613377571105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1301773488521576,
      "logits/rejected": 3.7007033824920654e-05,
      "logps/chosen": -206.9010467529297,
      "logps/rejected": -245.0270538330078,
      "loss": 0.6505,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3972427248954773,
      "rewards/margins": 0.09943942725658417,
      "rewards/rejected": -0.49668216705322266,
      "step": 320
    },
    {
      "epoch": 0.04200539788991576,
      "grad_norm": 4.0028557777404785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6528405547142029,
      "logits/rejected": 0.41055533289909363,
      "logps/chosen": -473.3560485839844,
      "logps/rejected": -428.9506530761719,
      "loss": 0.5092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42504408955574036,
      "rewards/margins": 0.6664940714836121,
      "rewards/rejected": -1.0915380716323853,
      "step": 321
    },
    {
      "epoch": 0.04213625582726752,
      "grad_norm": 7.046688079833984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3123434782028198,
      "logits/rejected": 0.3709983825683594,
      "logps/chosen": -418.6291198730469,
      "logps/rejected": -474.7488098144531,
      "loss": 0.8606,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8661642074584961,
      "rewards/margins": -0.2499156892299652,
      "rewards/rejected": -0.6162484884262085,
      "step": 322
    },
    {
      "epoch": 0.04226711376461929,
      "grad_norm": 6.480327129364014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3393354117870331,
      "logits/rejected": 0.3003735840320587,
      "logps/chosen": -496.8702087402344,
      "logps/rejected": -434.9725341796875,
      "loss": 0.8271,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8327785730361938,
      "rewards/margins": -0.14675159752368927,
      "rewards/rejected": -0.686026930809021,
      "step": 323
    },
    {
      "epoch": 0.04239797170197105,
      "grad_norm": 4.862349033355713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44144096970558167,
      "logits/rejected": 0.2658373713493347,
      "logps/chosen": -469.2283020019531,
      "logps/rejected": -386.7000732421875,
      "loss": 0.6952,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5603595972061157,
      "rewards/margins": 0.036246683448553085,
      "rewards/rejected": -0.5966062545776367,
      "step": 324
    },
    {
      "epoch": 0.04252882963932281,
      "grad_norm": 3.941622734069824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4473135471343994,
      "logits/rejected": 0.21104393899440765,
      "logps/chosen": -317.3961486816406,
      "logps/rejected": -299.91717529296875,
      "loss": 0.6199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7320964932441711,
      "rewards/margins": 0.2213314324617386,
      "rewards/rejected": -0.9534279108047485,
      "step": 325
    },
    {
      "epoch": 0.04265968757667457,
      "grad_norm": 4.489743232727051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3016971945762634,
      "logits/rejected": 0.3364534378051758,
      "logps/chosen": -466.7030944824219,
      "logps/rejected": -435.1914367675781,
      "loss": 0.5051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5478769540786743,
      "rewards/margins": 0.496024489402771,
      "rewards/rejected": -1.0439014434814453,
      "step": 326
    },
    {
      "epoch": 0.042790545514026335,
      "grad_norm": 5.330624103546143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12391272187232971,
      "logits/rejected": 0.22888723015785217,
      "logps/chosen": -360.1882019042969,
      "logps/rejected": -439.3838195800781,
      "loss": 0.7153,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5995538830757141,
      "rewards/margins": -0.012709993869066238,
      "rewards/rejected": -0.5868439078330994,
      "step": 327
    },
    {
      "epoch": 0.0429214034513781,
      "grad_norm": 10.52452564239502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5667113065719604,
      "logits/rejected": 0.5099827647209167,
      "logps/chosen": -599.6472778320312,
      "logps/rejected": -420.03509521484375,
      "loss": 0.8481,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.595486044883728,
      "rewards/margins": -0.15714474022388458,
      "rewards/rejected": -0.43834131956100464,
      "step": 328
    },
    {
      "epoch": 0.04305226138872986,
      "grad_norm": 5.106630325317383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4446288049221039,
      "logits/rejected": 0.3833131492137909,
      "logps/chosen": -293.18231201171875,
      "logps/rejected": -390.3575744628906,
      "loss": 0.7424,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.647983729839325,
      "rewards/margins": 0.12180481851100922,
      "rewards/rejected": -0.7697886228561401,
      "step": 329
    },
    {
      "epoch": 0.043183119326081626,
      "grad_norm": 5.730265140533447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28892117738723755,
      "logits/rejected": 0.31704002618789673,
      "logps/chosen": -466.5412902832031,
      "logps/rejected": -402.6695861816406,
      "loss": 0.5203,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46911221742630005,
      "rewards/margins": 0.6675317287445068,
      "rewards/rejected": -1.1366440057754517,
      "step": 330
    },
    {
      "epoch": 0.04331397726343338,
      "grad_norm": 4.535353660583496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48381465673446655,
      "logits/rejected": 0.1390381008386612,
      "logps/chosen": -397.1177062988281,
      "logps/rejected": -401.139404296875,
      "loss": 0.5434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44300687313079834,
      "rewards/margins": 0.39358335733413696,
      "rewards/rejected": -0.8365902304649353,
      "step": 331
    },
    {
      "epoch": 0.04344483520078515,
      "grad_norm": 5.96033239364624,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4513135254383087,
      "logits/rejected": 0.657164454460144,
      "logps/chosen": -509.11126708984375,
      "logps/rejected": -483.3865966796875,
      "loss": 0.7725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7068824172019958,
      "rewards/margins": -0.0883295089006424,
      "rewards/rejected": -0.6185529828071594,
      "step": 332
    },
    {
      "epoch": 0.04357569313813691,
      "grad_norm": 6.301385879516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4285838305950165,
      "logits/rejected": 0.1714092642068863,
      "logps/chosen": -314.2547912597656,
      "logps/rejected": -379.7529602050781,
      "loss": 0.5127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29686039686203003,
      "rewards/margins": 0.4449198544025421,
      "rewards/rejected": -0.7417802810668945,
      "step": 333
    },
    {
      "epoch": 0.043706551075488674,
      "grad_norm": 4.699800491333008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2647320628166199,
      "logits/rejected": 0.3295484781265259,
      "logps/chosen": -309.4952087402344,
      "logps/rejected": -323.27667236328125,
      "loss": 0.5628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.514696478843689,
      "rewards/margins": 0.32923316955566406,
      "rewards/rejected": -0.8439297080039978,
      "step": 334
    },
    {
      "epoch": 0.04383740901284044,
      "grad_norm": 7.043466091156006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4830375909805298,
      "logits/rejected": 0.40320858359336853,
      "logps/chosen": -659.7418823242188,
      "logps/rejected": -548.2400512695312,
      "loss": 0.8891,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5548927783966064,
      "rewards/margins": -0.24838122725486755,
      "rewards/rejected": -0.3065114915370941,
      "step": 335
    },
    {
      "epoch": 0.043968266950192195,
      "grad_norm": 4.300783157348633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5337944626808167,
      "logits/rejected": 0.27969515323638916,
      "logps/chosen": -423.9654846191406,
      "logps/rejected": -426.0820617675781,
      "loss": 0.5413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6679167151451111,
      "rewards/margins": 0.46227970719337463,
      "rewards/rejected": -1.130196452140808,
      "step": 336
    },
    {
      "epoch": 0.04409912488754396,
      "grad_norm": 5.625675201416016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36006492376327515,
      "logits/rejected": 0.1723405122756958,
      "logps/chosen": -397.7917785644531,
      "logps/rejected": -311.90411376953125,
      "loss": 0.7358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6702578663825989,
      "rewards/margins": 0.09412060678005219,
      "rewards/rejected": -0.7643784880638123,
      "step": 337
    },
    {
      "epoch": 0.04422998282489572,
      "grad_norm": 6.128083229064941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48696252703666687,
      "logits/rejected": 0.5413917899131775,
      "logps/chosen": -449.3508605957031,
      "logps/rejected": -425.4649353027344,
      "loss": 0.7572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5897800326347351,
      "rewards/margins": -0.0637035146355629,
      "rewards/rejected": -0.5260764956474304,
      "step": 338
    },
    {
      "epoch": 0.044360840762247486,
      "grad_norm": 8.216541290283203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06919179111719131,
      "logits/rejected": 0.24503767490386963,
      "logps/chosen": -597.565185546875,
      "logps/rejected": -539.2105712890625,
      "loss": 0.7627,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6614869832992554,
      "rewards/margins": -0.05542777106165886,
      "rewards/rejected": -0.6060592532157898,
      "step": 339
    },
    {
      "epoch": 0.04449169869959925,
      "grad_norm": 4.538233757019043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38181376457214355,
      "logits/rejected": 0.39029648900032043,
      "logps/chosen": -483.75323486328125,
      "logps/rejected": -452.52618408203125,
      "loss": 0.6512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3036687970161438,
      "rewards/margins": 0.13115385174751282,
      "rewards/rejected": -0.4348226487636566,
      "step": 340
    },
    {
      "epoch": 0.044622556636951013,
      "grad_norm": 5.91779899597168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2004283219575882,
      "logits/rejected": 0.2246713787317276,
      "logps/chosen": -361.9729309082031,
      "logps/rejected": -521.5323486328125,
      "loss": 0.7256,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.649956226348877,
      "rewards/margins": 0.06707629561424255,
      "rewards/rejected": -0.7170325517654419,
      "step": 341
    },
    {
      "epoch": 0.04475341457430277,
      "grad_norm": 5.325444221496582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48191624879837036,
      "logits/rejected": 0.4042791724205017,
      "logps/chosen": -526.9571533203125,
      "logps/rejected": -334.2409973144531,
      "loss": 0.5989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5402150750160217,
      "rewards/margins": 0.27155762910842896,
      "rewards/rejected": -0.8117727041244507,
      "step": 342
    },
    {
      "epoch": 0.044884272511654534,
      "grad_norm": 3.7208011150360107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4347180128097534,
      "logits/rejected": 0.4261515438556671,
      "logps/chosen": -511.4112854003906,
      "logps/rejected": -380.9693298339844,
      "loss": 0.4705,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0885690450668335,
      "rewards/margins": 0.7005400061607361,
      "rewards/rejected": -0.7891089916229248,
      "step": 343
    },
    {
      "epoch": 0.0450151304490063,
      "grad_norm": 3.7295665740966797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36632269620895386,
      "logits/rejected": 0.4731137454509735,
      "logps/chosen": -437.4705505371094,
      "logps/rejected": -355.2384033203125,
      "loss": 0.4927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24700680375099182,
      "rewards/margins": 0.5546793341636658,
      "rewards/rejected": -0.8016861081123352,
      "step": 344
    },
    {
      "epoch": 0.04514598838635806,
      "grad_norm": 6.2968220710754395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3828314542770386,
      "logits/rejected": 0.16752827167510986,
      "logps/chosen": -716.6494750976562,
      "logps/rejected": -563.9742431640625,
      "loss": 0.5776,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1113453060388565,
      "rewards/margins": 0.3456403911113739,
      "rewards/rejected": -0.4569856822490692,
      "step": 345
    },
    {
      "epoch": 0.045276846323709825,
      "grad_norm": 6.4899115562438965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1386888325214386,
      "logits/rejected": 0.03643109276890755,
      "logps/chosen": -590.6925659179688,
      "logps/rejected": -381.8724365234375,
      "loss": 0.6082,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11699697375297546,
      "rewards/margins": 0.4227835237979889,
      "rewards/rejected": -0.5397804379463196,
      "step": 346
    },
    {
      "epoch": 0.04540770426106158,
      "grad_norm": 5.096562385559082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2516324818134308,
      "logits/rejected": 0.16934612393379211,
      "logps/chosen": -360.4405517578125,
      "logps/rejected": -355.1954650878906,
      "loss": 0.4684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.365997314453125,
      "rewards/margins": 0.5665819048881531,
      "rewards/rejected": -0.9325792789459229,
      "step": 347
    },
    {
      "epoch": 0.045538562198413346,
      "grad_norm": 4.853593349456787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49931541085243225,
      "logits/rejected": 0.2341405153274536,
      "logps/chosen": -399.0271301269531,
      "logps/rejected": -406.79779052734375,
      "loss": 0.7433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6389026641845703,
      "rewards/margins": -0.012499237433075905,
      "rewards/rejected": -0.6264033913612366,
      "step": 348
    },
    {
      "epoch": 0.04566942013576511,
      "grad_norm": 4.560263156890869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31135424971580505,
      "logits/rejected": 0.4288477897644043,
      "logps/chosen": -543.7398681640625,
      "logps/rejected": -562.8826904296875,
      "loss": 0.489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11679859459400177,
      "rewards/margins": 0.5461567044258118,
      "rewards/rejected": -0.4293580949306488,
      "step": 349
    },
    {
      "epoch": 0.045800278073116873,
      "grad_norm": 3.834545135498047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1325097680091858,
      "logits/rejected": -0.08629310876131058,
      "logps/chosen": -403.8008728027344,
      "logps/rejected": -411.4405822753906,
      "loss": 0.5023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3457931578159332,
      "rewards/margins": 0.5125242471694946,
      "rewards/rejected": -0.8583173751831055,
      "step": 350
    },
    {
      "epoch": 0.04593113601046864,
      "grad_norm": 4.859841823577881,
      "learning_rate": 5e-06,
      "logits/chosen": 0.8225787281990051,
      "logits/rejected": 0.8638673424720764,
      "logps/chosen": -386.127685546875,
      "logps/rejected": -509.0428466796875,
      "loss": 0.5232,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32143253087997437,
      "rewards/margins": 0.5787950158119202,
      "rewards/rejected": -0.9002275466918945,
      "step": 351
    },
    {
      "epoch": 0.046061993947820394,
      "grad_norm": 4.64219331741333,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3379310965538025,
      "logits/rejected": 0.2721625566482544,
      "logps/chosen": -415.55499267578125,
      "logps/rejected": -278.8743896484375,
      "loss": 0.6468,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2162235528230667,
      "rewards/margins": 0.1546272188425064,
      "rewards/rejected": -0.3708507716655731,
      "step": 352
    },
    {
      "epoch": 0.04619285188517216,
      "grad_norm": 4.323669910430908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4359947741031647,
      "logits/rejected": 0.3926038444042206,
      "logps/chosen": -335.2861022949219,
      "logps/rejected": -372.2679443359375,
      "loss": 0.6064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1417219191789627,
      "rewards/margins": 0.21621686220169067,
      "rewards/rejected": -0.3579387366771698,
      "step": 353
    },
    {
      "epoch": 0.04632370982252392,
      "grad_norm": 3.967287540435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4636842906475067,
      "logits/rejected": 0.14782601594924927,
      "logps/chosen": -515.33837890625,
      "logps/rejected": -403.92559814453125,
      "loss": 0.4899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2665225863456726,
      "rewards/margins": 0.5653896331787109,
      "rewards/rejected": -0.8319122195243835,
      "step": 354
    },
    {
      "epoch": 0.046454567759875685,
      "grad_norm": 4.920809745788574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30735281109809875,
      "logits/rejected": 0.4664519727230072,
      "logps/chosen": -355.8701477050781,
      "logps/rejected": -407.55902099609375,
      "loss": 0.6805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2229452133178711,
      "rewards/margins": 0.1160566508769989,
      "rewards/rejected": -0.3390018343925476,
      "step": 355
    },
    {
      "epoch": 0.04658542569722745,
      "grad_norm": 5.199960231781006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5471001863479614,
      "logits/rejected": 0.5459061861038208,
      "logps/chosen": -585.5659790039062,
      "logps/rejected": -346.177978515625,
      "loss": 0.6375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44171908497810364,
      "rewards/margins": 0.1557687669992447,
      "rewards/rejected": -0.5974878668785095,
      "step": 356
    },
    {
      "epoch": 0.04671628363457921,
      "grad_norm": 3.754847764968872,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3668473958969116,
      "logits/rejected": 0.3988126218318939,
      "logps/chosen": -451.7940979003906,
      "logps/rejected": -403.95556640625,
      "loss": 0.481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07387657463550568,
      "rewards/margins": 0.5671768188476562,
      "rewards/rejected": -0.49330025911331177,
      "step": 357
    },
    {
      "epoch": 0.04684714157193097,
      "grad_norm": 4.496786117553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29178574681282043,
      "logits/rejected": 0.2977704107761383,
      "logps/chosen": -407.3742370605469,
      "logps/rejected": -384.39337158203125,
      "loss": 0.5319,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4786512553691864,
      "rewards/margins": 0.46364232897758484,
      "rewards/rejected": -0.9422935843467712,
      "step": 358
    },
    {
      "epoch": 0.046977999509282733,
      "grad_norm": 6.6220879554748535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3663659691810608,
      "logits/rejected": 0.7026095986366272,
      "logps/chosen": -324.99078369140625,
      "logps/rejected": -520.138671875,
      "loss": 0.7597,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4051618278026581,
      "rewards/margins": 0.09079037606716156,
      "rewards/rejected": -0.49595221877098083,
      "step": 359
    },
    {
      "epoch": 0.0471088574466345,
      "grad_norm": 7.878411769866943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19109544157981873,
      "logits/rejected": 0.23011232912540436,
      "logps/chosen": -555.9254760742188,
      "logps/rejected": -420.4494323730469,
      "loss": 0.8703,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.9685184955596924,
      "rewards/margins": -0.21939007937908173,
      "rewards/rejected": -0.7491283416748047,
      "step": 360
    },
    {
      "epoch": 0.04723971538398626,
      "grad_norm": 5.519693851470947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37855395674705505,
      "logits/rejected": 0.25814613699913025,
      "logps/chosen": -666.1868286132812,
      "logps/rejected": -623.4678955078125,
      "loss": 0.5482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006916239857673645,
      "rewards/margins": 0.45054376125335693,
      "rewards/rejected": -0.4436275362968445,
      "step": 361
    },
    {
      "epoch": 0.047370573321338025,
      "grad_norm": 5.550156116485596,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07602644711732864,
      "logits/rejected": 0.11730945855379105,
      "logps/chosen": -399.6539001464844,
      "logps/rejected": -524.8466796875,
      "loss": 0.7417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5794531106948853,
      "rewards/margins": 0.03028201311826706,
      "rewards/rejected": -0.6097350716590881,
      "step": 362
    },
    {
      "epoch": 0.04750143125868978,
      "grad_norm": 10.287353515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22098147869110107,
      "logits/rejected": 0.3222389817237854,
      "logps/chosen": -673.2698974609375,
      "logps/rejected": -546.5992431640625,
      "loss": 0.6306,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3474540710449219,
      "rewards/margins": 0.21686898171901703,
      "rewards/rejected": -0.5643230676651001,
      "step": 363
    },
    {
      "epoch": 0.047632289196041545,
      "grad_norm": 3.374514579772949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25939685106277466,
      "logits/rejected": -0.12302251905202866,
      "logps/chosen": -505.9617614746094,
      "logps/rejected": -427.97991943359375,
      "loss": 0.36,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2459770292043686,
      "rewards/margins": 0.9329763054847717,
      "rewards/rejected": -1.1789534091949463,
      "step": 364
    },
    {
      "epoch": 0.04776314713339331,
      "grad_norm": 4.1742472648620605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4788067936897278,
      "logits/rejected": 0.46788233518600464,
      "logps/chosen": -280.953857421875,
      "logps/rejected": -298.6064453125,
      "loss": 0.7104,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5414501428604126,
      "rewards/margins": 0.02580621838569641,
      "rewards/rejected": -0.5672563910484314,
      "step": 365
    },
    {
      "epoch": 0.04789400507074507,
      "grad_norm": 7.333527565002441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43837276101112366,
      "logits/rejected": 0.5532659292221069,
      "logps/chosen": -420.18341064453125,
      "logps/rejected": -490.07464599609375,
      "loss": 0.91,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.883705735206604,
      "rewards/margins": -0.3341071903705597,
      "rewards/rejected": -0.5495985150337219,
      "step": 366
    },
    {
      "epoch": 0.04802486300809684,
      "grad_norm": 4.79817008972168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7854149341583252,
      "logits/rejected": 0.5047216415405273,
      "logps/chosen": -524.4227294921875,
      "logps/rejected": -382.4329833984375,
      "loss": 0.5761,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4342457056045532,
      "rewards/margins": 0.353346049785614,
      "rewards/rejected": -0.787591814994812,
      "step": 367
    },
    {
      "epoch": 0.0481557209454486,
      "grad_norm": 6.352809429168701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10800497978925705,
      "logits/rejected": 0.2636061906814575,
      "logps/chosen": -387.42877197265625,
      "logps/rejected": -402.96697998046875,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6056193113327026,
      "rewards/margins": 0.11250972747802734,
      "rewards/rejected": -0.7181289792060852,
      "step": 368
    },
    {
      "epoch": 0.04828657888280036,
      "grad_norm": 5.795500755310059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18671759963035583,
      "logits/rejected": 0.14370635151863098,
      "logps/chosen": -463.53216552734375,
      "logps/rejected": -428.3824462890625,
      "loss": 0.6633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3800165355205536,
      "rewards/margins": 0.18934211134910583,
      "rewards/rejected": -0.5693587064743042,
      "step": 369
    },
    {
      "epoch": 0.04841743682015212,
      "grad_norm": 5.719783782958984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5187828540802002,
      "logits/rejected": 0.2117764800786972,
      "logps/chosen": -390.083740234375,
      "logps/rejected": -367.75128173828125,
      "loss": 0.7493,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7838624715805054,
      "rewards/margins": -0.02518463507294655,
      "rewards/rejected": -0.7586778402328491,
      "step": 370
    },
    {
      "epoch": 0.048548294757503885,
      "grad_norm": 4.609052658081055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40960270166397095,
      "logits/rejected": 0.3702288269996643,
      "logps/chosen": -423.83428955078125,
      "logps/rejected": -386.14007568359375,
      "loss": 0.584,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6489156484603882,
      "rewards/margins": 0.29099202156066895,
      "rewards/rejected": -0.9399076700210571,
      "step": 371
    },
    {
      "epoch": 0.04867915269485565,
      "grad_norm": 4.22052001953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5767711997032166,
      "logits/rejected": 0.718499481678009,
      "logps/chosen": -515.8160400390625,
      "logps/rejected": -403.9260559082031,
      "loss": 0.5538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3520669937133789,
      "rewards/margins": 0.3919641375541687,
      "rewards/rejected": -0.7440311908721924,
      "step": 372
    },
    {
      "epoch": 0.04881001063220741,
      "grad_norm": 5.339836597442627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.9158172607421875,
      "logits/rejected": 0.6914252638816833,
      "logps/chosen": -542.4751586914062,
      "logps/rejected": -545.0469360351562,
      "loss": 0.5064,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1719736009836197,
      "rewards/margins": 0.5915106534957886,
      "rewards/rejected": -0.7634841799736023,
      "step": 373
    },
    {
      "epoch": 0.04894086856955917,
      "grad_norm": 4.335776329040527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030425770208239555,
      "logits/rejected": 0.08126901090145111,
      "logps/chosen": -456.5830078125,
      "logps/rejected": -475.46429443359375,
      "loss": 0.5738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10356464982032776,
      "rewards/margins": 0.36687394976615906,
      "rewards/rejected": -0.47043856978416443,
      "step": 374
    },
    {
      "epoch": 0.04907172650691093,
      "grad_norm": 4.645480155944824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38417983055114746,
      "logits/rejected": 0.3736307621002197,
      "logps/chosen": -465.94781494140625,
      "logps/rejected": -519.579833984375,
      "loss": 0.5332,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3229566812515259,
      "rewards/margins": 0.45606333017349243,
      "rewards/rejected": -0.779019832611084,
      "step": 375
    },
    {
      "epoch": 0.0492025844442627,
      "grad_norm": 5.324519634246826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3888380825519562,
      "logits/rejected": 0.3206060230731964,
      "logps/chosen": -597.7364501953125,
      "logps/rejected": -472.7025146484375,
      "loss": 0.4952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22066479921340942,
      "rewards/margins": 0.508928120136261,
      "rewards/rejected": -0.7295928597450256,
      "step": 376
    },
    {
      "epoch": 0.04933344238161446,
      "grad_norm": 4.734320163726807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6045580506324768,
      "logits/rejected": 0.584625244140625,
      "logps/chosen": -375.42633056640625,
      "logps/rejected": -440.8330383300781,
      "loss": 0.6224,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.26900118589401245,
      "rewards/margins": 0.3152645528316498,
      "rewards/rejected": -0.5842657089233398,
      "step": 377
    },
    {
      "epoch": 0.049464300318966224,
      "grad_norm": 5.437359809875488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2797057330608368,
      "logits/rejected": 0.30505385994911194,
      "logps/chosen": -504.5202941894531,
      "logps/rejected": -503.3941345214844,
      "loss": 0.5144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5224525332450867,
      "rewards/margins": 0.4365684688091278,
      "rewards/rejected": -0.9590210318565369,
      "step": 378
    },
    {
      "epoch": 0.04959515825631799,
      "grad_norm": 5.0333075523376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34909787774086,
      "logits/rejected": 0.513622522354126,
      "logps/chosen": -401.98297119140625,
      "logps/rejected": -453.35833740234375,
      "loss": 0.7099,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.525621771812439,
      "rewards/margins": 0.17178764939308167,
      "rewards/rejected": -0.6974093914031982,
      "step": 379
    },
    {
      "epoch": 0.049726016193669745,
      "grad_norm": 4.964475154876709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13717612624168396,
      "logits/rejected": 0.07927295565605164,
      "logps/chosen": -320.0760498046875,
      "logps/rejected": -259.5789489746094,
      "loss": 0.7256,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2577781677246094,
      "rewards/margins": -0.051784902811050415,
      "rewards/rejected": -0.20599327981472015,
      "step": 380
    },
    {
      "epoch": 0.04985687413102151,
      "grad_norm": 4.733691692352295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49204397201538086,
      "logits/rejected": 0.3608075678348541,
      "logps/chosen": -492.89678955078125,
      "logps/rejected": -351.9506530761719,
      "loss": 0.5875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2173570692539215,
      "rewards/margins": 0.30047357082366943,
      "rewards/rejected": -0.5178306698799133,
      "step": 381
    },
    {
      "epoch": 0.04998773206837327,
      "grad_norm": 6.483283996582031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5478978753089905,
      "logits/rejected": 0.467539519071579,
      "logps/chosen": -477.31378173828125,
      "logps/rejected": -389.2564697265625,
      "loss": 0.7872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5173025131225586,
      "rewards/margins": -0.08057059347629547,
      "rewards/rejected": -0.4367319643497467,
      "step": 382
    },
    {
      "epoch": 0.050118590005725036,
      "grad_norm": 3.952644109725952,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35198381543159485,
      "logits/rejected": 0.2254774570465088,
      "logps/chosen": -497.94775390625,
      "logps/rejected": -428.8301696777344,
      "loss": 0.4728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02497539296746254,
      "rewards/margins": 0.628230094909668,
      "rewards/rejected": -0.6532055139541626,
      "step": 383
    },
    {
      "epoch": 0.0502494479430768,
      "grad_norm": 5.551051616668701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31607168912887573,
      "logits/rejected": 0.59564208984375,
      "logps/chosen": -512.815185546875,
      "logps/rejected": -425.5899658203125,
      "loss": 0.6451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4508523941040039,
      "rewards/margins": 0.20454846322536469,
      "rewards/rejected": -0.6554008722305298,
      "step": 384
    },
    {
      "epoch": 0.05038030588042856,
      "grad_norm": 4.7678303718566895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3264741599559784,
      "logits/rejected": 0.4961984157562256,
      "logps/chosen": -476.63604736328125,
      "logps/rejected": -441.5626220703125,
      "loss": 0.3955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15712089836597443,
      "rewards/margins": 1.056589126586914,
      "rewards/rejected": -0.899468183517456,
      "step": 385
    },
    {
      "epoch": 0.05051116381778032,
      "grad_norm": 7.467518329620361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25702235102653503,
      "logits/rejected": 0.3387482166290283,
      "logps/chosen": -358.51507568359375,
      "logps/rejected": -552.7086791992188,
      "loss": 0.5317,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17622984945774078,
      "rewards/margins": 0.4306342899799347,
      "rewards/rejected": -0.6068642139434814,
      "step": 386
    },
    {
      "epoch": 0.050642021755132084,
      "grad_norm": 4.66990327835083,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1895490437746048,
      "logits/rejected": 0.26743972301483154,
      "logps/chosen": -243.79940795898438,
      "logps/rejected": -355.79302978515625,
      "loss": 0.5988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5972406268119812,
      "rewards/margins": 0.25658896565437317,
      "rewards/rejected": -0.853829562664032,
      "step": 387
    },
    {
      "epoch": 0.05077287969248385,
      "grad_norm": 5.099401950836182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5573741793632507,
      "logits/rejected": 0.5331375598907471,
      "logps/chosen": -481.830078125,
      "logps/rejected": -403.22161865234375,
      "loss": 0.776,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4496731162071228,
      "rewards/margins": 0.05844744294881821,
      "rewards/rejected": -0.508120596408844,
      "step": 388
    },
    {
      "epoch": 0.05090373762983561,
      "grad_norm": 5.1514892578125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46755433082580566,
      "logits/rejected": 0.39541497826576233,
      "logps/chosen": -531.1399536132812,
      "logps/rejected": -454.01318359375,
      "loss": 0.605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.038180164992809296,
      "rewards/margins": 0.3221462368965149,
      "rewards/rejected": -0.3603263795375824,
      "step": 389
    },
    {
      "epoch": 0.051034595567187375,
      "grad_norm": 6.40706729888916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3649815320968628,
      "logits/rejected": 0.3542778789997101,
      "logps/chosen": -366.4481201171875,
      "logps/rejected": -480.8264465332031,
      "loss": 0.8265,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7480525970458984,
      "rewards/margins": -0.0963214859366417,
      "rewards/rejected": -0.6517311334609985,
      "step": 390
    },
    {
      "epoch": 0.05116545350453913,
      "grad_norm": 4.157971382141113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14984022080898285,
      "logits/rejected": 0.1305897831916809,
      "logps/chosen": -359.5758972167969,
      "logps/rejected": -583.3049926757812,
      "loss": 0.5225,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17129822075366974,
      "rewards/margins": 0.471300333738327,
      "rewards/rejected": -0.642598569393158,
      "step": 391
    },
    {
      "epoch": 0.051296311441890896,
      "grad_norm": 3.9816460609436035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5327082872390747,
      "logits/rejected": 0.4733927845954895,
      "logps/chosen": -490.3185729980469,
      "logps/rejected": -460.5480651855469,
      "loss": 0.4821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4812370240688324,
      "rewards/margins": 0.6656774878501892,
      "rewards/rejected": -1.1469144821166992,
      "step": 392
    },
    {
      "epoch": 0.05142716937924266,
      "grad_norm": 4.2139739990234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32818230986595154,
      "logits/rejected": 0.5714169144630432,
      "logps/chosen": -336.80584716796875,
      "logps/rejected": -376.0710754394531,
      "loss": 0.4709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22960235178470612,
      "rewards/margins": 0.5870074033737183,
      "rewards/rejected": -0.8166097402572632,
      "step": 393
    },
    {
      "epoch": 0.051558027316594424,
      "grad_norm": 4.692296028137207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.254138320684433,
      "logits/rejected": 0.3266112208366394,
      "logps/chosen": -470.7752380371094,
      "logps/rejected": -522.823974609375,
      "loss": 0.5556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5310914516448975,
      "rewards/margins": 0.35953959822654724,
      "rewards/rejected": -0.8906311392784119,
      "step": 394
    },
    {
      "epoch": 0.05168888525394619,
      "grad_norm": 4.228846073150635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30813831090927124,
      "logits/rejected": 0.43667274713516235,
      "logps/chosen": -428.6998291015625,
      "logps/rejected": -462.8140869140625,
      "loss": 0.4428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19115372002124786,
      "rewards/margins": 0.6732363104820251,
      "rewards/rejected": -0.8643900156021118,
      "step": 395
    },
    {
      "epoch": 0.051819743191297944,
      "grad_norm": 7.060044765472412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0755300223827362,
      "logits/rejected": 0.057444725185632706,
      "logps/chosen": -514.0188598632812,
      "logps/rejected": -410.98675537109375,
      "loss": 0.8409,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48966315388679504,
      "rewards/margins": 0.07947397232055664,
      "rewards/rejected": -0.5691371560096741,
      "step": 396
    },
    {
      "epoch": 0.05195060112864971,
      "grad_norm": 4.103015899658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11989480257034302,
      "logits/rejected": 0.2550228238105774,
      "logps/chosen": -575.7409057617188,
      "logps/rejected": -440.15814208984375,
      "loss": 0.5146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2603285014629364,
      "rewards/margins": 0.50152188539505,
      "rewards/rejected": -0.7618503570556641,
      "step": 397
    },
    {
      "epoch": 0.05208145906600147,
      "grad_norm": 3.968024492263794,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5667606592178345,
      "logits/rejected": 0.5617504119873047,
      "logps/chosen": -329.6603698730469,
      "logps/rejected": -373.29376220703125,
      "loss": 0.533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41774389147758484,
      "rewards/margins": 0.42517775297164917,
      "rewards/rejected": -0.8429216742515564,
      "step": 398
    },
    {
      "epoch": 0.052212317003353236,
      "grad_norm": 6.5704145431518555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24092762172222137,
      "logits/rejected": 0.15651409327983856,
      "logps/chosen": -355.4664306640625,
      "logps/rejected": -374.23052978515625,
      "loss": 0.73,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7454679608345032,
      "rewards/margins": 0.07510644197463989,
      "rewards/rejected": -0.8205744028091431,
      "step": 399
    },
    {
      "epoch": 0.052343174940705,
      "grad_norm": 3.4290237426757812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6810815334320068,
      "logits/rejected": 0.5640336871147156,
      "logps/chosen": -414.2606506347656,
      "logps/rejected": -375.78985595703125,
      "loss": 0.516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3897251486778259,
      "rewards/margins": 0.7167699337005615,
      "rewards/rejected": -1.1064951419830322,
      "step": 400
    },
    {
      "epoch": 0.05247403287805676,
      "grad_norm": 6.262301445007324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0723343938589096,
      "logits/rejected": -0.02348300814628601,
      "logps/chosen": -373.191162109375,
      "logps/rejected": -405.5129089355469,
      "loss": 0.7548,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.8089256286621094,
      "rewards/margins": -0.052230626344680786,
      "rewards/rejected": -0.756695032119751,
      "step": 401
    },
    {
      "epoch": 0.05260489081540852,
      "grad_norm": 5.607093334197998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29206010699272156,
      "logits/rejected": 0.11949144303798676,
      "logps/chosen": -659.7827758789062,
      "logps/rejected": -527.06787109375,
      "loss": 0.6403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48394665122032166,
      "rewards/margins": 0.22433507442474365,
      "rewards/rejected": -0.7082816958427429,
      "step": 402
    },
    {
      "epoch": 0.052735748752760284,
      "grad_norm": 5.971174240112305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3153980076313019,
      "logits/rejected": 0.392687052488327,
      "logps/chosen": -429.14453125,
      "logps/rejected": -354.40203857421875,
      "loss": 0.7701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8622105121612549,
      "rewards/margins": -0.08318766951560974,
      "rewards/rejected": -0.7790228128433228,
      "step": 403
    },
    {
      "epoch": 0.05286660669011205,
      "grad_norm": 5.763871669769287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5355242490768433,
      "logits/rejected": 0.5159804224967957,
      "logps/chosen": -431.3869934082031,
      "logps/rejected": -443.681640625,
      "loss": 0.7204,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4259645640850067,
      "rewards/margins": 0.23579785227775574,
      "rewards/rejected": -0.6617624163627625,
      "step": 404
    },
    {
      "epoch": 0.05299746462746381,
      "grad_norm": 5.614152908325195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15980875492095947,
      "logits/rejected": 0.1326213777065277,
      "logps/chosen": -308.5592346191406,
      "logps/rejected": -364.29083251953125,
      "loss": 0.6205,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3413456082344055,
      "rewards/margins": 0.26905232667922974,
      "rewards/rejected": -0.6103979349136353,
      "step": 405
    },
    {
      "epoch": 0.053128322564815575,
      "grad_norm": 6.436009407043457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5161055326461792,
      "logits/rejected": 0.29261285066604614,
      "logps/chosen": -498.444580078125,
      "logps/rejected": -324.0926513671875,
      "loss": 0.7777,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7996549606323242,
      "rewards/margins": -0.05496346950531006,
      "rewards/rejected": -0.7446914315223694,
      "step": 406
    },
    {
      "epoch": 0.05325918050216733,
      "grad_norm": 4.993484973907471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21146030724048615,
      "logits/rejected": 0.17217117547988892,
      "logps/chosen": -401.2413024902344,
      "logps/rejected": -404.30157470703125,
      "loss": 0.5794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5475267171859741,
      "rewards/margins": 0.2905128598213196,
      "rewards/rejected": -0.8380396366119385,
      "step": 407
    },
    {
      "epoch": 0.053390038439519096,
      "grad_norm": 4.740962505340576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3619588017463684,
      "logits/rejected": 0.49064186215400696,
      "logps/chosen": -292.80377197265625,
      "logps/rejected": -388.0535888671875,
      "loss": 0.6628,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20442791283130646,
      "rewards/margins": 0.2340455949306488,
      "rewards/rejected": -0.43847352266311646,
      "step": 408
    },
    {
      "epoch": 0.05352089637687086,
      "grad_norm": 4.655536651611328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18612641096115112,
      "logits/rejected": 0.18665044009685516,
      "logps/chosen": -496.61505126953125,
      "logps/rejected": -619.779052734375,
      "loss": 0.596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8784471750259399,
      "rewards/margins": 0.37077829241752625,
      "rewards/rejected": -1.249225378036499,
      "step": 409
    },
    {
      "epoch": 0.05365175431422262,
      "grad_norm": 4.358259201049805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27112752199172974,
      "logits/rejected": 0.40508347749710083,
      "logps/chosen": -454.2763671875,
      "logps/rejected": -625.9933471679688,
      "loss": 0.4724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39809632301330566,
      "rewards/margins": 0.6606103777885437,
      "rewards/rejected": -1.0587067604064941,
      "step": 410
    },
    {
      "epoch": 0.05378261225157439,
      "grad_norm": 4.797057628631592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3141469359397888,
      "logits/rejected": 0.5057408809661865,
      "logps/chosen": -587.4581298828125,
      "logps/rejected": -601.342041015625,
      "loss": 0.5229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4196302592754364,
      "rewards/margins": 0.45025408267974854,
      "rewards/rejected": -0.8698843121528625,
      "step": 411
    },
    {
      "epoch": 0.053913470188926144,
      "grad_norm": 4.32167387008667,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010378178209066391,
      "logits/rejected": -0.00014625303447246552,
      "logps/chosen": -311.29840087890625,
      "logps/rejected": -435.73919677734375,
      "loss": 0.6138,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4938714802265167,
      "rewards/margins": 0.29291000962257385,
      "rewards/rejected": -0.7867815494537354,
      "step": 412
    },
    {
      "epoch": 0.05404432812627791,
      "grad_norm": 4.9122819900512695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4889191687107086,
      "logits/rejected": 0.6189199090003967,
      "logps/chosen": -357.4922790527344,
      "logps/rejected": -430.1509094238281,
      "loss": 0.5412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42547282576560974,
      "rewards/margins": 0.6022558808326721,
      "rewards/rejected": -1.0277286767959595,
      "step": 413
    },
    {
      "epoch": 0.05417518606362967,
      "grad_norm": 4.7753376960754395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13277451694011688,
      "logits/rejected": 0.2458568662405014,
      "logps/chosen": -470.77020263671875,
      "logps/rejected": -438.95343017578125,
      "loss": 0.5848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6692528128623962,
      "rewards/margins": 0.3246036469936371,
      "rewards/rejected": -0.9938564300537109,
      "step": 414
    },
    {
      "epoch": 0.054306044000981435,
      "grad_norm": 8.787352561950684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1353362500667572,
      "logits/rejected": 0.07407788932323456,
      "logps/chosen": -432.8913269042969,
      "logps/rejected": -376.928955078125,
      "loss": 0.9342,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1645634174346924,
      "rewards/margins": -0.17478150129318237,
      "rewards/rejected": -0.9897819757461548,
      "step": 415
    },
    {
      "epoch": 0.0544369019383332,
      "grad_norm": 4.530889987945557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3500402569770813,
      "logits/rejected": 0.3423832952976227,
      "logps/chosen": -473.42486572265625,
      "logps/rejected": -527.3277587890625,
      "loss": 0.5398,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14516256749629974,
      "rewards/margins": 0.3661903142929077,
      "rewards/rejected": -0.5113529562950134,
      "step": 416
    },
    {
      "epoch": 0.05456775987568496,
      "grad_norm": 3.466731309890747,
      "learning_rate": 5e-06,
      "logits/chosen": 0.559111475944519,
      "logits/rejected": 0.6898939609527588,
      "logps/chosen": -514.976318359375,
      "logps/rejected": -567.918701171875,
      "loss": 0.4174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21852703392505646,
      "rewards/margins": 0.825924277305603,
      "rewards/rejected": -1.044451355934143,
      "step": 417
    },
    {
      "epoch": 0.05469861781303672,
      "grad_norm": 5.007676124572754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7688736915588379,
      "logits/rejected": 0.838505744934082,
      "logps/chosen": -586.6442260742188,
      "logps/rejected": -533.419921875,
      "loss": 0.5382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3785751461982727,
      "rewards/margins": 0.3789365887641907,
      "rewards/rejected": -0.7575117349624634,
      "step": 418
    },
    {
      "epoch": 0.05482947575038848,
      "grad_norm": 5.043973922729492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.299996018409729,
      "logits/rejected": 0.17491284012794495,
      "logps/chosen": -536.312744140625,
      "logps/rejected": -462.08770751953125,
      "loss": 0.6355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34303703904151917,
      "rewards/margins": 0.5113903284072876,
      "rewards/rejected": -0.8544273376464844,
      "step": 419
    },
    {
      "epoch": 0.05496033368774025,
      "grad_norm": 6.438761234283447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25189855694770813,
      "logits/rejected": 0.21465790271759033,
      "logps/chosen": -435.83984375,
      "logps/rejected": -353.8233947753906,
      "loss": 0.6504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6368288397789001,
      "rewards/margins": 0.2795042395591736,
      "rewards/rejected": -0.9163330793380737,
      "step": 420
    },
    {
      "epoch": 0.05509119162509201,
      "grad_norm": 5.618251800537109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06281203776597977,
      "logits/rejected": -0.015920735895633698,
      "logps/chosen": -524.2833862304688,
      "logps/rejected": -465.61993408203125,
      "loss": 0.5344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47533437609672546,
      "rewards/margins": 0.37728309631347656,
      "rewards/rejected": -0.8526175022125244,
      "step": 421
    },
    {
      "epoch": 0.055222049562443774,
      "grad_norm": 4.7275800704956055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0222356878221035,
      "logits/rejected": 0.10800282657146454,
      "logps/chosen": -311.0155944824219,
      "logps/rejected": -353.17645263671875,
      "loss": 0.5771,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5479047894477844,
      "rewards/margins": 0.34116706252098083,
      "rewards/rejected": -0.8890717625617981,
      "step": 422
    },
    {
      "epoch": 0.05535290749979553,
      "grad_norm": 4.712391376495361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33574149012565613,
      "logits/rejected": 0.09346447885036469,
      "logps/chosen": -495.10382080078125,
      "logps/rejected": -424.1919250488281,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2675766050815582,
      "rewards/margins": 0.445791631937027,
      "rewards/rejected": -0.7133681774139404,
      "step": 423
    },
    {
      "epoch": 0.055483765437147295,
      "grad_norm": 3.965358018875122,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21193072199821472,
      "logits/rejected": 0.3352919816970825,
      "logps/chosen": -412.3184814453125,
      "logps/rejected": -303.33258056640625,
      "loss": 0.5327,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13469238579273224,
      "rewards/margins": 0.42800939083099365,
      "rewards/rejected": -0.5627018213272095,
      "step": 424
    },
    {
      "epoch": 0.05561462337449906,
      "grad_norm": 4.272294521331787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11483415961265564,
      "logits/rejected": 0.3837011158466339,
      "logps/chosen": -330.95025634765625,
      "logps/rejected": -331.4795837402344,
      "loss": 0.7256,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6728130578994751,
      "rewards/margins": 0.010585129261016846,
      "rewards/rejected": -0.6833981871604919,
      "step": 425
    },
    {
      "epoch": 0.05574548131185082,
      "grad_norm": 5.182027816772461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3977421522140503,
      "logits/rejected": 0.27327024936676025,
      "logps/chosen": -468.1171875,
      "logps/rejected": -447.00634765625,
      "loss": 0.6958,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6684328317642212,
      "rewards/margins": 0.016926009207963943,
      "rewards/rejected": -0.6853588223457336,
      "step": 426
    },
    {
      "epoch": 0.055876339249202586,
      "grad_norm": 6.033624649047852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19770921766757965,
      "logits/rejected": 0.272643506526947,
      "logps/chosen": -516.6209106445312,
      "logps/rejected": -552.79931640625,
      "loss": 0.7539,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3200126886367798,
      "rewards/margins": 0.0015113651752471924,
      "rewards/rejected": -0.32152408361434937,
      "step": 427
    },
    {
      "epoch": 0.05600719718655435,
      "grad_norm": 6.791658401489258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0855417549610138,
      "logits/rejected": 0.21243956685066223,
      "logps/chosen": -353.091552734375,
      "logps/rejected": -447.1809387207031,
      "loss": 0.7512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5268398523330688,
      "rewards/margins": 0.12382584810256958,
      "rewards/rejected": -0.6506656408309937,
      "step": 428
    },
    {
      "epoch": 0.05613805512390611,
      "grad_norm": 6.533362865447998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45778563618659973,
      "logits/rejected": 0.42273908853530884,
      "logps/chosen": -530.5917358398438,
      "logps/rejected": -440.5181579589844,
      "loss": 0.7733,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5945266485214233,
      "rewards/margins": -0.10547713935375214,
      "rewards/rejected": -0.4890495538711548,
      "step": 429
    },
    {
      "epoch": 0.05626891306125787,
      "grad_norm": 4.660378932952881,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6822637915611267,
      "logits/rejected": 0.4411051571369171,
      "logps/chosen": -380.7889709472656,
      "logps/rejected": -479.6388854980469,
      "loss": 0.4908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5502247214317322,
      "rewards/margins": 0.6911433935165405,
      "rewards/rejected": -1.241368055343628,
      "step": 430
    },
    {
      "epoch": 0.056399770998609634,
      "grad_norm": 4.633436679840088,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4719991981983185,
      "logits/rejected": 0.41947394609451294,
      "logps/chosen": -284.58380126953125,
      "logps/rejected": -387.20849609375,
      "loss": 0.5377,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5537294745445251,
      "rewards/margins": 0.5436075329780579,
      "rewards/rejected": -1.097337007522583,
      "step": 431
    },
    {
      "epoch": 0.0565306289359614,
      "grad_norm": 4.910153388977051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5000312924385071,
      "logits/rejected": 0.50224769115448,
      "logps/chosen": -603.9701538085938,
      "logps/rejected": -616.691162109375,
      "loss": 0.5029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40797439217567444,
      "rewards/margins": 0.5008542537689209,
      "rewards/rejected": -0.9088287949562073,
      "step": 432
    },
    {
      "epoch": 0.05666148687331316,
      "grad_norm": 4.448690891265869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12256281077861786,
      "logits/rejected": 0.05952281504869461,
      "logps/chosen": -451.0677795410156,
      "logps/rejected": -493.0669250488281,
      "loss": 0.5077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7721680402755737,
      "rewards/margins": 0.5748912692070007,
      "rewards/rejected": -1.3470592498779297,
      "step": 433
    },
    {
      "epoch": 0.05679234481066492,
      "grad_norm": 7.010317802429199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6968498229980469,
      "logits/rejected": 0.3897630572319031,
      "logps/chosen": -570.1236572265625,
      "logps/rejected": -390.83062744140625,
      "loss": 0.6929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5407438278198242,
      "rewards/margins": 0.16117411851882935,
      "rewards/rejected": -0.7019180059432983,
      "step": 434
    },
    {
      "epoch": 0.05692320274801668,
      "grad_norm": 5.283968925476074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3829385042190552,
      "logits/rejected": 0.3109271824359894,
      "logps/chosen": -309.55059814453125,
      "logps/rejected": -469.02313232421875,
      "loss": 0.5847,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5819095969200134,
      "rewards/margins": 0.4415434002876282,
      "rewards/rejected": -1.0234529972076416,
      "step": 435
    },
    {
      "epoch": 0.057054060685368446,
      "grad_norm": 5.096354007720947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16348117589950562,
      "logits/rejected": 0.3685970902442932,
      "logps/chosen": -403.4810791015625,
      "logps/rejected": -464.12823486328125,
      "loss": 0.5767,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43898046016693115,
      "rewards/margins": 0.49315589666366577,
      "rewards/rejected": -0.9321363568305969,
      "step": 436
    },
    {
      "epoch": 0.05718491862272021,
      "grad_norm": 3.876539707183838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6085591912269592,
      "logits/rejected": 0.4991881847381592,
      "logps/chosen": -454.13165283203125,
      "logps/rejected": -435.3784484863281,
      "loss": 0.4802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20066224038600922,
      "rewards/margins": 0.9124728441238403,
      "rewards/rejected": -1.1131352186203003,
      "step": 437
    },
    {
      "epoch": 0.057315776560071974,
      "grad_norm": 4.425087928771973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48371049761772156,
      "logits/rejected": 0.39843228459358215,
      "logps/chosen": -352.18701171875,
      "logps/rejected": -324.0437927246094,
      "loss": 0.6197,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6779739260673523,
      "rewards/margins": 0.1793006956577301,
      "rewards/rejected": -0.8572746515274048,
      "step": 438
    },
    {
      "epoch": 0.05744663449742374,
      "grad_norm": 3.467874050140381,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32407426834106445,
      "logits/rejected": 0.3471176028251648,
      "logps/chosen": -510.87884521484375,
      "logps/rejected": -521.2025146484375,
      "loss": 0.3657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30020925402641296,
      "rewards/margins": 1.1410155296325684,
      "rewards/rejected": -1.4412246942520142,
      "step": 439
    },
    {
      "epoch": 0.057577492434775494,
      "grad_norm": 4.494588851928711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18636222183704376,
      "logits/rejected": 0.07386263459920883,
      "logps/chosen": -604.80126953125,
      "logps/rejected": -454.7237548828125,
      "loss": 0.5955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43297919631004333,
      "rewards/margins": 0.26909390091896057,
      "rewards/rejected": -0.7020730972290039,
      "step": 440
    },
    {
      "epoch": 0.05770835037212726,
      "grad_norm": 4.949033737182617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49837231636047363,
      "logits/rejected": 0.5299926996231079,
      "logps/chosen": -544.2364501953125,
      "logps/rejected": -414.1546936035156,
      "loss": 0.6069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7614791989326477,
      "rewards/margins": 0.42009997367858887,
      "rewards/rejected": -1.181579351425171,
      "step": 441
    },
    {
      "epoch": 0.05783920830947902,
      "grad_norm": 3.9963161945343018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39576929807662964,
      "logits/rejected": 0.5404410362243652,
      "logps/chosen": -428.2747802734375,
      "logps/rejected": -370.63323974609375,
      "loss": 0.5056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40746593475341797,
      "rewards/margins": 0.6479582190513611,
      "rewards/rejected": -1.0554240942001343,
      "step": 442
    },
    {
      "epoch": 0.057970066246830786,
      "grad_norm": 4.498307228088379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22577707469463348,
      "logits/rejected": 0.15085700154304504,
      "logps/chosen": -493.85882568359375,
      "logps/rejected": -635.8921508789062,
      "loss": 0.4933,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6526603698730469,
      "rewards/margins": 0.5750545859336853,
      "rewards/rejected": -1.2277148962020874,
      "step": 443
    },
    {
      "epoch": 0.05810092418418255,
      "grad_norm": 5.4460296630859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22516565024852753,
      "logits/rejected": 0.3090493381023407,
      "logps/chosen": -530.7571411132812,
      "logps/rejected": -390.09771728515625,
      "loss": 0.611,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1454010009765625,
      "rewards/margins": 0.4311577081680298,
      "rewards/rejected": -0.5765587091445923,
      "step": 444
    },
    {
      "epoch": 0.058231782121534306,
      "grad_norm": 4.624357223510742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30973851680755615,
      "logits/rejected": 0.4169328808784485,
      "logps/chosen": -320.4376220703125,
      "logps/rejected": -280.2177429199219,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7013155221939087,
      "rewards/margins": 0.13030433654785156,
      "rewards/rejected": -0.8316198587417603,
      "step": 445
    },
    {
      "epoch": 0.05836264005888607,
      "grad_norm": 3.591616153717041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1498579978942871,
      "logits/rejected": 0.057139135897159576,
      "logps/chosen": -268.35748291015625,
      "logps/rejected": -560.8555297851562,
      "loss": 0.5025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3822975158691406,
      "rewards/margins": 0.6890121698379517,
      "rewards/rejected": -1.0713096857070923,
      "step": 446
    },
    {
      "epoch": 0.058493497996237834,
      "grad_norm": 4.863090515136719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.547264814376831,
      "logits/rejected": 0.5190572738647461,
      "logps/chosen": -371.09930419921875,
      "logps/rejected": -278.1636047363281,
      "loss": 0.5366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7376329898834229,
      "rewards/margins": 0.5143299102783203,
      "rewards/rejected": -1.2519629001617432,
      "step": 447
    },
    {
      "epoch": 0.0586243559335896,
      "grad_norm": 5.178096294403076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20436687767505646,
      "logits/rejected": 0.1261453926563263,
      "logps/chosen": -661.370361328125,
      "logps/rejected": -448.060302734375,
      "loss": 0.6362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05178222060203552,
      "rewards/margins": 0.2885570526123047,
      "rewards/rejected": -0.3403392434120178,
      "step": 448
    },
    {
      "epoch": 0.05875521387094136,
      "grad_norm": 6.923094749450684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35210004448890686,
      "logits/rejected": 0.3450281322002411,
      "logps/chosen": -536.8370361328125,
      "logps/rejected": -380.17828369140625,
      "loss": 0.6478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4789566397666931,
      "rewards/margins": 0.6475642919540405,
      "rewards/rejected": -1.1265208721160889,
      "step": 449
    },
    {
      "epoch": 0.058886071808293125,
      "grad_norm": 6.463877201080322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16950057446956635,
      "logits/rejected": -0.03609827905893326,
      "logps/chosen": -528.856689453125,
      "logps/rejected": -434.5424499511719,
      "loss": 0.7813,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.37967896461486816,
      "rewards/margins": 0.0574025958776474,
      "rewards/rejected": -0.43708154559135437,
      "step": 450
    },
    {
      "epoch": 0.05901692974564488,
      "grad_norm": 6.467247009277344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4172683656215668,
      "logits/rejected": 0.3605814576148987,
      "logps/chosen": -469.73638916015625,
      "logps/rejected": -443.6038513183594,
      "loss": 0.7362,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.960321843624115,
      "rewards/margins": -0.03824082016944885,
      "rewards/rejected": -0.9220809936523438,
      "step": 451
    },
    {
      "epoch": 0.059147787682996646,
      "grad_norm": 5.796024799346924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3374707102775574,
      "logits/rejected": 0.22170265018939972,
      "logps/chosen": -472.92498779296875,
      "logps/rejected": -409.31842041015625,
      "loss": 0.6362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6275498867034912,
      "rewards/margins": 0.20101889967918396,
      "rewards/rejected": -0.8285688161849976,
      "step": 452
    },
    {
      "epoch": 0.05927864562034841,
      "grad_norm": 4.628364562988281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3192419409751892,
      "logits/rejected": 0.26117709279060364,
      "logps/chosen": -432.73291015625,
      "logps/rejected": -494.53753662109375,
      "loss": 0.5954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5540552139282227,
      "rewards/margins": 0.4301210045814514,
      "rewards/rejected": -0.9841762781143188,
      "step": 453
    },
    {
      "epoch": 0.05940950355770017,
      "grad_norm": 5.199400901794434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3820810317993164,
      "logits/rejected": 0.30050450563430786,
      "logps/chosen": -556.9623413085938,
      "logps/rejected": -408.31805419921875,
      "loss": 0.652,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5979270935058594,
      "rewards/margins": 0.22527295351028442,
      "rewards/rejected": -0.8231999278068542,
      "step": 454
    },
    {
      "epoch": 0.05954036149505194,
      "grad_norm": 4.363189697265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.864765465259552,
      "logits/rejected": 0.6522434949874878,
      "logps/chosen": -485.8663330078125,
      "logps/rejected": -365.7467041015625,
      "loss": 0.4929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1629621684551239,
      "rewards/margins": 0.5085233449935913,
      "rewards/rejected": -0.6714855432510376,
      "step": 455
    },
    {
      "epoch": 0.059671219432403694,
      "grad_norm": 4.544659614562988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27911603450775146,
      "logits/rejected": -0.5633991956710815,
      "logps/chosen": -621.5517578125,
      "logps/rejected": -771.674072265625,
      "loss": 0.4496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05708085000514984,
      "rewards/margins": 0.6576162576675415,
      "rewards/rejected": -0.7146970629692078,
      "step": 456
    },
    {
      "epoch": 0.05980207736975546,
      "grad_norm": 5.375748157501221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4358479678630829,
      "logits/rejected": 0.44717055559158325,
      "logps/chosen": -544.7939453125,
      "logps/rejected": -642.2274780273438,
      "loss": 0.548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4170875549316406,
      "rewards/margins": 0.5292768478393555,
      "rewards/rejected": -0.9463644623756409,
      "step": 457
    },
    {
      "epoch": 0.05993293530710722,
      "grad_norm": 5.159587383270264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32107654213905334,
      "logits/rejected": 0.21045050024986267,
      "logps/chosen": -553.2581787109375,
      "logps/rejected": -422.33013916015625,
      "loss": 0.6527,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.311056911945343,
      "rewards/margins": 0.1981658935546875,
      "rewards/rejected": -0.5092227458953857,
      "step": 458
    },
    {
      "epoch": 0.060063793244458985,
      "grad_norm": 3.22552752494812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7092405557632446,
      "logits/rejected": 0.7450686693191528,
      "logps/chosen": -526.48681640625,
      "logps/rejected": -460.3039245605469,
      "loss": 0.3443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7462587356567383,
      "rewards/margins": 1.0918352603912354,
      "rewards/rejected": -1.8380941152572632,
      "step": 459
    },
    {
      "epoch": 0.06019465118181075,
      "grad_norm": 4.415960311889648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15615126490592957,
      "logits/rejected": 0.0876069962978363,
      "logps/chosen": -437.817626953125,
      "logps/rejected": -385.0320739746094,
      "loss": 0.6001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37423551082611084,
      "rewards/margins": 0.22812330722808838,
      "rewards/rejected": -0.6023588180541992,
      "step": 460
    },
    {
      "epoch": 0.06032550911916251,
      "grad_norm": 6.473568439483643,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29467150568962097,
      "logits/rejected": 0.404013454914093,
      "logps/chosen": -498.0099792480469,
      "logps/rejected": -457.3915100097656,
      "loss": 0.758,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6664029955863953,
      "rewards/margins": -0.0005098432302474976,
      "rewards/rejected": -0.6658931970596313,
      "step": 461
    },
    {
      "epoch": 0.06045636705651427,
      "grad_norm": 4.928778648376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2771000564098358,
      "logits/rejected": 0.2062758505344391,
      "logps/chosen": -321.1441345214844,
      "logps/rejected": -549.69873046875,
      "loss": 0.5378,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6303939819335938,
      "rewards/margins": 0.4707058072090149,
      "rewards/rejected": -1.1010998487472534,
      "step": 462
    },
    {
      "epoch": 0.06058722499386603,
      "grad_norm": 3.9659533500671387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5022592544555664,
      "logits/rejected": 0.5619113445281982,
      "logps/chosen": -334.2074279785156,
      "logps/rejected": -421.9951477050781,
      "loss": 0.5853,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3300548493862152,
      "rewards/margins": 0.49514392018318176,
      "rewards/rejected": -0.825198769569397,
      "step": 463
    },
    {
      "epoch": 0.0607180829312178,
      "grad_norm": 5.619907855987549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29221564531326294,
      "logits/rejected": 0.35984426736831665,
      "logps/chosen": -369.420654296875,
      "logps/rejected": -356.82904052734375,
      "loss": 0.6088,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8382096290588379,
      "rewards/margins": 0.515814483165741,
      "rewards/rejected": -1.3540241718292236,
      "step": 464
    },
    {
      "epoch": 0.06084894086856956,
      "grad_norm": 7.371360778808594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3392415940761566,
      "logits/rejected": 0.17544545233249664,
      "logps/chosen": -376.62109375,
      "logps/rejected": -450.63641357421875,
      "loss": 0.8484,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8294481039047241,
      "rewards/margins": -0.14888912439346313,
      "rewards/rejected": -0.6805590391159058,
      "step": 465
    },
    {
      "epoch": 0.060979798805921324,
      "grad_norm": 5.0566301345825195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5061843991279602,
      "logits/rejected": 0.39620351791381836,
      "logps/chosen": -672.2318725585938,
      "logps/rejected": -811.8057250976562,
      "loss": 0.4623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10574609041213989,
      "rewards/margins": 0.7990738749504089,
      "rewards/rejected": -0.9048199653625488,
      "step": 466
    },
    {
      "epoch": 0.06111065674327308,
      "grad_norm": 3.9272549152374268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3051409125328064,
      "logits/rejected": 0.20505546033382416,
      "logps/chosen": -378.9649963378906,
      "logps/rejected": -413.69488525390625,
      "loss": 0.5038,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5507898330688477,
      "rewards/margins": 0.5993170142173767,
      "rewards/rejected": -1.1501067876815796,
      "step": 467
    },
    {
      "epoch": 0.061241514680624845,
      "grad_norm": 6.624296188354492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1275874674320221,
      "logits/rejected": 0.23832282423973083,
      "logps/chosen": -385.0984802246094,
      "logps/rejected": -443.3272399902344,
      "loss": 0.8874,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7110063433647156,
      "rewards/margins": 0.005857884883880615,
      "rewards/rejected": -0.7168641686439514,
      "step": 468
    },
    {
      "epoch": 0.06137237261797661,
      "grad_norm": 5.389120578765869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5105584859848022,
      "logits/rejected": 0.5742197036743164,
      "logps/chosen": -494.4242858886719,
      "logps/rejected": -432.22930908203125,
      "loss": 0.5857,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7418290376663208,
      "rewards/margins": 0.5760170817375183,
      "rewards/rejected": -1.3178460597991943,
      "step": 469
    },
    {
      "epoch": 0.06150323055532837,
      "grad_norm": 7.3665900230407715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4189265966415405,
      "logits/rejected": 0.6101194024085999,
      "logps/chosen": -501.7556457519531,
      "logps/rejected": -408.375732421875,
      "loss": 0.7916,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9930009841918945,
      "rewards/margins": -0.06492440402507782,
      "rewards/rejected": -0.9280766248703003,
      "step": 470
    },
    {
      "epoch": 0.061634088492680136,
      "grad_norm": 6.443260669708252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2047952264547348,
      "logits/rejected": 0.3408237099647522,
      "logps/chosen": -387.7115173339844,
      "logps/rejected": -388.58642578125,
      "loss": 0.741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8022707104682922,
      "rewards/margins": 0.012657910585403442,
      "rewards/rejected": -0.8149285912513733,
      "step": 471
    },
    {
      "epoch": 0.06176494643003189,
      "grad_norm": 6.036807537078857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6713466048240662,
      "logits/rejected": 0.4756684899330139,
      "logps/chosen": -600.9600830078125,
      "logps/rejected": -556.8439331054688,
      "loss": 0.7557,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2849002778530121,
      "rewards/margins": 0.12386321276426315,
      "rewards/rejected": -0.4087635278701782,
      "step": 472
    },
    {
      "epoch": 0.06189580436738366,
      "grad_norm": 4.17779016494751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.457653671503067,
      "logits/rejected": 0.544472336769104,
      "logps/chosen": -465.710693359375,
      "logps/rejected": -453.8117370605469,
      "loss": 0.3673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6215923428535461,
      "rewards/margins": 1.0718680620193481,
      "rewards/rejected": -1.69346022605896,
      "step": 473
    },
    {
      "epoch": 0.06202666230473542,
      "grad_norm": 5.582540512084961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45443591475486755,
      "logits/rejected": 0.3159729242324829,
      "logps/chosen": -476.013427734375,
      "logps/rejected": -355.8685302734375,
      "loss": 0.5888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4631842076778412,
      "rewards/margins": 0.42469847202301025,
      "rewards/rejected": -0.887882649898529,
      "step": 474
    },
    {
      "epoch": 0.062157520242087184,
      "grad_norm": 4.487755298614502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4256126880645752,
      "logits/rejected": 0.23560255765914917,
      "logps/chosen": -418.79205322265625,
      "logps/rejected": -417.3794250488281,
      "loss": 0.5354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43891602754592896,
      "rewards/margins": 0.39329952001571655,
      "rewards/rejected": -0.832215428352356,
      "step": 475
    },
    {
      "epoch": 0.06228837817943895,
      "grad_norm": 3.860386371612549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.520655632019043,
      "logits/rejected": 0.6839446425437927,
      "logps/chosen": -330.85711669921875,
      "logps/rejected": -443.52996826171875,
      "loss": 0.477,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47094279527664185,
      "rewards/margins": 0.8249423503875732,
      "rewards/rejected": -1.2958852052688599,
      "step": 476
    },
    {
      "epoch": 0.06241923611679071,
      "grad_norm": 3.928570032119751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21586227416992188,
      "logits/rejected": 0.18653099238872528,
      "logps/chosen": -298.1973876953125,
      "logps/rejected": -366.412109375,
      "loss": 0.5409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5286200046539307,
      "rewards/margins": 0.46803683042526245,
      "rewards/rejected": -0.9966568350791931,
      "step": 477
    },
    {
      "epoch": 0.06255009405414247,
      "grad_norm": 5.601781368255615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07237295806407928,
      "logits/rejected": 0.25853756070137024,
      "logps/chosen": -287.2655334472656,
      "logps/rejected": -350.8507995605469,
      "loss": 0.6409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6416211724281311,
      "rewards/margins": 0.23997651040554047,
      "rewards/rejected": -0.881597638130188,
      "step": 478
    },
    {
      "epoch": 0.06268095199149423,
      "grad_norm": 5.728729724884033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46641331911087036,
      "logits/rejected": 0.36231207847595215,
      "logps/chosen": -407.72845458984375,
      "logps/rejected": -465.91033935546875,
      "loss": 0.482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44832172989845276,
      "rewards/margins": 0.638398289680481,
      "rewards/rejected": -1.0867201089859009,
      "step": 479
    },
    {
      "epoch": 0.062811809928846,
      "grad_norm": 5.185812950134277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17145510017871857,
      "logits/rejected": 0.3331228494644165,
      "logps/chosen": -509.7002258300781,
      "logps/rejected": -445.5340576171875,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6872223019599915,
      "rewards/margins": 0.5577545166015625,
      "rewards/rejected": -1.2449767589569092,
      "step": 480
    },
    {
      "epoch": 0.06294266786619776,
      "grad_norm": 4.98557710647583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14104682207107544,
      "logits/rejected": 0.17279276251792908,
      "logps/chosen": -340.18597412109375,
      "logps/rejected": -408.5054931640625,
      "loss": 0.6759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.909493088722229,
      "rewards/margins": 0.07884234189987183,
      "rewards/rejected": -0.988335371017456,
      "step": 481
    },
    {
      "epoch": 0.06307352580354952,
      "grad_norm": 4.262895107269287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4468449652194977,
      "logits/rejected": 0.4642684757709503,
      "logps/chosen": -438.61651611328125,
      "logps/rejected": -520.1322021484375,
      "loss": 0.3964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5009598135948181,
      "rewards/margins": 0.8251272439956665,
      "rewards/rejected": -1.3260871171951294,
      "step": 482
    },
    {
      "epoch": 0.06320438374090129,
      "grad_norm": 5.074166297912598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1262533962726593,
      "logits/rejected": -0.02218322455883026,
      "logps/chosen": -670.086669921875,
      "logps/rejected": -512.5274658203125,
      "loss": 0.5411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43843021988868713,
      "rewards/margins": 0.4471500515937805,
      "rewards/rejected": -0.8855802416801453,
      "step": 483
    },
    {
      "epoch": 0.06333524167825305,
      "grad_norm": 5.145401477813721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2762184143066406,
      "logits/rejected": 0.3656116724014282,
      "logps/chosen": -302.2057189941406,
      "logps/rejected": -369.5859069824219,
      "loss": 0.6402,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5297874808311462,
      "rewards/margins": 0.26632288098335266,
      "rewards/rejected": -0.7961103916168213,
      "step": 484
    },
    {
      "epoch": 0.06346609961560482,
      "grad_norm": 3.8883144855499268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12239926308393478,
      "logits/rejected": 0.3333231806755066,
      "logps/chosen": -508.135498046875,
      "logps/rejected": -421.6748046875,
      "loss": 0.4503,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40460169315338135,
      "rewards/margins": 0.7339563369750977,
      "rewards/rejected": -1.138558030128479,
      "step": 485
    },
    {
      "epoch": 0.06359695755295657,
      "grad_norm": 3.4782607555389404,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21710096299648285,
      "logits/rejected": 0.09286434203386307,
      "logps/chosen": -359.9994812011719,
      "logps/rejected": -316.3490295410156,
      "loss": 0.4758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5720140337944031,
      "rewards/margins": 0.7226948142051697,
      "rewards/rejected": -1.2947088479995728,
      "step": 486
    },
    {
      "epoch": 0.06372781549030833,
      "grad_norm": 6.1529388427734375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07661262154579163,
      "logits/rejected": 0.34954261779785156,
      "logps/chosen": -533.4407958984375,
      "logps/rejected": -469.18438720703125,
      "loss": 0.5183,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3172285258769989,
      "rewards/margins": 0.6065200567245483,
      "rewards/rejected": -0.9237486720085144,
      "step": 487
    },
    {
      "epoch": 0.06385867342766009,
      "grad_norm": 4.598169803619385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41958582401275635,
      "logits/rejected": 0.337886780500412,
      "logps/chosen": -384.5510559082031,
      "logps/rejected": -437.0316162109375,
      "loss": 0.5899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6240663528442383,
      "rewards/margins": 0.44510018825531006,
      "rewards/rejected": -1.0691665410995483,
      "step": 488
    },
    {
      "epoch": 0.06398953136501186,
      "grad_norm": 4.938222408294678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1683783084154129,
      "logits/rejected": 0.35077664256095886,
      "logps/chosen": -413.7267150878906,
      "logps/rejected": -408.2244567871094,
      "loss": 0.5643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3156667649745941,
      "rewards/margins": 0.6792837381362915,
      "rewards/rejected": -0.994950532913208,
      "step": 489
    },
    {
      "epoch": 0.06412038930236362,
      "grad_norm": 9.855839729309082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22387196123600006,
      "logits/rejected": -0.029449373483657837,
      "logps/chosen": -516.2759399414062,
      "logps/rejected": -438.107421875,
      "loss": 1.0179,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9933485388755798,
      "rewards/margins": -0.07198679447174072,
      "rewards/rejected": -0.9213616847991943,
      "step": 490
    },
    {
      "epoch": 0.06425124723971538,
      "grad_norm": 5.197096824645996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23132863640785217,
      "logits/rejected": 0.2954958975315094,
      "logps/chosen": -671.4124755859375,
      "logps/rejected": -642.1815185546875,
      "loss": 0.4229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1896630972623825,
      "rewards/margins": 0.7188913226127625,
      "rewards/rejected": -0.9085544943809509,
      "step": 491
    },
    {
      "epoch": 0.06438210517706715,
      "grad_norm": 6.9346923828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2097262740135193,
      "logits/rejected": 0.13759014010429382,
      "logps/chosen": -586.8693237304688,
      "logps/rejected": -447.086669921875,
      "loss": 0.6772,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7690728902816772,
      "rewards/margins": 0.24175386130809784,
      "rewards/rejected": -1.010826826095581,
      "step": 492
    },
    {
      "epoch": 0.06451296311441891,
      "grad_norm": 4.675838470458984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1150389313697815,
      "logits/rejected": 0.12448623776435852,
      "logps/chosen": -311.8426513671875,
      "logps/rejected": -389.0731201171875,
      "loss": 0.485,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7104243040084839,
      "rewards/margins": 0.6331127285957336,
      "rewards/rejected": -1.3435370922088623,
      "step": 493
    },
    {
      "epoch": 0.06464382105177068,
      "grad_norm": 4.3598222732543945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40150368213653564,
      "logits/rejected": 0.37850648164749146,
      "logps/chosen": -456.921142578125,
      "logps/rejected": -381.134033203125,
      "loss": 0.5236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21604996919631958,
      "rewards/margins": 0.49595606327056885,
      "rewards/rejected": -0.7120059728622437,
      "step": 494
    },
    {
      "epoch": 0.06477467898912244,
      "grad_norm": 5.0489959716796875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.645038902759552,
      "logits/rejected": 0.3767755627632141,
      "logps/chosen": -379.85858154296875,
      "logps/rejected": -387.5264587402344,
      "loss": 0.6121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6595259308815002,
      "rewards/margins": 0.18649424612522125,
      "rewards/rejected": -0.8460201621055603,
      "step": 495
    },
    {
      "epoch": 0.0649055369264742,
      "grad_norm": 5.123305797576904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2973235845565796,
      "logits/rejected": 0.3505513668060303,
      "logps/chosen": -364.66839599609375,
      "logps/rejected": -363.9310607910156,
      "loss": 0.5241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7557939291000366,
      "rewards/margins": 0.6030572652816772,
      "rewards/rejected": -1.3588511943817139,
      "step": 496
    },
    {
      "epoch": 0.06503639486382595,
      "grad_norm": 9.006680488586426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4433591961860657,
      "logits/rejected": 0.4471875727176666,
      "logps/chosen": -322.9666748046875,
      "logps/rejected": -481.85009765625,
      "loss": 0.5266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7841860055923462,
      "rewards/margins": 0.6707889437675476,
      "rewards/rejected": -1.4549750089645386,
      "step": 497
    },
    {
      "epoch": 0.06516725280117772,
      "grad_norm": 3.7351481914520264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20314665138721466,
      "logits/rejected": -0.003618907183408737,
      "logps/chosen": -484.8147277832031,
      "logps/rejected": -435.0269775390625,
      "loss": 0.414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35931190848350525,
      "rewards/margins": 0.8692611455917358,
      "rewards/rejected": -0.5099493265151978,
      "step": 498
    },
    {
      "epoch": 0.06529811073852948,
      "grad_norm": 4.425527095794678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22391614317893982,
      "logits/rejected": 0.09404464066028595,
      "logps/chosen": -501.106689453125,
      "logps/rejected": -490.41632080078125,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5271738171577454,
      "rewards/margins": 0.8024820685386658,
      "rewards/rejected": -1.3296557664871216,
      "step": 499
    },
    {
      "epoch": 0.06542896867588124,
      "grad_norm": 4.592318534851074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23478415608406067,
      "logits/rejected": 0.2461715042591095,
      "logps/chosen": -477.075439453125,
      "logps/rejected": -570.1871337890625,
      "loss": 0.4947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8587995767593384,
      "rewards/margins": 0.8810587525367737,
      "rewards/rejected": -1.7398582696914673,
      "step": 500
    },
    {
      "epoch": 0.06555982661323301,
      "grad_norm": 5.381325721740723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09738492965698242,
      "logits/rejected": 0.12533092498779297,
      "logps/chosen": -472.84100341796875,
      "logps/rejected": -333.0188903808594,
      "loss": 0.5648,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7485946416854858,
      "rewards/margins": 0.4743099808692932,
      "rewards/rejected": -1.2229046821594238,
      "step": 501
    },
    {
      "epoch": 0.06569068455058477,
      "grad_norm": 7.819185733795166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35075703263282776,
      "logits/rejected": 0.033383458852767944,
      "logps/chosen": -545.942626953125,
      "logps/rejected": -358.7849426269531,
      "loss": 0.7167,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9398473501205444,
      "rewards/margins": 0.33769363164901733,
      "rewards/rejected": -1.277540922164917,
      "step": 502
    },
    {
      "epoch": 0.06582154248793654,
      "grad_norm": 6.392020225524902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.367838054895401,
      "logits/rejected": 0.24223610758781433,
      "logps/chosen": -299.19537353515625,
      "logps/rejected": -357.01910400390625,
      "loss": 0.7837,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3029406070709229,
      "rewards/margins": 0.006224453449249268,
      "rewards/rejected": -1.3091650009155273,
      "step": 503
    },
    {
      "epoch": 0.0659524004252883,
      "grad_norm": 3.6052377223968506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3759520649909973,
      "logits/rejected": 0.3350003957748413,
      "logps/chosen": -755.2445678710938,
      "logps/rejected": -537.104248046875,
      "loss": 0.3495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.39363613724708557,
      "rewards/margins": 1.3335996866226196,
      "rewards/rejected": -0.9399635791778564,
      "step": 504
    },
    {
      "epoch": 0.06608325836264006,
      "grad_norm": 4.760889053344727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4667196571826935,
      "logits/rejected": 0.577705979347229,
      "logps/chosen": -508.9747009277344,
      "logps/rejected": -472.1490478515625,
      "loss": 0.4675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.106533408164978,
      "rewards/margins": 0.700360119342804,
      "rewards/rejected": -1.8068937063217163,
      "step": 505
    },
    {
      "epoch": 0.06621411629999183,
      "grad_norm": 5.754538536071777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4814890921115875,
      "logits/rejected": 0.45453765988349915,
      "logps/chosen": -466.85284423828125,
      "logps/rejected": -442.8245849609375,
      "loss": 0.7028,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5088443756103516,
      "rewards/margins": 0.021827317774295807,
      "rewards/rejected": -0.5306717157363892,
      "step": 506
    },
    {
      "epoch": 0.06634497423734359,
      "grad_norm": 6.517983913421631,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34930238127708435,
      "logits/rejected": 0.40119606256484985,
      "logps/chosen": -681.5460815429688,
      "logps/rejected": -791.8630981445312,
      "loss": 0.6678,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.24442140758037567,
      "rewards/margins": 0.20553509891033173,
      "rewards/rejected": -0.4499565362930298,
      "step": 507
    },
    {
      "epoch": 0.06647583217469534,
      "grad_norm": 5.052639484405518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2526627779006958,
      "logits/rejected": 0.5357983708381653,
      "logps/chosen": -497.04864501953125,
      "logps/rejected": -469.3175048828125,
      "loss": 0.6266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22795850038528442,
      "rewards/margins": 0.29971447587013245,
      "rewards/rejected": -0.5276728868484497,
      "step": 508
    },
    {
      "epoch": 0.0666066901120471,
      "grad_norm": 4.019892692565918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40243056416511536,
      "logits/rejected": 0.39483949542045593,
      "logps/chosen": -378.15130615234375,
      "logps/rejected": -296.363037109375,
      "loss": 0.5612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8377168774604797,
      "rewards/margins": 0.37486588954925537,
      "rewards/rejected": -1.2125828266143799,
      "step": 509
    },
    {
      "epoch": 0.06673754804939887,
      "grad_norm": 4.422303676605225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024794677272439003,
      "logits/rejected": 0.045499518513679504,
      "logps/chosen": -411.3489990234375,
      "logps/rejected": -409.951171875,
      "loss": 0.565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4282650053501129,
      "rewards/margins": 0.3239990174770355,
      "rewards/rejected": -0.7522640228271484,
      "step": 510
    },
    {
      "epoch": 0.06686840598675063,
      "grad_norm": 5.255345344543457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23750782012939453,
      "logits/rejected": 0.16359946131706238,
      "logps/chosen": -343.6884765625,
      "logps/rejected": -338.15673828125,
      "loss": 0.6567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8836188912391663,
      "rewards/margins": 0.12320299446582794,
      "rewards/rejected": -1.006821870803833,
      "step": 511
    },
    {
      "epoch": 0.0669992639241024,
      "grad_norm": 10.921525955200195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.62391197681427,
      "logits/rejected": 0.6028766632080078,
      "logps/chosen": -534.55712890625,
      "logps/rejected": -518.9895629882812,
      "loss": 0.985,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.45633065700531,
      "rewards/margins": -0.3266100287437439,
      "rewards/rejected": -1.129720687866211,
      "step": 512
    },
    {
      "epoch": 0.06713012186145416,
      "grad_norm": 3.7542598247528076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27405187487602234,
      "logits/rejected": 0.3229363262653351,
      "logps/chosen": -473.4634704589844,
      "logps/rejected": -292.8964538574219,
      "loss": 0.4797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5952989459037781,
      "rewards/margins": 0.6515983939170837,
      "rewards/rejected": -1.2468972206115723,
      "step": 513
    },
    {
      "epoch": 0.06726097979880592,
      "grad_norm": 4.07521915435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4619543254375458,
      "logits/rejected": 0.488553911447525,
      "logps/chosen": -346.2802429199219,
      "logps/rejected": -432.6019592285156,
      "loss": 0.3797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5195028781890869,
      "rewards/margins": 0.9195532202720642,
      "rewards/rejected": -1.4390560388565063,
      "step": 514
    },
    {
      "epoch": 0.06739183773615769,
      "grad_norm": 4.076389312744141,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42893916368484497,
      "logits/rejected": 0.4293685555458069,
      "logps/chosen": -615.6761474609375,
      "logps/rejected": -484.32318115234375,
      "loss": 0.4464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0581611692905426,
      "rewards/margins": 0.7160491943359375,
      "rewards/rejected": -0.7742103338241577,
      "step": 515
    },
    {
      "epoch": 0.06752269567350945,
      "grad_norm": 4.742478370666504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26823270320892334,
      "logits/rejected": 0.013831518590450287,
      "logps/chosen": -355.93927001953125,
      "logps/rejected": -375.6499328613281,
      "loss": 0.5705,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6784014105796814,
      "rewards/margins": 0.3541356921195984,
      "rewards/rejected": -1.0325369834899902,
      "step": 516
    },
    {
      "epoch": 0.06765355361086121,
      "grad_norm": 6.134192943572998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2592359185218811,
      "logits/rejected": 0.41670364141464233,
      "logps/chosen": -536.7958984375,
      "logps/rejected": -495.7362365722656,
      "loss": 0.5779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5118123888969421,
      "rewards/margins": 0.5923646092414856,
      "rewards/rejected": -1.1041769981384277,
      "step": 517
    },
    {
      "epoch": 0.06778441154821298,
      "grad_norm": 8.488051414489746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5073630213737488,
      "logits/rejected": 0.2890082597732544,
      "logps/chosen": -698.7438354492188,
      "logps/rejected": -453.303466796875,
      "loss": 0.9194,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2113453149795532,
      "rewards/margins": -0.05402296781539917,
      "rewards/rejected": -1.1573224067687988,
      "step": 518
    },
    {
      "epoch": 0.06791526948556473,
      "grad_norm": 6.129323959350586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35495615005493164,
      "logits/rejected": 0.22283409535884857,
      "logps/chosen": -522.9888916015625,
      "logps/rejected": -503.564208984375,
      "loss": 0.66,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.786168098449707,
      "rewards/margins": 0.2575725317001343,
      "rewards/rejected": -1.0437406301498413,
      "step": 519
    },
    {
      "epoch": 0.06804612742291649,
      "grad_norm": 8.18643856048584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3477596342563629,
      "logits/rejected": 0.2987535893917084,
      "logps/chosen": -648.4492797851562,
      "logps/rejected": -505.14776611328125,
      "loss": 0.6493,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7810691595077515,
      "rewards/margins": 0.5192638635635376,
      "rewards/rejected": -1.3003331422805786,
      "step": 520
    },
    {
      "epoch": 0.06817698536026826,
      "grad_norm": 6.91464900970459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4945608973503113,
      "logits/rejected": 0.44301795959472656,
      "logps/chosen": -572.7655639648438,
      "logps/rejected": -383.81427001953125,
      "loss": 0.7386,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5576450824737549,
      "rewards/margins": 0.31957459449768066,
      "rewards/rejected": -0.8772196173667908,
      "step": 521
    },
    {
      "epoch": 0.06830784329762002,
      "grad_norm": 7.13662052154541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.635473370552063,
      "logits/rejected": 0.579005777835846,
      "logps/chosen": -426.03814697265625,
      "logps/rejected": -411.415771484375,
      "loss": 0.7228,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.341781735420227,
      "rewards/margins": 0.13456635177135468,
      "rewards/rejected": -1.4763479232788086,
      "step": 522
    },
    {
      "epoch": 0.06843870123497178,
      "grad_norm": 8.331840515136719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6219268441200256,
      "logits/rejected": 0.4642566442489624,
      "logps/chosen": -492.01507568359375,
      "logps/rejected": -476.18011474609375,
      "loss": 0.9011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1289616823196411,
      "rewards/margins": -0.16432476043701172,
      "rewards/rejected": -0.9646368026733398,
      "step": 523
    },
    {
      "epoch": 0.06856955917232355,
      "grad_norm": 4.332093715667725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41541263461112976,
      "logits/rejected": 0.2949833571910858,
      "logps/chosen": -331.54693603515625,
      "logps/rejected": -326.97088623046875,
      "loss": 0.5562,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9598820805549622,
      "rewards/margins": 0.4181840121746063,
      "rewards/rejected": -1.3780661821365356,
      "step": 524
    },
    {
      "epoch": 0.06870041710967531,
      "grad_norm": 4.070459842681885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3676692545413971,
      "logits/rejected": 0.262445867061615,
      "logps/chosen": -354.76080322265625,
      "logps/rejected": -377.67950439453125,
      "loss": 0.5325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0858360230922699,
      "rewards/margins": 0.5450953841209412,
      "rewards/rejected": -0.6309314966201782,
      "step": 525
    },
    {
      "epoch": 0.06883127504702707,
      "grad_norm": 6.545503616333008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41069769859313965,
      "logits/rejected": 0.4242354929447174,
      "logps/chosen": -324.3160095214844,
      "logps/rejected": -331.19305419921875,
      "loss": 0.6275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8904852271080017,
      "rewards/margins": 0.48697417974472046,
      "rewards/rejected": -1.3774592876434326,
      "step": 526
    },
    {
      "epoch": 0.06896213298437884,
      "grad_norm": 3.3001530170440674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36866647005081177,
      "logits/rejected": 0.25404655933380127,
      "logps/chosen": -370.91131591796875,
      "logps/rejected": -360.387939453125,
      "loss": 0.4316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33586886525154114,
      "rewards/margins": 0.780133843421936,
      "rewards/rejected": -1.1160025596618652,
      "step": 527
    },
    {
      "epoch": 0.0690929909217306,
      "grad_norm": 4.0473456382751465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39704954624176025,
      "logits/rejected": 0.44530194997787476,
      "logps/chosen": -355.3213195800781,
      "logps/rejected": -358.1923828125,
      "loss": 0.5084,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38380834460258484,
      "rewards/margins": 0.573140561580658,
      "rewards/rejected": -0.9569488763809204,
      "step": 528
    },
    {
      "epoch": 0.06922384885908237,
      "grad_norm": 3.7283387184143066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6499537229537964,
      "logits/rejected": 0.4823562800884247,
      "logps/chosen": -473.42535400390625,
      "logps/rejected": -406.8976745605469,
      "loss": 0.3681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3371618092060089,
      "rewards/margins": 0.9233304858207703,
      "rewards/rejected": -1.2604924440383911,
      "step": 529
    },
    {
      "epoch": 0.06935470679643412,
      "grad_norm": 6.186201095581055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2413814514875412,
      "logits/rejected": 0.23776544630527496,
      "logps/chosen": -582.166259765625,
      "logps/rejected": -529.814453125,
      "loss": 0.7752,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5268020629882812,
      "rewards/margins": -0.006748560816049576,
      "rewards/rejected": -0.520053505897522,
      "step": 530
    },
    {
      "epoch": 0.06948556473378588,
      "grad_norm": 4.973233222961426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5728267431259155,
      "logits/rejected": 0.5113343000411987,
      "logps/chosen": -538.994384765625,
      "logps/rejected": -450.9759826660156,
      "loss": 0.5429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0917816236615181,
      "rewards/margins": 0.5354502201080322,
      "rewards/rejected": -0.627231776714325,
      "step": 531
    },
    {
      "epoch": 0.06961642267113764,
      "grad_norm": 5.903800010681152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30590155720710754,
      "logits/rejected": 0.23494789004325867,
      "logps/chosen": -569.7425537109375,
      "logps/rejected": -408.41748046875,
      "loss": 0.5921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0174903869628906,
      "rewards/margins": 0.37137335538864136,
      "rewards/rejected": -1.3888638019561768,
      "step": 532
    },
    {
      "epoch": 0.0697472806084894,
      "grad_norm": 5.072930335998535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4241987466812134,
      "logits/rejected": 0.19205625355243683,
      "logps/chosen": -460.5841979980469,
      "logps/rejected": -345.00152587890625,
      "loss": 0.6145,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7204166650772095,
      "rewards/margins": 0.3588939905166626,
      "rewards/rejected": -1.079310655593872,
      "step": 533
    },
    {
      "epoch": 0.06987813854584117,
      "grad_norm": 5.2868828773498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43254226446151733,
      "logits/rejected": 0.27060362696647644,
      "logps/chosen": -422.1627197265625,
      "logps/rejected": -377.1828918457031,
      "loss": 0.6034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.689021646976471,
      "rewards/margins": 0.2963700294494629,
      "rewards/rejected": -0.9853916764259338,
      "step": 534
    },
    {
      "epoch": 0.07000899648319293,
      "grad_norm": 5.761595249176025,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6567197442054749,
      "logits/rejected": 0.6085025668144226,
      "logps/chosen": -501.04583740234375,
      "logps/rejected": -575.8863525390625,
      "loss": 0.7667,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6929832696914673,
      "rewards/margins": 0.08020000904798508,
      "rewards/rejected": -0.7731832265853882,
      "step": 535
    },
    {
      "epoch": 0.0701398544205447,
      "grad_norm": 8.21072006225586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3535008728504181,
      "logits/rejected": 0.2779059410095215,
      "logps/chosen": -374.13043212890625,
      "logps/rejected": -312.128173828125,
      "loss": 0.7882,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1154406070709229,
      "rewards/margins": -0.005288317799568176,
      "rewards/rejected": -1.110152244567871,
      "step": 536
    },
    {
      "epoch": 0.07027071235789646,
      "grad_norm": 5.433618068695068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5193707942962646,
      "logits/rejected": 0.5546920299530029,
      "logps/chosen": -541.5003051757812,
      "logps/rejected": -469.6351623535156,
      "loss": 0.4726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5355352163314819,
      "rewards/margins": 0.6052950620651245,
      "rewards/rejected": -1.1408302783966064,
      "step": 537
    },
    {
      "epoch": 0.07040157029524823,
      "grad_norm": 7.367741107940674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3781636357307434,
      "logits/rejected": 0.19489556550979614,
      "logps/chosen": -318.892578125,
      "logps/rejected": -436.2252197265625,
      "loss": 0.7658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6378538608551025,
      "rewards/margins": 0.03349991887807846,
      "rewards/rejected": -0.6713536977767944,
      "step": 538
    },
    {
      "epoch": 0.07053242823259999,
      "grad_norm": 6.389745235443115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4483412504196167,
      "logits/rejected": 0.5082813501358032,
      "logps/chosen": -414.11151123046875,
      "logps/rejected": -370.31005859375,
      "loss": 0.7598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8030401468276978,
      "rewards/margins": 0.11050910502672195,
      "rewards/rejected": -0.9135491847991943,
      "step": 539
    },
    {
      "epoch": 0.07066328616995175,
      "grad_norm": 5.112846851348877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7558637857437134,
      "logits/rejected": 0.7876338362693787,
      "logps/chosen": -532.7705078125,
      "logps/rejected": -547.9224853515625,
      "loss": 0.4435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37398892641067505,
      "rewards/margins": 0.9051690697669983,
      "rewards/rejected": -1.279158115386963,
      "step": 540
    },
    {
      "epoch": 0.0707941441073035,
      "grad_norm": 5.758161544799805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5759280920028687,
      "logits/rejected": 0.5228615999221802,
      "logps/chosen": -598.6896362304688,
      "logps/rejected": -435.2933349609375,
      "loss": 0.5839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.694064199924469,
      "rewards/margins": 0.5672056078910828,
      "rewards/rejected": -1.2612698078155518,
      "step": 541
    },
    {
      "epoch": 0.07092500204465527,
      "grad_norm": 5.166536808013916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3213072717189789,
      "logits/rejected": 0.48297375440597534,
      "logps/chosen": -592.8460083007812,
      "logps/rejected": -594.5549926757812,
      "loss": 0.5073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6406227350234985,
      "rewards/margins": 0.555005669593811,
      "rewards/rejected": -1.1956284046173096,
      "step": 542
    },
    {
      "epoch": 0.07105585998200703,
      "grad_norm": 6.006649971008301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2698959708213806,
      "logits/rejected": 0.3808152973651886,
      "logps/chosen": -364.7806701660156,
      "logps/rejected": -372.19989013671875,
      "loss": 0.7314,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0853103399276733,
      "rewards/margins": 0.09357605874538422,
      "rewards/rejected": -1.1788862943649292,
      "step": 543
    },
    {
      "epoch": 0.0711867179193588,
      "grad_norm": 4.651983737945557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3831998109817505,
      "logits/rejected": 0.31706154346466064,
      "logps/chosen": -516.0861206054688,
      "logps/rejected": -498.4683837890625,
      "loss": 0.4937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9291689395904541,
      "rewards/margins": 0.5761547088623047,
      "rewards/rejected": -1.5053236484527588,
      "step": 544
    },
    {
      "epoch": 0.07131757585671056,
      "grad_norm": 5.716291904449463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4087727665901184,
      "logits/rejected": 0.338661253452301,
      "logps/chosen": -685.156494140625,
      "logps/rejected": -425.2008972167969,
      "loss": 0.5576,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3379104733467102,
      "rewards/margins": 0.4840582013130188,
      "rewards/rejected": -0.821968674659729,
      "step": 545
    },
    {
      "epoch": 0.07144843379406232,
      "grad_norm": 4.5160322189331055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3030378222465515,
      "logits/rejected": 0.27881911396980286,
      "logps/chosen": -479.2321472167969,
      "logps/rejected": -542.3941650390625,
      "loss": 0.5117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28163450956344604,
      "rewards/margins": 0.5527158975601196,
      "rewards/rejected": -0.8343504071235657,
      "step": 546
    },
    {
      "epoch": 0.07157929173141409,
      "grad_norm": 3.9595866203308105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2834388017654419,
      "logits/rejected": 0.39339837431907654,
      "logps/chosen": -416.88214111328125,
      "logps/rejected": -406.8829345703125,
      "loss": 0.4803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31566768884658813,
      "rewards/margins": 0.6469439268112183,
      "rewards/rejected": -0.9626116156578064,
      "step": 547
    },
    {
      "epoch": 0.07171014966876585,
      "grad_norm": 6.047715663909912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35629409551620483,
      "logits/rejected": 0.28313350677490234,
      "logps/chosen": -402.5541687011719,
      "logps/rejected": -302.7913818359375,
      "loss": 0.6954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5944446921348572,
      "rewards/margins": 0.13345834612846375,
      "rewards/rejected": -0.7279029488563538,
      "step": 548
    },
    {
      "epoch": 0.07184100760611761,
      "grad_norm": 5.5562872886657715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5171433687210083,
      "logits/rejected": 0.5265737175941467,
      "logps/chosen": -434.56341552734375,
      "logps/rejected": -475.6893615722656,
      "loss": 0.5636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2665250599384308,
      "rewards/margins": 0.6407535076141357,
      "rewards/rejected": -0.9072787165641785,
      "step": 549
    },
    {
      "epoch": 0.07197186554346938,
      "grad_norm": 4.779975414276123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4672985076904297,
      "logits/rejected": 0.4307418465614319,
      "logps/chosen": -593.59716796875,
      "logps/rejected": -504.39892578125,
      "loss": 0.4744,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1680612564086914,
      "rewards/margins": 0.7008628845214844,
      "rewards/rejected": -0.8689241409301758,
      "step": 550
    },
    {
      "epoch": 0.07210272348082114,
      "grad_norm": 3.915134906768799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41699549555778503,
      "logits/rejected": 0.25471729040145874,
      "logps/chosen": -414.9356994628906,
      "logps/rejected": -341.91595458984375,
      "loss": 0.5256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23399008810520172,
      "rewards/margins": 0.601203203201294,
      "rewards/rejected": -0.8351933360099792,
      "step": 551
    },
    {
      "epoch": 0.07223358141817289,
      "grad_norm": 5.866732120513916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3287663161754608,
      "logits/rejected": 0.14956465363502502,
      "logps/chosen": -372.9248352050781,
      "logps/rejected": -279.57781982421875,
      "loss": 0.6159,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6803819537162781,
      "rewards/margins": 0.29614585638046265,
      "rewards/rejected": -0.976527750492096,
      "step": 552
    },
    {
      "epoch": 0.07236443935552465,
      "grad_norm": 5.879206657409668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17602024972438812,
      "logits/rejected": 0.15607067942619324,
      "logps/chosen": -427.7956237792969,
      "logps/rejected": -482.1649169921875,
      "loss": 0.6162,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6339375376701355,
      "rewards/margins": 0.47243279218673706,
      "rewards/rejected": -1.106370449066162,
      "step": 553
    },
    {
      "epoch": 0.07249529729287642,
      "grad_norm": 5.474693298339844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2210582196712494,
      "logits/rejected": 0.266406774520874,
      "logps/chosen": -354.8912048339844,
      "logps/rejected": -393.9143371582031,
      "loss": 0.5732,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6601938009262085,
      "rewards/margins": 0.4428331255912781,
      "rewards/rejected": -1.1030269861221313,
      "step": 554
    },
    {
      "epoch": 0.07262615523022818,
      "grad_norm": 3.1322576999664307,
      "learning_rate": 5e-06,
      "logits/chosen": 0.252920925617218,
      "logits/rejected": 0.3213740289211273,
      "logps/chosen": -413.0888977050781,
      "logps/rejected": -339.71331787109375,
      "loss": 0.4156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3698883056640625,
      "rewards/margins": 0.8007223010063171,
      "rewards/rejected": -1.1706106662750244,
      "step": 555
    },
    {
      "epoch": 0.07275701316757995,
      "grad_norm": 3.8296256065368652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5042593479156494,
      "logits/rejected": 0.5927265882492065,
      "logps/chosen": -595.9881591796875,
      "logps/rejected": -548.644287109375,
      "loss": 0.3473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09357012808322906,
      "rewards/margins": 1.0895363092422485,
      "rewards/rejected": -0.9959661364555359,
      "step": 556
    },
    {
      "epoch": 0.07288787110493171,
      "grad_norm": 5.748876571655273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6325517296791077,
      "logits/rejected": 0.5880264639854431,
      "logps/chosen": -473.6835632324219,
      "logps/rejected": -402.2953186035156,
      "loss": 0.6826,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8144634366035461,
      "rewards/margins": 0.15604554116725922,
      "rewards/rejected": -0.9705089330673218,
      "step": 557
    },
    {
      "epoch": 0.07301872904228347,
      "grad_norm": 4.281466007232666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5812399983406067,
      "logits/rejected": 0.6339294910430908,
      "logps/chosen": -529.0734252929688,
      "logps/rejected": -433.7259521484375,
      "loss": 0.5047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24378183484077454,
      "rewards/margins": 0.5467783212661743,
      "rewards/rejected": -0.7905601263046265,
      "step": 558
    },
    {
      "epoch": 0.07314958697963524,
      "grad_norm": 5.744758605957031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18344327807426453,
      "logits/rejected": 0.11273995786905289,
      "logps/chosen": -341.4320373535156,
      "logps/rejected": -350.67669677734375,
      "loss": 0.7142,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4218931198120117,
      "rewards/margins": 0.010934039950370789,
      "rewards/rejected": -0.4328271746635437,
      "step": 559
    },
    {
      "epoch": 0.073280444916987,
      "grad_norm": 4.2284955978393555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15962524712085724,
      "logits/rejected": 0.2702086269855499,
      "logps/chosen": -508.2572021484375,
      "logps/rejected": -501.21295166015625,
      "loss": 0.326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5080711245536804,
      "rewards/margins": 1.2153689861297607,
      "rewards/rejected": -1.7234400510787964,
      "step": 560
    },
    {
      "epoch": 0.07341130285433876,
      "grad_norm": 5.396412372589111,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3714447617530823,
      "logits/rejected": 0.5047827959060669,
      "logps/chosen": -534.0289306640625,
      "logps/rejected": -414.2251281738281,
      "loss": 0.5552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22197628021240234,
      "rewards/margins": 0.6102740168571472,
      "rewards/rejected": -0.8322502374649048,
      "step": 561
    },
    {
      "epoch": 0.07354216079169053,
      "grad_norm": 4.89509916305542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34351441264152527,
      "logits/rejected": 0.20901814103126526,
      "logps/chosen": -400.0853271484375,
      "logps/rejected": -345.4581604003906,
      "loss": 0.6342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6858786344528198,
      "rewards/margins": 0.20637035369873047,
      "rewards/rejected": -0.8922489881515503,
      "step": 562
    },
    {
      "epoch": 0.07367301872904228,
      "grad_norm": 7.0296478271484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4132855534553528,
      "logits/rejected": 0.2718999683856964,
      "logps/chosen": -432.42620849609375,
      "logps/rejected": -512.9171142578125,
      "loss": 0.6807,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0406957864761353,
      "rewards/margins": 0.22021961212158203,
      "rewards/rejected": -1.2609153985977173,
      "step": 563
    },
    {
      "epoch": 0.07380387666639404,
      "grad_norm": 3.6556155681610107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25659552216529846,
      "logits/rejected": 0.44894957542419434,
      "logps/chosen": -373.5753479003906,
      "logps/rejected": -345.7671203613281,
      "loss": 0.4866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30677586793899536,
      "rewards/margins": 0.557172954082489,
      "rewards/rejected": -0.8639488220214844,
      "step": 564
    },
    {
      "epoch": 0.0739347346037458,
      "grad_norm": 6.282023906707764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39974167943000793,
      "logits/rejected": 0.5017340183258057,
      "logps/chosen": -411.369873046875,
      "logps/rejected": -335.95867919921875,
      "loss": 0.7021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8614736795425415,
      "rewards/margins": 0.1366748958826065,
      "rewards/rejected": -0.9981485605239868,
      "step": 565
    },
    {
      "epoch": 0.07406559254109757,
      "grad_norm": 3.803328275680542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3260113596916199,
      "logits/rejected": 0.36697104573249817,
      "logps/chosen": -311.53167724609375,
      "logps/rejected": -357.60113525390625,
      "loss": 0.5723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.749454140663147,
      "rewards/margins": 0.3504640460014343,
      "rewards/rejected": -1.0999181270599365,
      "step": 566
    },
    {
      "epoch": 0.07419645047844933,
      "grad_norm": 4.9874396324157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06415658444166183,
      "logits/rejected": -0.12912745773792267,
      "logps/chosen": -387.5881652832031,
      "logps/rejected": -348.4391174316406,
      "loss": 0.6668,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4854108989238739,
      "rewards/margins": 0.10197964310646057,
      "rewards/rejected": -0.5873905420303345,
      "step": 567
    },
    {
      "epoch": 0.0743273084158011,
      "grad_norm": 4.797722816467285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3766574263572693,
      "logits/rejected": 0.36330747604370117,
      "logps/chosen": -410.744140625,
      "logps/rejected": -508.0826110839844,
      "loss": 0.4392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4619722366333008,
      "rewards/margins": 0.9908376932144165,
      "rewards/rejected": -1.4528099298477173,
      "step": 568
    },
    {
      "epoch": 0.07445816635315286,
      "grad_norm": 11.37069034576416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32707178592681885,
      "logits/rejected": 0.4578709006309509,
      "logps/chosen": -510.02911376953125,
      "logps/rejected": -428.27606201171875,
      "loss": 1.2483,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1476128101348877,
      "rewards/margins": -0.4107092320919037,
      "rewards/rejected": -0.7369035482406616,
      "step": 569
    },
    {
      "epoch": 0.07458902429050462,
      "grad_norm": 5.578540325164795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48030704259872437,
      "logits/rejected": 0.22989079356193542,
      "logps/chosen": -622.8952026367188,
      "logps/rejected": -541.45849609375,
      "loss": 0.5526,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6623947024345398,
      "rewards/margins": 0.4794224500656128,
      "rewards/rejected": -1.1418170928955078,
      "step": 570
    },
    {
      "epoch": 0.07471988222785639,
      "grad_norm": 7.587085247039795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11091676354408264,
      "logits/rejected": 0.22963814437389374,
      "logps/chosen": -484.3772888183594,
      "logps/rejected": -380.6324768066406,
      "loss": 0.771,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9781915545463562,
      "rewards/margins": 0.07988112419843674,
      "rewards/rejected": -1.0580726861953735,
      "step": 571
    },
    {
      "epoch": 0.07485074016520815,
      "grad_norm": 3.437798023223877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48232442140579224,
      "logits/rejected": 0.3955938518047333,
      "logps/chosen": -621.775390625,
      "logps/rejected": -559.1240844726562,
      "loss": 0.3397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20310936868190765,
      "rewards/margins": 0.9758033752441406,
      "rewards/rejected": -0.7726940512657166,
      "step": 572
    },
    {
      "epoch": 0.0749815981025599,
      "grad_norm": 6.060243606567383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07132180780172348,
      "logits/rejected": -0.05847782641649246,
      "logps/chosen": -365.0321960449219,
      "logps/rejected": -369.216552734375,
      "loss": 0.6537,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8291462063789368,
      "rewards/margins": 0.16620388627052307,
      "rewards/rejected": -0.9953500032424927,
      "step": 573
    },
    {
      "epoch": 0.07511245603991167,
      "grad_norm": 5.5098676681518555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3170020878314972,
      "logits/rejected": 0.36078399419784546,
      "logps/chosen": -413.48687744140625,
      "logps/rejected": -489.5687255859375,
      "loss": 0.4756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3386644423007965,
      "rewards/margins": 0.6719368100166321,
      "rewards/rejected": -1.010601282119751,
      "step": 574
    },
    {
      "epoch": 0.07524331397726343,
      "grad_norm": 4.49830961227417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6862958073616028,
      "logits/rejected": 0.6309306621551514,
      "logps/chosen": -496.69384765625,
      "logps/rejected": -403.9462585449219,
      "loss": 0.5243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7028360962867737,
      "rewards/margins": 0.5485152006149292,
      "rewards/rejected": -1.2513513565063477,
      "step": 575
    },
    {
      "epoch": 0.07537417191461519,
      "grad_norm": 6.373063564300537,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7126224637031555,
      "logits/rejected": 0.6185997724533081,
      "logps/chosen": -453.33563232421875,
      "logps/rejected": -462.6956787109375,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4432470500469208,
      "rewards/margins": 0.1440078616142273,
      "rewards/rejected": -0.5872548818588257,
      "step": 576
    },
    {
      "epoch": 0.07550502985196696,
      "grad_norm": 3.7267727851867676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03506498038768768,
      "logits/rejected": 0.17442482709884644,
      "logps/chosen": -248.89695739746094,
      "logps/rejected": -327.09228515625,
      "loss": 0.5305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7745964527130127,
      "rewards/margins": 0.4785751402378082,
      "rewards/rejected": -1.2531715631484985,
      "step": 577
    },
    {
      "epoch": 0.07563588778931872,
      "grad_norm": 7.03753137588501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042436257004737854,
      "logits/rejected": 0.07635557651519775,
      "logps/chosen": -561.4857788085938,
      "logps/rejected": -381.1213073730469,
      "loss": 0.5418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2232629805803299,
      "rewards/margins": 0.9119146466255188,
      "rewards/rejected": -1.135177493095398,
      "step": 578
    },
    {
      "epoch": 0.07576674572667048,
      "grad_norm": 5.630202770233154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5324530005455017,
      "logits/rejected": 0.37878498435020447,
      "logps/chosen": -377.5098571777344,
      "logps/rejected": -435.8712158203125,
      "loss": 0.6967,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5339750647544861,
      "rewards/margins": 0.3880954682826996,
      "rewards/rejected": -0.9220705032348633,
      "step": 579
    },
    {
      "epoch": 0.07589760366402225,
      "grad_norm": 7.821728706359863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48111385107040405,
      "logits/rejected": 0.7227789759635925,
      "logps/chosen": -515.9563598632812,
      "logps/rejected": -508.7351989746094,
      "loss": 0.7528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8201965093612671,
      "rewards/margins": 0.060179151594638824,
      "rewards/rejected": -0.8803757429122925,
      "step": 580
    },
    {
      "epoch": 0.07602846160137401,
      "grad_norm": 4.996608734130859,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3890743851661682,
      "logits/rejected": 0.3864770233631134,
      "logps/chosen": -412.99969482421875,
      "logps/rejected": -450.030029296875,
      "loss": 0.5521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18826924264431,
      "rewards/margins": 0.6313654184341431,
      "rewards/rejected": -0.8196346163749695,
      "step": 581
    },
    {
      "epoch": 0.07615931953872578,
      "grad_norm": 4.224368095397949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2511407434940338,
      "logits/rejected": 0.1943853199481964,
      "logps/chosen": -325.4731750488281,
      "logps/rejected": -487.1764221191406,
      "loss": 0.529,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2920379638671875,
      "rewards/margins": 0.4818224012851715,
      "rewards/rejected": -0.7738603949546814,
      "step": 582
    },
    {
      "epoch": 0.07629017747607754,
      "grad_norm": 6.504881381988525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4134092330932617,
      "logits/rejected": 0.2834189236164093,
      "logps/chosen": -459.2058410644531,
      "logps/rejected": -627.3902587890625,
      "loss": 0.7238,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011598780751228333,
      "rewards/margins": 0.02964724600315094,
      "rewards/rejected": -0.04124603047966957,
      "step": 583
    },
    {
      "epoch": 0.07642103541342929,
      "grad_norm": 4.4142866134643555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19837507605552673,
      "logits/rejected": 0.17139290273189545,
      "logps/chosen": -746.8282470703125,
      "logps/rejected": -634.1548461914062,
      "loss": 0.4015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21640872955322266,
      "rewards/margins": 1.2056630849838257,
      "rewards/rejected": -1.4220718145370483,
      "step": 584
    },
    {
      "epoch": 0.07655189335078105,
      "grad_norm": 4.542256832122803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09947000443935394,
      "logits/rejected": 0.393707811832428,
      "logps/chosen": -245.8990936279297,
      "logps/rejected": -307.49444580078125,
      "loss": 0.5988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5421686172485352,
      "rewards/margins": 0.24923114478588104,
      "rewards/rejected": -0.7913997769355774,
      "step": 585
    },
    {
      "epoch": 0.07668275128813282,
      "grad_norm": 5.697918891906738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2513849139213562,
      "logits/rejected": 0.3568103015422821,
      "logps/chosen": -373.8467102050781,
      "logps/rejected": -405.53729248046875,
      "loss": 0.5762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2333230972290039,
      "rewards/margins": 0.5017656683921814,
      "rewards/rejected": -0.7350887656211853,
      "step": 586
    },
    {
      "epoch": 0.07681360922548458,
      "grad_norm": 6.46734094619751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4872171878814697,
      "logits/rejected": 0.3994636535644531,
      "logps/chosen": -436.31610107421875,
      "logps/rejected": -365.2832946777344,
      "loss": 0.8515,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7711702585220337,
      "rewards/margins": -0.1049284040927887,
      "rewards/rejected": -0.6662418842315674,
      "step": 587
    },
    {
      "epoch": 0.07694446716283634,
      "grad_norm": 4.5301127433776855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4621230959892273,
      "logits/rejected": 0.25730323791503906,
      "logps/chosen": -420.05865478515625,
      "logps/rejected": -428.44427490234375,
      "loss": 0.5862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7331928610801697,
      "rewards/margins": 0.3724142014980316,
      "rewards/rejected": -1.105607032775879,
      "step": 588
    },
    {
      "epoch": 0.07707532510018811,
      "grad_norm": 6.637760162353516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49131736159324646,
      "logits/rejected": 0.637022078037262,
      "logps/chosen": -631.3074951171875,
      "logps/rejected": -557.6849365234375,
      "loss": 0.6016,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7114502191543579,
      "rewards/margins": 0.3156578242778778,
      "rewards/rejected": -1.027108073234558,
      "step": 589
    },
    {
      "epoch": 0.07720618303753987,
      "grad_norm": 5.564302444458008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1319786012172699,
      "logits/rejected": 0.10307706147432327,
      "logps/chosen": -613.5867919921875,
      "logps/rejected": -576.0784912109375,
      "loss": 0.5463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8054447174072266,
      "rewards/margins": 0.913963258266449,
      "rewards/rejected": -1.7194081544876099,
      "step": 590
    },
    {
      "epoch": 0.07733704097489164,
      "grad_norm": 6.865749359130859,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5051469206809998,
      "logits/rejected": 0.38690754771232605,
      "logps/chosen": -381.4940490722656,
      "logps/rejected": -332.7532958984375,
      "loss": 0.8507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0207101106643677,
      "rewards/margins": 0.17025698721408844,
      "rewards/rejected": -1.1909672021865845,
      "step": 591
    },
    {
      "epoch": 0.0774678989122434,
      "grad_norm": 7.841463565826416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1564977765083313,
      "logits/rejected": 0.3083302974700928,
      "logps/chosen": -571.4124145507812,
      "logps/rejected": -487.4947814941406,
      "loss": 0.8838,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0785176753997803,
      "rewards/margins": -0.021799996495246887,
      "rewards/rejected": -1.0567176342010498,
      "step": 592
    },
    {
      "epoch": 0.07759875684959516,
      "grad_norm": 3.903111457824707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5895205140113831,
      "logits/rejected": 0.3392395079135895,
      "logps/chosen": -870.1170654296875,
      "logps/rejected": -536.3009643554688,
      "loss": 0.3333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.003688409924507141,
      "rewards/margins": 1.2940902709960938,
      "rewards/rejected": -1.2977787256240845,
      "step": 593
    },
    {
      "epoch": 0.07772961478694693,
      "grad_norm": 4.106026649475098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6583698987960815,
      "logits/rejected": 0.49732813239097595,
      "logps/chosen": -422.431396484375,
      "logps/rejected": -430.88677978515625,
      "loss": 0.5348,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4070817828178406,
      "rewards/margins": 0.38976627588272095,
      "rewards/rejected": -0.7968481779098511,
      "step": 594
    },
    {
      "epoch": 0.07786047272429868,
      "grad_norm": 4.02243709564209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20079660415649414,
      "logits/rejected": 0.47314587235450745,
      "logps/chosen": -435.7830810546875,
      "logps/rejected": -443.17205810546875,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31482332944869995,
      "rewards/margins": 0.914890706539154,
      "rewards/rejected": -1.2297139167785645,
      "step": 595
    },
    {
      "epoch": 0.07799133066165044,
      "grad_norm": 6.113710880279541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24151621758937836,
      "logits/rejected": 0.4053685665130615,
      "logps/chosen": -369.5921325683594,
      "logps/rejected": -408.4423828125,
      "loss": 0.698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.405161738395691,
      "rewards/margins": 0.23246952891349792,
      "rewards/rejected": -1.6376312971115112,
      "step": 596
    },
    {
      "epoch": 0.0781221885990022,
      "grad_norm": 5.232089996337891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3651454448699951,
      "logits/rejected": 0.2943764626979828,
      "logps/chosen": -483.1075439453125,
      "logps/rejected": -439.61328125,
      "loss": 0.4923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6346903443336487,
      "rewards/margins": 0.9216452836990356,
      "rewards/rejected": -1.5563355684280396,
      "step": 597
    },
    {
      "epoch": 0.07825304653635397,
      "grad_norm": 8.865558624267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06781446188688278,
      "logits/rejected": 0.14466488361358643,
      "logps/chosen": -448.77032470703125,
      "logps/rejected": -402.7646484375,
      "loss": 0.7307,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7100845575332642,
      "rewards/margins": 0.6348116397857666,
      "rewards/rejected": -1.3448960781097412,
      "step": 598
    },
    {
      "epoch": 0.07838390447370573,
      "grad_norm": 6.943960666656494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40529805421829224,
      "logits/rejected": 0.2998383045196533,
      "logps/chosen": -308.8167419433594,
      "logps/rejected": -365.0606384277344,
      "loss": 0.7295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9043148159980774,
      "rewards/margins": 0.22233466804027557,
      "rewards/rejected": -1.1266494989395142,
      "step": 599
    },
    {
      "epoch": 0.0785147624110575,
      "grad_norm": 4.518582820892334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20563268661499023,
      "logits/rejected": 0.03570017218589783,
      "logps/chosen": -340.0049743652344,
      "logps/rejected": -396.41802978515625,
      "loss": 0.4297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6435152292251587,
      "rewards/margins": 0.8929790258407593,
      "rewards/rejected": -1.536494255065918,
      "step": 600
    },
    {
      "epoch": 0.07864562034840926,
      "grad_norm": 4.83181619644165,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26775240898132324,
      "logits/rejected": 0.3111736476421356,
      "logps/chosen": -661.511962890625,
      "logps/rejected": -638.921875,
      "loss": 0.3331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25054892897605896,
      "rewards/margins": 1.0135536193847656,
      "rewards/rejected": -0.7630046606063843,
      "step": 601
    },
    {
      "epoch": 0.07877647828576102,
      "grad_norm": 6.060476779937744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.057155098766088486,
      "logits/rejected": 0.14107057452201843,
      "logps/chosen": -463.57574462890625,
      "logps/rejected": -564.4234008789062,
      "loss": 0.5039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0836358070373535,
      "rewards/margins": 0.47007372975349426,
      "rewards/rejected": -1.5537093877792358,
      "step": 602
    },
    {
      "epoch": 0.07890733622311279,
      "grad_norm": 4.775518417358398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11939436942338943,
      "logits/rejected": 0.39443227648735046,
      "logps/chosen": -469.60906982421875,
      "logps/rejected": -375.1692199707031,
      "loss": 0.4291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14235132932662964,
      "rewards/margins": 0.7405291795730591,
      "rewards/rejected": -0.882880687713623,
      "step": 603
    },
    {
      "epoch": 0.07903819416046455,
      "grad_norm": 6.439942836761475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5924036502838135,
      "logits/rejected": 0.6414279341697693,
      "logps/chosen": -507.16717529296875,
      "logps/rejected": -390.12896728515625,
      "loss": 0.8063,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.3330358266830444,
      "rewards/margins": -0.1045173704624176,
      "rewards/rejected": -1.2285186052322388,
      "step": 604
    },
    {
      "epoch": 0.07916905209781631,
      "grad_norm": 6.842353343963623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3685915172100067,
      "logits/rejected": 0.3555654287338257,
      "logps/chosen": -594.8318481445312,
      "logps/rejected": -553.3847045898438,
      "loss": 0.6296,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4099411368370056,
      "rewards/margins": 0.3281477391719818,
      "rewards/rejected": -0.7380887866020203,
      "step": 605
    },
    {
      "epoch": 0.07929991003516806,
      "grad_norm": 7.732364654541016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5818745493888855,
      "logits/rejected": 0.5077831149101257,
      "logps/chosen": -398.8362121582031,
      "logps/rejected": -348.3378601074219,
      "loss": 0.7916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9364335536956787,
      "rewards/margins": 0.13003842532634735,
      "rewards/rejected": -1.066472053527832,
      "step": 606
    },
    {
      "epoch": 0.07943076797251983,
      "grad_norm": 8.572183609008789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5494917631149292,
      "logits/rejected": 0.5618902444839478,
      "logps/chosen": -445.146240234375,
      "logps/rejected": -430.952880859375,
      "loss": 0.5561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9399220943450928,
      "rewards/margins": 0.5128353834152222,
      "rewards/rejected": -1.4527573585510254,
      "step": 607
    },
    {
      "epoch": 0.07956162590987159,
      "grad_norm": 8.005697250366211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5264142155647278,
      "logits/rejected": 0.43511825799942017,
      "logps/chosen": -440.6544494628906,
      "logps/rejected": -332.2230224609375,
      "loss": 0.8876,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9453064203262329,
      "rewards/margins": -0.12507399916648865,
      "rewards/rejected": -0.8202325105667114,
      "step": 608
    },
    {
      "epoch": 0.07969248384722336,
      "grad_norm": 5.148588180541992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3442939221858978,
      "logits/rejected": 0.4571460485458374,
      "logps/chosen": -476.26263427734375,
      "logps/rejected": -428.5596618652344,
      "loss": 0.4922,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7235433459281921,
      "rewards/margins": 0.5649979114532471,
      "rewards/rejected": -1.288541316986084,
      "step": 609
    },
    {
      "epoch": 0.07982334178457512,
      "grad_norm": 5.771223545074463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47838112711906433,
      "logits/rejected": 0.34290722012519836,
      "logps/chosen": -682.5574340820312,
      "logps/rejected": -694.0902709960938,
      "loss": 0.52,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2116168886423111,
      "rewards/margins": 0.7824417352676392,
      "rewards/rejected": -0.9940586686134338,
      "step": 610
    },
    {
      "epoch": 0.07995419972192688,
      "grad_norm": 6.026559352874756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7121723890304565,
      "logits/rejected": 0.56844162940979,
      "logps/chosen": -415.5705871582031,
      "logps/rejected": -453.989013671875,
      "loss": 0.7051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1826814413070679,
      "rewards/margins": 0.19275875389575958,
      "rewards/rejected": -1.3754403591156006,
      "step": 611
    },
    {
      "epoch": 0.08008505765927865,
      "grad_norm": 6.333983898162842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3876331150531769,
      "logits/rejected": 0.454009085893631,
      "logps/chosen": -425.7310485839844,
      "logps/rejected": -589.8071899414062,
      "loss": 0.5477,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7538512945175171,
      "rewards/margins": 0.9559643268585205,
      "rewards/rejected": -1.709815502166748,
      "step": 612
    },
    {
      "epoch": 0.08021591559663041,
      "grad_norm": 5.739504337310791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4673055410385132,
      "logits/rejected": 0.6031284332275391,
      "logps/chosen": -540.8517456054688,
      "logps/rejected": -599.75,
      "loss": 0.5163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5888700485229492,
      "rewards/margins": 0.8456286191940308,
      "rewards/rejected": -1.4344985485076904,
      "step": 613
    },
    {
      "epoch": 0.08034677353398217,
      "grad_norm": 4.955618858337402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3549942970275879,
      "logits/rejected": 0.33106377720832825,
      "logps/chosen": -599.3329467773438,
      "logps/rejected": -401.10528564453125,
      "loss": 0.5715,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03150041028857231,
      "rewards/margins": 0.852580726146698,
      "rewards/rejected": -0.8840811252593994,
      "step": 614
    },
    {
      "epoch": 0.08047763147133394,
      "grad_norm": 4.372398853302002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4283871352672577,
      "logits/rejected": 0.35318344831466675,
      "logps/chosen": -425.9813537597656,
      "logps/rejected": -454.7784423828125,
      "loss": 0.5527,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2807786166667938,
      "rewards/margins": 0.4072263538837433,
      "rewards/rejected": -0.6880050301551819,
      "step": 615
    },
    {
      "epoch": 0.0806084894086857,
      "grad_norm": 10.337958335876465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33282768726348877,
      "logits/rejected": 0.27063581347465515,
      "logps/chosen": -553.2041015625,
      "logps/rejected": -414.6070251464844,
      "loss": 0.971,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.191310167312622,
      "rewards/margins": -0.3539770245552063,
      "rewards/rejected": -0.837333083152771,
      "step": 616
    },
    {
      "epoch": 0.08073934734603745,
      "grad_norm": 8.000749588012695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28136253356933594,
      "logits/rejected": -0.10299983620643616,
      "logps/chosen": -595.323486328125,
      "logps/rejected": -604.5377197265625,
      "loss": 0.744,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9596537351608276,
      "rewards/margins": 0.06650947034358978,
      "rewards/rejected": -1.0261632204055786,
      "step": 617
    },
    {
      "epoch": 0.08087020528338922,
      "grad_norm": 5.832149028778076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1944277137517929,
      "logits/rejected": 0.11748946458101273,
      "logps/chosen": -475.3701477050781,
      "logps/rejected": -531.9950561523438,
      "loss": 0.6643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6488937735557556,
      "rewards/margins": 0.29072093963623047,
      "rewards/rejected": -0.9396146535873413,
      "step": 618
    },
    {
      "epoch": 0.08100106322074098,
      "grad_norm": 5.27000617980957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15531522035598755,
      "logits/rejected": 0.12565922737121582,
      "logps/chosen": -444.3047790527344,
      "logps/rejected": -375.3468017578125,
      "loss": 0.5164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29149535298347473,
      "rewards/margins": 0.5492376685142517,
      "rewards/rejected": -0.8407330513000488,
      "step": 619
    },
    {
      "epoch": 0.08113192115809274,
      "grad_norm": 7.446763038635254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24480822682380676,
      "logits/rejected": 0.12082576751708984,
      "logps/chosen": -559.597900390625,
      "logps/rejected": -534.2866821289062,
      "loss": 0.7889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.767646074295044,
      "rewards/margins": 0.02757381647825241,
      "rewards/rejected": -0.7952198386192322,
      "step": 620
    },
    {
      "epoch": 0.0812627790954445,
      "grad_norm": 6.935051918029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4155985116958618,
      "logits/rejected": 0.5966413021087646,
      "logps/chosen": -494.93994140625,
      "logps/rejected": -546.80712890625,
      "loss": 0.6929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3793393671512604,
      "rewards/margins": 0.33370599150657654,
      "rewards/rejected": -0.7130453586578369,
      "step": 621
    },
    {
      "epoch": 0.08139363703279627,
      "grad_norm": 4.616891384124756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1875094324350357,
      "logits/rejected": 0.1552537977695465,
      "logps/chosen": -491.1944274902344,
      "logps/rejected": -386.51123046875,
      "loss": 0.5521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0237908363342285,
      "rewards/margins": 0.7334362268447876,
      "rewards/rejected": -1.7572271823883057,
      "step": 622
    },
    {
      "epoch": 0.08152449497014803,
      "grad_norm": 6.181064605712891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3242219388484955,
      "logits/rejected": 0.3278437852859497,
      "logps/chosen": -442.26385498046875,
      "logps/rejected": -483.918701171875,
      "loss": 0.5958,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8190769553184509,
      "rewards/margins": 0.37236136198043823,
      "rewards/rejected": -1.1914383172988892,
      "step": 623
    },
    {
      "epoch": 0.0816553529074998,
      "grad_norm": 8.293919563293457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4672517776489258,
      "logits/rejected": 0.28493204712867737,
      "logps/chosen": -439.84033203125,
      "logps/rejected": -329.2613220214844,
      "loss": 0.6863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.622031569480896,
      "rewards/margins": 0.10864201188087463,
      "rewards/rejected": -0.7306736707687378,
      "step": 624
    },
    {
      "epoch": 0.08178621084485156,
      "grad_norm": 4.7347002029418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47789236903190613,
      "logits/rejected": 0.4428996443748474,
      "logps/chosen": -494.90771484375,
      "logps/rejected": -676.5873413085938,
      "loss": 0.457,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4069698452949524,
      "rewards/margins": 0.917076051235199,
      "rewards/rejected": -1.324046015739441,
      "step": 625
    },
    {
      "epoch": 0.08191706878220333,
      "grad_norm": 5.708249092102051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22533018887043,
      "logits/rejected": 0.24437302350997925,
      "logps/chosen": -402.61962890625,
      "logps/rejected": -435.631103515625,
      "loss": 0.6388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9094488024711609,
      "rewards/margins": 0.5622310638427734,
      "rewards/rejected": -1.471679925918579,
      "step": 626
    },
    {
      "epoch": 0.08204792671955509,
      "grad_norm": 6.163599967956543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.284095823764801,
      "logits/rejected": 0.2144695669412613,
      "logps/chosen": -620.782470703125,
      "logps/rejected": -473.8240051269531,
      "loss": 0.5334,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5377227663993835,
      "rewards/margins": 0.5896827578544617,
      "rewards/rejected": -1.1274055242538452,
      "step": 627
    },
    {
      "epoch": 0.08217878465690684,
      "grad_norm": 3.937933921813965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4588659405708313,
      "logits/rejected": 0.3458237051963806,
      "logps/chosen": -444.3167724609375,
      "logps/rejected": -441.48760986328125,
      "loss": 0.4462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07856824994087219,
      "rewards/margins": 0.8828451037406921,
      "rewards/rejected": -0.9614133834838867,
      "step": 628
    },
    {
      "epoch": 0.0823096425942586,
      "grad_norm": 4.046438694000244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27248841524124146,
      "logits/rejected": 0.16147559881210327,
      "logps/chosen": -518.845458984375,
      "logps/rejected": -546.0999755859375,
      "loss": 0.3542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2959021329879761,
      "rewards/margins": 1.376673698425293,
      "rewards/rejected": -1.6725757122039795,
      "step": 629
    },
    {
      "epoch": 0.08244050053161037,
      "grad_norm": 4.958939552307129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.520575761795044,
      "logits/rejected": 0.30248603224754333,
      "logps/chosen": -455.98828125,
      "logps/rejected": -459.309326171875,
      "loss": 0.5999,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6421880722045898,
      "rewards/margins": 0.21616230905056,
      "rewards/rejected": -0.858350396156311,
      "step": 630
    },
    {
      "epoch": 0.08257135846896213,
      "grad_norm": 5.958204746246338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2995547354221344,
      "logits/rejected": 0.2395501583814621,
      "logps/chosen": -297.2327880859375,
      "logps/rejected": -533.1495361328125,
      "loss": 0.6347,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6194088459014893,
      "rewards/margins": 0.5696460008621216,
      "rewards/rejected": -1.1890548467636108,
      "step": 631
    },
    {
      "epoch": 0.0827022164063139,
      "grad_norm": 4.965888500213623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14624613523483276,
      "logits/rejected": 0.4211803376674652,
      "logps/chosen": -422.95751953125,
      "logps/rejected": -423.090087890625,
      "loss": 0.467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6720766425132751,
      "rewards/margins": 0.6008323431015015,
      "rewards/rejected": -1.2729090452194214,
      "step": 632
    },
    {
      "epoch": 0.08283307434366566,
      "grad_norm": 6.3570356369018555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.376274973154068,
      "logits/rejected": 0.38022691011428833,
      "logps/chosen": -436.07135009765625,
      "logps/rejected": -348.6569519042969,
      "loss": 0.6758,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0261834859848022,
      "rewards/margins": 0.11804141104221344,
      "rewards/rejected": -1.1442248821258545,
      "step": 633
    },
    {
      "epoch": 0.08296393228101742,
      "grad_norm": 7.872862815856934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5978165864944458,
      "logits/rejected": 0.5434811115264893,
      "logps/chosen": -349.8284912109375,
      "logps/rejected": -504.9044189453125,
      "loss": 0.9766,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9495737552642822,
      "rewards/margins": -0.34142494201660156,
      "rewards/rejected": -0.6081488132476807,
      "step": 634
    },
    {
      "epoch": 0.08309479021836919,
      "grad_norm": 4.995038986206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6045863628387451,
      "logits/rejected": 0.34691354632377625,
      "logps/chosen": -422.1309814453125,
      "logps/rejected": -411.5340270996094,
      "loss": 0.4087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5022117495536804,
      "rewards/margins": 0.9395780563354492,
      "rewards/rejected": -1.441789984703064,
      "step": 635
    },
    {
      "epoch": 0.08322564815572095,
      "grad_norm": 4.77762508392334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5715696215629578,
      "logits/rejected": 0.45048975944519043,
      "logps/chosen": -394.9236755371094,
      "logps/rejected": -366.5840759277344,
      "loss": 0.4948,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7067022323608398,
      "rewards/margins": 0.4979928731918335,
      "rewards/rejected": -1.2046951055526733,
      "step": 636
    },
    {
      "epoch": 0.08335650609307271,
      "grad_norm": 5.393298625946045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30442970991134644,
      "logits/rejected": 0.3134022355079651,
      "logps/chosen": -652.6697998046875,
      "logps/rejected": -591.4445190429688,
      "loss": 0.5477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2717861235141754,
      "rewards/margins": 0.3730102777481079,
      "rewards/rejected": -0.6447964310646057,
      "step": 637
    },
    {
      "epoch": 0.08348736403042448,
      "grad_norm": 5.447670936584473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2889634966850281,
      "logits/rejected": 0.2479490339756012,
      "logps/chosen": -428.6384582519531,
      "logps/rejected": -415.3304748535156,
      "loss": 0.5756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3785644769668579,
      "rewards/margins": 0.32995396852493286,
      "rewards/rejected": -0.708518385887146,
      "step": 638
    },
    {
      "epoch": 0.08361822196777623,
      "grad_norm": 7.760899066925049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4494265913963318,
      "logits/rejected": 0.5862332582473755,
      "logps/chosen": -398.87701416015625,
      "logps/rejected": -690.203857421875,
      "loss": 0.7867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5869243741035461,
      "rewards/margins": 0.18511013686656952,
      "rewards/rejected": -0.7720344662666321,
      "step": 639
    },
    {
      "epoch": 0.08374907990512799,
      "grad_norm": 5.665530681610107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047599099576473236,
      "logits/rejected": 0.05168839171528816,
      "logps/chosen": -319.5279235839844,
      "logps/rejected": -353.1741943359375,
      "loss": 0.7096,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1539037227630615,
      "rewards/margins": 0.07700862735509872,
      "rewards/rejected": -1.2309123277664185,
      "step": 640
    },
    {
      "epoch": 0.08387993784247975,
      "grad_norm": 8.260770797729492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32372987270355225,
      "logits/rejected": 0.30254024267196655,
      "logps/chosen": -630.8812866210938,
      "logps/rejected": -526.6857299804688,
      "loss": 0.7832,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4415365159511566,
      "rewards/margins": 0.07848627120256424,
      "rewards/rejected": -0.5200228095054626,
      "step": 641
    },
    {
      "epoch": 0.08401079577983152,
      "grad_norm": 6.372807025909424,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6419788002967834,
      "logits/rejected": 0.4262661933898926,
      "logps/chosen": -460.701416015625,
      "logps/rejected": -379.1751708984375,
      "loss": 0.7596,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5997520685195923,
      "rewards/margins": -0.061101146042346954,
      "rewards/rejected": -0.5386509299278259,
      "step": 642
    },
    {
      "epoch": 0.08414165371718328,
      "grad_norm": 4.005831718444824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5686343908309937,
      "logits/rejected": 0.558276355266571,
      "logps/chosen": -346.2499084472656,
      "logps/rejected": -377.18316650390625,
      "loss": 0.3789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7231609225273132,
      "rewards/margins": 0.8638200163841248,
      "rewards/rejected": -1.586980938911438,
      "step": 643
    },
    {
      "epoch": 0.08427251165453505,
      "grad_norm": 4.925214767456055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4655352234840393,
      "logits/rejected": 0.43716174364089966,
      "logps/chosen": -447.569580078125,
      "logps/rejected": -545.9219970703125,
      "loss": 0.61,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5977300405502319,
      "rewards/margins": 0.3649175465106964,
      "rewards/rejected": -0.9626476168632507,
      "step": 644
    },
    {
      "epoch": 0.08440336959188681,
      "grad_norm": 4.510190963745117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23542168736457825,
      "logits/rejected": -0.17362800240516663,
      "logps/chosen": -388.1128234863281,
      "logps/rejected": -594.4652099609375,
      "loss": 0.4387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3230358362197876,
      "rewards/margins": 1.026951551437378,
      "rewards/rejected": -1.349987268447876,
      "step": 645
    },
    {
      "epoch": 0.08453422752923857,
      "grad_norm": 6.505947113037109,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4999985098838806,
      "logits/rejected": 0.46168434619903564,
      "logps/chosen": -433.7054138183594,
      "logps/rejected": -407.11041259765625,
      "loss": 0.7436,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8176323175430298,
      "rewards/margins": -0.049497805535793304,
      "rewards/rejected": -0.7681345343589783,
      "step": 646
    },
    {
      "epoch": 0.08466508546659034,
      "grad_norm": 5.3151655197143555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47301244735717773,
      "logits/rejected": 0.6047568321228027,
      "logps/chosen": -460.1078796386719,
      "logps/rejected": -405.2734375,
      "loss": 0.585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6606385707855225,
      "rewards/margins": 0.5534639954566956,
      "rewards/rejected": -1.2141026258468628,
      "step": 647
    },
    {
      "epoch": 0.0847959434039421,
      "grad_norm": 5.517400741577148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38026338815689087,
      "logits/rejected": 0.2823834717273712,
      "logps/chosen": -589.7760009765625,
      "logps/rejected": -503.83526611328125,
      "loss": 0.5738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7287096381187439,
      "rewards/margins": 0.35953521728515625,
      "rewards/rejected": -1.0882447957992554,
      "step": 648
    },
    {
      "epoch": 0.08492680134129386,
      "grad_norm": 4.422211170196533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29710519313812256,
      "logits/rejected": 0.302089124917984,
      "logps/chosen": -264.60418701171875,
      "logps/rejected": -334.1761779785156,
      "loss": 0.6304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8938072919845581,
      "rewards/margins": 0.19050811231136322,
      "rewards/rejected": -1.084315299987793,
      "step": 649
    },
    {
      "epoch": 0.08505765927864561,
      "grad_norm": 6.3458123207092285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6038203835487366,
      "logits/rejected": 0.6036616563796997,
      "logps/chosen": -565.8695068359375,
      "logps/rejected": -683.81982421875,
      "loss": 0.4689,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7060847878456116,
      "rewards/margins": 0.5945852994918823,
      "rewards/rejected": -1.3006701469421387,
      "step": 650
    },
    {
      "epoch": 0.08518851721599738,
      "grad_norm": 4.13938045501709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21974988281726837,
      "logits/rejected": 0.25342634320259094,
      "logps/chosen": -343.1983642578125,
      "logps/rejected": -466.7223205566406,
      "loss": 0.417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5337058901786804,
      "rewards/margins": 0.8673902750015259,
      "rewards/rejected": -1.401096224784851,
      "step": 651
    },
    {
      "epoch": 0.08531937515334914,
      "grad_norm": 3.7139358520507812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48540085554122925,
      "logits/rejected": 0.36174362897872925,
      "logps/chosen": -585.0426635742188,
      "logps/rejected": -442.3195495605469,
      "loss": 0.3806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13925513625144958,
      "rewards/margins": 1.0845041275024414,
      "rewards/rejected": -0.9452489614486694,
      "step": 652
    },
    {
      "epoch": 0.0854502330907009,
      "grad_norm": 3.4343464374542236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23698538541793823,
      "logits/rejected": 0.17784440517425537,
      "logps/chosen": -344.70123291015625,
      "logps/rejected": -449.8933410644531,
      "loss": 0.3438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12573832273483276,
      "rewards/margins": 1.4582011699676514,
      "rewards/rejected": -1.583939552307129,
      "step": 653
    },
    {
      "epoch": 0.08558109102805267,
      "grad_norm": 7.901891708374023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04568255692720413,
      "logits/rejected": -0.1714884340763092,
      "logps/chosen": -453.1829528808594,
      "logps/rejected": -444.10272216796875,
      "loss": 0.7328,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9014967083930969,
      "rewards/margins": 0.09510824829339981,
      "rewards/rejected": -0.9966049194335938,
      "step": 654
    },
    {
      "epoch": 0.08571194896540443,
      "grad_norm": 5.999365329742432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4193783402442932,
      "logits/rejected": 0.2629624903202057,
      "logps/chosen": -457.82098388671875,
      "logps/rejected": -349.6984558105469,
      "loss": 0.6944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4664863646030426,
      "rewards/margins": 0.4713001251220703,
      "rewards/rejected": -0.9377865195274353,
      "step": 655
    },
    {
      "epoch": 0.0858428069027562,
      "grad_norm": 10.404253005981445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3394370675086975,
      "logits/rejected": 0.27484485507011414,
      "logps/chosen": -589.0396728515625,
      "logps/rejected": -583.4990234375,
      "loss": 1.0743,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.1863186359405518,
      "rewards/margins": -0.534440815448761,
      "rewards/rejected": -0.6518778204917908,
      "step": 656
    },
    {
      "epoch": 0.08597366484010796,
      "grad_norm": 5.72047233581543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43125033378601074,
      "logits/rejected": 0.4162310063838959,
      "logps/chosen": -457.3994445800781,
      "logps/rejected": -368.9095458984375,
      "loss": 0.6028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9180370569229126,
      "rewards/margins": 0.372010737657547,
      "rewards/rejected": -1.2900477647781372,
      "step": 657
    },
    {
      "epoch": 0.08610452277745972,
      "grad_norm": 5.367183685302734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4088808000087738,
      "logits/rejected": 0.2981851398944855,
      "logps/chosen": -520.4360961914062,
      "logps/rejected": -557.209716796875,
      "loss": 0.5225,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39953482151031494,
      "rewards/margins": 0.7196288108825684,
      "rewards/rejected": -1.1191635131835938,
      "step": 658
    },
    {
      "epoch": 0.08623538071481149,
      "grad_norm": 5.5511627197265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4667923152446747,
      "logits/rejected": 0.13734577596187592,
      "logps/chosen": -435.2369384765625,
      "logps/rejected": -271.68060302734375,
      "loss": 0.7368,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.7236831188201904,
      "rewards/margins": -0.00824524462223053,
      "rewards/rejected": -0.7154378294944763,
      "step": 659
    },
    {
      "epoch": 0.08636623865216325,
      "grad_norm": 7.114288330078125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3615143299102783,
      "logits/rejected": 0.05908382683992386,
      "logps/chosen": -723.4950561523438,
      "logps/rejected": -473.6373291015625,
      "loss": 0.7001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2512649595737457,
      "rewards/margins": 0.3393884301185608,
      "rewards/rejected": -0.5906534790992737,
      "step": 660
    },
    {
      "epoch": 0.086497096589515,
      "grad_norm": 6.290470123291016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6756845116615295,
      "logits/rejected": 0.6219408512115479,
      "logps/chosen": -904.8900146484375,
      "logps/rejected": -562.5648803710938,
      "loss": 0.6278,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05300043523311615,
      "rewards/margins": 0.5458244681358337,
      "rewards/rejected": -0.5988249182701111,
      "step": 661
    },
    {
      "epoch": 0.08662795452686677,
      "grad_norm": 5.341019153594971,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6793391108512878,
      "logits/rejected": 0.6455901861190796,
      "logps/chosen": -399.5241394042969,
      "logps/rejected": -390.973388671875,
      "loss": 0.5674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5903297662734985,
      "rewards/margins": 0.6872483491897583,
      "rewards/rejected": -1.2775779962539673,
      "step": 662
    },
    {
      "epoch": 0.08675881246421853,
      "grad_norm": 6.580105304718018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5424731969833374,
      "logits/rejected": 0.6513180732727051,
      "logps/chosen": -386.5048522949219,
      "logps/rejected": -403.2961730957031,
      "loss": 0.6055,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7536126971244812,
      "rewards/margins": 0.4244275689125061,
      "rewards/rejected": -1.1780403852462769,
      "step": 663
    },
    {
      "epoch": 0.0868896704015703,
      "grad_norm": 4.959917068481445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14140114188194275,
      "logits/rejected": -0.057998500764369965,
      "logps/chosen": -293.80523681640625,
      "logps/rejected": -313.69732666015625,
      "loss": 0.6073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7652215957641602,
      "rewards/margins": 0.24350221455097198,
      "rewards/rejected": -1.0087238550186157,
      "step": 664
    },
    {
      "epoch": 0.08702052833892206,
      "grad_norm": 6.403116703033447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38854795694351196,
      "logits/rejected": 0.5479158759117126,
      "logps/chosen": -405.345947265625,
      "logps/rejected": -459.8509521484375,
      "loss": 0.4912,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7371470928192139,
      "rewards/margins": 0.667815089225769,
      "rewards/rejected": -1.404962182044983,
      "step": 665
    },
    {
      "epoch": 0.08715138627627382,
      "grad_norm": 4.885306358337402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6633328795433044,
      "logits/rejected": 0.5579544305801392,
      "logps/chosen": -515.0944213867188,
      "logps/rejected": -388.9912109375,
      "loss": 0.567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7758451700210571,
      "rewards/margins": 0.5306392312049866,
      "rewards/rejected": -1.3064844608306885,
      "step": 666
    },
    {
      "epoch": 0.08728224421362558,
      "grad_norm": 4.514158248901367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3744954764842987,
      "logits/rejected": 0.3256849944591522,
      "logps/chosen": -573.0634765625,
      "logps/rejected": -463.60052490234375,
      "loss": 0.3885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4237947165966034,
      "rewards/margins": 0.8310315012931824,
      "rewards/rejected": -1.2548261880874634,
      "step": 667
    },
    {
      "epoch": 0.08741310215097735,
      "grad_norm": 8.260677337646484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30834272503852844,
      "logits/rejected": 0.29672932624816895,
      "logps/chosen": -602.3072509765625,
      "logps/rejected": -685.6792602539062,
      "loss": 0.7124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5991920828819275,
      "rewards/margins": 0.15306051075458527,
      "rewards/rejected": -0.7522526383399963,
      "step": 668
    },
    {
      "epoch": 0.08754396008832911,
      "grad_norm": 3.2809054851531982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4124307632446289,
      "logits/rejected": 0.2687782049179077,
      "logps/chosen": -440.902587890625,
      "logps/rejected": -367.9664306640625,
      "loss": 0.3132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4449581503868103,
      "rewards/margins": 1.1416338682174683,
      "rewards/rejected": -1.5865919589996338,
      "step": 669
    },
    {
      "epoch": 0.08767481802568088,
      "grad_norm": 5.063741207122803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2251700907945633,
      "logits/rejected": 0.2091866433620453,
      "logps/chosen": -533.6047973632812,
      "logps/rejected": -647.2791137695312,
      "loss": 0.5677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21623495221138,
      "rewards/margins": 0.5467533469200134,
      "rewards/rejected": -0.7629883289337158,
      "step": 670
    },
    {
      "epoch": 0.08780567596303264,
      "grad_norm": 5.932919502258301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20687761902809143,
      "logits/rejected": 0.4124317169189453,
      "logps/chosen": -518.001220703125,
      "logps/rejected": -443.7752990722656,
      "loss": 0.6785,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45147159695625305,
      "rewards/margins": 0.21600687503814697,
      "rewards/rejected": -0.6674784421920776,
      "step": 671
    },
    {
      "epoch": 0.08793653390038439,
      "grad_norm": 4.953705310821533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.361147403717041,
      "logits/rejected": 0.16338211297988892,
      "logps/chosen": -476.4145202636719,
      "logps/rejected": -436.56256103515625,
      "loss": 0.5673,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0275850296020508,
      "rewards/margins": 0.4568820595741272,
      "rewards/rejected": -1.4844671487808228,
      "step": 672
    },
    {
      "epoch": 0.08806739183773615,
      "grad_norm": 8.646240234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3508482277393341,
      "logits/rejected": 0.3608815670013428,
      "logps/chosen": -579.8150024414062,
      "logps/rejected": -372.12176513671875,
      "loss": 0.7178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7085148096084595,
      "rewards/margins": 0.3384881019592285,
      "rewards/rejected": -1.047002911567688,
      "step": 673
    },
    {
      "epoch": 0.08819824977508792,
      "grad_norm": 5.898714065551758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4258599281311035,
      "logits/rejected": 0.41100114583969116,
      "logps/chosen": -364.16351318359375,
      "logps/rejected": -339.337646484375,
      "loss": 0.5851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4903780221939087,
      "rewards/margins": 0.3478716015815735,
      "rewards/rejected": -0.8382495641708374,
      "step": 674
    },
    {
      "epoch": 0.08832910771243968,
      "grad_norm": 7.617144584655762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.524128794670105,
      "logits/rejected": 0.4225170612335205,
      "logps/chosen": -424.6582336425781,
      "logps/rejected": -356.3681945800781,
      "loss": 0.6773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7181860208511353,
      "rewards/margins": 0.3044971823692322,
      "rewards/rejected": -1.0226831436157227,
      "step": 675
    },
    {
      "epoch": 0.08845996564979144,
      "grad_norm": 5.815899848937988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4003332853317261,
      "logits/rejected": 0.21186040341854095,
      "logps/chosen": -509.03656005859375,
      "logps/rejected": -509.53167724609375,
      "loss": 0.5857,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4704548418521881,
      "rewards/margins": 0.38011589646339417,
      "rewards/rejected": -0.8505707383155823,
      "step": 676
    },
    {
      "epoch": 0.08859082358714321,
      "grad_norm": 5.689420223236084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31875935196876526,
      "logits/rejected": 0.3333325982093811,
      "logps/chosen": -660.153564453125,
      "logps/rejected": -475.7791748046875,
      "loss": 0.5815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03772564232349396,
      "rewards/margins": 0.38848018646240234,
      "rewards/rejected": -0.4262058436870575,
      "step": 677
    },
    {
      "epoch": 0.08872168152449497,
      "grad_norm": 6.1490559577941895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0994575172662735,
      "logits/rejected": 0.14257478713989258,
      "logps/chosen": -290.0334777832031,
      "logps/rejected": -349.9130859375,
      "loss": 0.7279,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9287548661231995,
      "rewards/margins": 0.16723859310150146,
      "rewards/rejected": -1.0959935188293457,
      "step": 678
    },
    {
      "epoch": 0.08885253946184674,
      "grad_norm": 5.607381820678711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4614053964614868,
      "logits/rejected": 0.5171580910682678,
      "logps/chosen": -428.7187805175781,
      "logps/rejected": -364.3254089355469,
      "loss": 0.6394,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4923093914985657,
      "rewards/margins": 0.3522724211215973,
      "rewards/rejected": -0.8445818424224854,
      "step": 679
    },
    {
      "epoch": 0.0889833973991985,
      "grad_norm": 4.495773792266846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1270691454410553,
      "logits/rejected": 0.3332135081291199,
      "logps/chosen": -400.4246826171875,
      "logps/rejected": -433.61572265625,
      "loss": 0.441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4035015404224396,
      "rewards/margins": 0.7766963839530945,
      "rewards/rejected": -1.1801979541778564,
      "step": 680
    },
    {
      "epoch": 0.08911425533655026,
      "grad_norm": 4.73716926574707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6389967203140259,
      "logits/rejected": 0.7016500234603882,
      "logps/chosen": -415.7015380859375,
      "logps/rejected": -329.39300537109375,
      "loss": 0.5504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.507201611995697,
      "rewards/margins": 0.39069807529449463,
      "rewards/rejected": -0.8978996276855469,
      "step": 681
    },
    {
      "epoch": 0.08924511327390203,
      "grad_norm": 4.430295944213867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3542560935020447,
      "logits/rejected": 0.0981588214635849,
      "logps/chosen": -498.51776123046875,
      "logps/rejected": -642.53857421875,
      "loss": 0.2649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5997048616409302,
      "rewards/margins": 1.3047055006027222,
      "rewards/rejected": -1.9044102430343628,
      "step": 682
    },
    {
      "epoch": 0.08937597121125378,
      "grad_norm": 5.871896266937256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38998857140541077,
      "logits/rejected": 0.34276363253593445,
      "logps/chosen": -511.8193664550781,
      "logps/rejected": -394.195556640625,
      "loss": 0.6356,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48264676332473755,
      "rewards/margins": 0.2977987229824066,
      "rewards/rejected": -0.7804455161094666,
      "step": 683
    },
    {
      "epoch": 0.08950682914860554,
      "grad_norm": 6.565781593322754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3073452115058899,
      "logits/rejected": 0.2754797339439392,
      "logps/chosen": -377.7909240722656,
      "logps/rejected": -426.5263671875,
      "loss": 0.5927,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4958367943763733,
      "rewards/margins": 0.3204280734062195,
      "rewards/rejected": -0.8162649869918823,
      "step": 684
    },
    {
      "epoch": 0.0896376870859573,
      "grad_norm": 6.483433723449707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27145084738731384,
      "logits/rejected": 0.2904900908470154,
      "logps/chosen": -313.4565124511719,
      "logps/rejected": -386.46246337890625,
      "loss": 0.7212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38846150040626526,
      "rewards/margins": 0.39963263273239136,
      "rewards/rejected": -0.7880941033363342,
      "step": 685
    },
    {
      "epoch": 0.08976854502330907,
      "grad_norm": 4.078463554382324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45291152596473694,
      "logits/rejected": 0.5046011805534363,
      "logps/chosen": -540.7127685546875,
      "logps/rejected": -374.11737060546875,
      "loss": 0.4602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2542875111103058,
      "rewards/margins": 0.8462235927581787,
      "rewards/rejected": -1.1005111932754517,
      "step": 686
    },
    {
      "epoch": 0.08989940296066083,
      "grad_norm": 4.2256550788879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.678504228591919,
      "logits/rejected": 0.4000365138053894,
      "logps/chosen": -427.9520263671875,
      "logps/rejected": -317.6056823730469,
      "loss": 0.4215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08184738457202911,
      "rewards/margins": 1.1422345638275146,
      "rewards/rejected": -1.2240817546844482,
      "step": 687
    },
    {
      "epoch": 0.0900302608980126,
      "grad_norm": 5.453498363494873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3498777151107788,
      "logits/rejected": 0.12104902416467667,
      "logps/chosen": -441.67724609375,
      "logps/rejected": -368.92529296875,
      "loss": 0.5443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7615394592285156,
      "rewards/margins": 0.45852091908454895,
      "rewards/rejected": -1.2200604677200317,
      "step": 688
    },
    {
      "epoch": 0.09016111883536436,
      "grad_norm": 8.271175384521484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.236581489443779,
      "logits/rejected": 0.4207648038864136,
      "logps/chosen": -642.1660766601562,
      "logps/rejected": -602.2376708984375,
      "loss": 0.7409,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1577293425798416,
      "rewards/margins": 0.17028865218162537,
      "rewards/rejected": -0.3280179798603058,
      "step": 689
    },
    {
      "epoch": 0.09029197677271612,
      "grad_norm": 6.2503743171691895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4035411477088928,
      "logits/rejected": 0.3844783306121826,
      "logps/chosen": -650.7162475585938,
      "logps/rejected": -398.2698974609375,
      "loss": 0.502,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04958438128232956,
      "rewards/margins": 0.5381544232368469,
      "rewards/rejected": -0.48857006430625916,
      "step": 690
    },
    {
      "epoch": 0.09042283471006789,
      "grad_norm": 10.498993873596191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4546740651130676,
      "logits/rejected": 0.2899254262447357,
      "logps/chosen": -725.5247802734375,
      "logps/rejected": -611.4658203125,
      "loss": 1.281,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8555446267127991,
      "rewards/margins": -0.5752202272415161,
      "rewards/rejected": -0.28032436966896057,
      "step": 691
    },
    {
      "epoch": 0.09055369264741965,
      "grad_norm": 5.8158345222473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029390543699264526,
      "logits/rejected": 0.2581597566604614,
      "logps/chosen": -447.90460205078125,
      "logps/rejected": -438.8733215332031,
      "loss": 0.5414,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21623137593269348,
      "rewards/margins": 0.43165168166160583,
      "rewards/rejected": -0.6478830575942993,
      "step": 692
    },
    {
      "epoch": 0.0906845505847714,
      "grad_norm": 4.840320110321045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7654616832733154,
      "logits/rejected": 0.791343092918396,
      "logps/chosen": -360.0086364746094,
      "logps/rejected": -339.3619384765625,
      "loss": 0.6585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5865045785903931,
      "rewards/margins": 0.17922398447990417,
      "rewards/rejected": -0.7657285928726196,
      "step": 693
    },
    {
      "epoch": 0.09081540852212316,
      "grad_norm": 5.131875991821289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38122525811195374,
      "logits/rejected": 0.49827876687049866,
      "logps/chosen": -364.87738037109375,
      "logps/rejected": -397.9121398925781,
      "loss": 0.7036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6838744878768921,
      "rewards/margins": 0.23398303985595703,
      "rewards/rejected": -0.9178575277328491,
      "step": 694
    },
    {
      "epoch": 0.09094626645947493,
      "grad_norm": 4.801453590393066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3369833827018738,
      "logits/rejected": 0.012417629361152649,
      "logps/chosen": -351.8399658203125,
      "logps/rejected": -284.795166015625,
      "loss": 0.6113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7120378613471985,
      "rewards/margins": 0.22286148369312286,
      "rewards/rejected": -0.9348993301391602,
      "step": 695
    },
    {
      "epoch": 0.09107712439682669,
      "grad_norm": 6.298206806182861,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1869867742061615,
      "logits/rejected": 0.1572311818599701,
      "logps/chosen": -509.18743896484375,
      "logps/rejected": -510.8410339355469,
      "loss": 0.5311,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6018692255020142,
      "rewards/margins": 0.5578792691230774,
      "rewards/rejected": -1.1597485542297363,
      "step": 696
    },
    {
      "epoch": 0.09120798233417846,
      "grad_norm": 5.848387718200684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49746978282928467,
      "logits/rejected": 0.2994571626186371,
      "logps/chosen": -562.6471557617188,
      "logps/rejected": -463.68511962890625,
      "loss": 0.5948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6872402429580688,
      "rewards/margins": 0.5358637571334839,
      "rewards/rejected": -1.2231038808822632,
      "step": 697
    },
    {
      "epoch": 0.09133884027153022,
      "grad_norm": 2.95072865486145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6229607462882996,
      "logits/rejected": 0.48109424114227295,
      "logps/chosen": -455.65093994140625,
      "logps/rejected": -375.406005859375,
      "loss": 0.2944,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35089442133903503,
      "rewards/margins": 1.5434091091156006,
      "rewards/rejected": -1.1925147771835327,
      "step": 698
    },
    {
      "epoch": 0.09146969820888198,
      "grad_norm": 5.027590751647949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6697076559066772,
      "logits/rejected": 0.38720080256462097,
      "logps/chosen": -446.86138916015625,
      "logps/rejected": -434.0693054199219,
      "loss": 0.5333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09631841629743576,
      "rewards/margins": 0.7538261413574219,
      "rewards/rejected": -0.6575076580047607,
      "step": 699
    },
    {
      "epoch": 0.09160055614623375,
      "grad_norm": 4.886260509490967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5516397356987,
      "logits/rejected": 0.60331130027771,
      "logps/chosen": -457.10626220703125,
      "logps/rejected": -456.30511474609375,
      "loss": 0.5427,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7733684778213501,
      "rewards/margins": 0.6157127022743225,
      "rewards/rejected": -1.3890811204910278,
      "step": 700
    },
    {
      "epoch": 0.09173141408358551,
      "grad_norm": 5.765064239501953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16716322302818298,
      "logits/rejected": 0.3202767074108124,
      "logps/chosen": -361.994140625,
      "logps/rejected": -407.7281188964844,
      "loss": 0.5122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04623183608055115,
      "rewards/margins": 0.4660770297050476,
      "rewards/rejected": -0.41984519362449646,
      "step": 701
    },
    {
      "epoch": 0.09186227202093727,
      "grad_norm": 4.900849342346191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2728828489780426,
      "logits/rejected": 0.35790273547172546,
      "logps/chosen": -363.37017822265625,
      "logps/rejected": -519.22509765625,
      "loss": 0.5358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40062353014945984,
      "rewards/margins": 0.5361566543579102,
      "rewards/rejected": -0.9367800951004028,
      "step": 702
    },
    {
      "epoch": 0.09199312995828904,
      "grad_norm": 5.349212646484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.516849935054779,
      "logits/rejected": 0.31617969274520874,
      "logps/chosen": -613.0821533203125,
      "logps/rejected": -470.55181884765625,
      "loss": 0.4747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1095491424202919,
      "rewards/margins": 0.788235068321228,
      "rewards/rejected": -0.8977842926979065,
      "step": 703
    },
    {
      "epoch": 0.09212398789564079,
      "grad_norm": 5.699394702911377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3440055847167969,
      "logits/rejected": -0.057627350091934204,
      "logps/chosen": -457.24609375,
      "logps/rejected": -413.7205505371094,
      "loss": 0.5817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26021480560302734,
      "rewards/margins": 0.25861722230911255,
      "rewards/rejected": -0.5188320279121399,
      "step": 704
    },
    {
      "epoch": 0.09225484583299255,
      "grad_norm": 4.621347904205322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4841787815093994,
      "logits/rejected": 0.6172223687171936,
      "logps/chosen": -594.6505737304688,
      "logps/rejected": -496.143310546875,
      "loss": 0.4448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17519760131835938,
      "rewards/margins": 1.0790131092071533,
      "rewards/rejected": -1.2542107105255127,
      "step": 705
    },
    {
      "epoch": 0.09238570377034432,
      "grad_norm": 5.211137294769287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1881594955921173,
      "logits/rejected": 0.17436987161636353,
      "logps/chosen": -317.1043701171875,
      "logps/rejected": -440.01446533203125,
      "loss": 0.4851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12442588061094284,
      "rewards/margins": 0.6715377569198608,
      "rewards/rejected": -0.5471119284629822,
      "step": 706
    },
    {
      "epoch": 0.09251656170769608,
      "grad_norm": 8.145364761352539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49837952852249146,
      "logits/rejected": 0.38478851318359375,
      "logps/chosen": -463.6714782714844,
      "logps/rejected": -507.5233459472656,
      "loss": 0.7399,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8187200427055359,
      "rewards/margins": 0.103324294090271,
      "rewards/rejected": -0.9220443964004517,
      "step": 707
    },
    {
      "epoch": 0.09264741964504784,
      "grad_norm": 6.357842922210693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21192201972007751,
      "logits/rejected": 0.2205989956855774,
      "logps/chosen": -432.5211486816406,
      "logps/rejected": -426.5934143066406,
      "loss": 0.7671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24151268601417542,
      "rewards/margins": 0.2548738121986389,
      "rewards/rejected": -0.4963865280151367,
      "step": 708
    },
    {
      "epoch": 0.09277827758239961,
      "grad_norm": 4.6807122230529785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05790191888809204,
      "logits/rejected": 0.20387813448905945,
      "logps/chosen": -338.33660888671875,
      "logps/rejected": -345.0587158203125,
      "loss": 0.5348,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42138445377349854,
      "rewards/margins": 0.4464859366416931,
      "rewards/rejected": -0.8678703308105469,
      "step": 709
    },
    {
      "epoch": 0.09290913551975137,
      "grad_norm": 7.265817165374756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28343096375465393,
      "logits/rejected": 0.21311834454536438,
      "logps/chosen": -447.14935302734375,
      "logps/rejected": -420.703369140625,
      "loss": 0.8372,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5584539771080017,
      "rewards/margins": -0.2261360138654709,
      "rewards/rejected": -0.33231791853904724,
      "step": 710
    },
    {
      "epoch": 0.09303999345710313,
      "grad_norm": 4.174752712249756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12575805187225342,
      "logits/rejected": 0.11718851327896118,
      "logps/chosen": -398.8992919921875,
      "logps/rejected": -483.78692626953125,
      "loss": 0.3696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24066467583179474,
      "rewards/margins": 1.006359577178955,
      "rewards/rejected": -1.2470242977142334,
      "step": 711
    },
    {
      "epoch": 0.0931708513944549,
      "grad_norm": 17.68074607849121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26424288749694824,
      "logits/rejected": 0.20246174931526184,
      "logps/chosen": -700.61865234375,
      "logps/rejected": -571.3438720703125,
      "loss": 0.7436,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.401022344827652,
      "rewards/margins": 0.11990111321210861,
      "rewards/rejected": -0.5209234356880188,
      "step": 712
    },
    {
      "epoch": 0.09330170933180666,
      "grad_norm": 4.146368503570557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2979326844215393,
      "logits/rejected": 0.33024728298187256,
      "logps/chosen": -513.2257690429688,
      "logps/rejected": -550.67333984375,
      "loss": 0.5215,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3072279095649719,
      "rewards/margins": 0.7806205749511719,
      "rewards/rejected": -1.087848424911499,
      "step": 713
    },
    {
      "epoch": 0.09343256726915843,
      "grad_norm": 4.484841346740723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5831443667411804,
      "logits/rejected": 0.4406909942626953,
      "logps/chosen": -434.05596923828125,
      "logps/rejected": -352.36822509765625,
      "loss": 0.4498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3125844895839691,
      "rewards/margins": 0.8303918838500977,
      "rewards/rejected": -1.1429764032363892,
      "step": 714
    },
    {
      "epoch": 0.09356342520651018,
      "grad_norm": 5.782400608062744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5254899859428406,
      "logits/rejected": 0.5070971250534058,
      "logps/chosen": -463.85601806640625,
      "logps/rejected": -363.9640197753906,
      "loss": 0.5916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8995311856269836,
      "rewards/margins": 0.37444424629211426,
      "rewards/rejected": -1.2739754915237427,
      "step": 715
    },
    {
      "epoch": 0.09369428314386194,
      "grad_norm": 6.157803058624268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5091860294342041,
      "logits/rejected": 0.5529441833496094,
      "logps/chosen": -576.4866943359375,
      "logps/rejected": -552.9181518554688,
      "loss": 0.4478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13679847121238708,
      "rewards/margins": 0.6968666315078735,
      "rewards/rejected": -0.8336650729179382,
      "step": 716
    },
    {
      "epoch": 0.0938251410812137,
      "grad_norm": 7.684889793395996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3135860562324524,
      "logits/rejected": 0.16123907268047333,
      "logps/chosen": -476.4811096191406,
      "logps/rejected": -360.2386779785156,
      "loss": 0.6753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.528555691242218,
      "rewards/margins": 0.1358024775981903,
      "rewards/rejected": -0.6643581390380859,
      "step": 717
    },
    {
      "epoch": 0.09395599901856547,
      "grad_norm": 3.8076887130737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30905240774154663,
      "logits/rejected": 0.3050799071788788,
      "logps/chosen": -343.9082946777344,
      "logps/rejected": -357.0591125488281,
      "loss": 0.4086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07847174257040024,
      "rewards/margins": 0.803805947303772,
      "rewards/rejected": -0.8822776675224304,
      "step": 718
    },
    {
      "epoch": 0.09408685695591723,
      "grad_norm": 4.956506729125977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0033862069249153137,
      "logits/rejected": 0.065555140376091,
      "logps/chosen": -236.41653442382812,
      "logps/rejected": -338.99725341796875,
      "loss": 0.6176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.875117301940918,
      "rewards/margins": 0.4713646471500397,
      "rewards/rejected": -1.3464818000793457,
      "step": 719
    },
    {
      "epoch": 0.094217714893269,
      "grad_norm": 4.3567728996276855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2619630992412567,
      "logits/rejected": 0.32240626215934753,
      "logps/chosen": -429.2402648925781,
      "logps/rejected": -434.2297058105469,
      "loss": 0.5376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7018824219703674,
      "rewards/margins": 0.4577717185020447,
      "rewards/rejected": -1.1596540212631226,
      "step": 720
    },
    {
      "epoch": 0.09434857283062076,
      "grad_norm": 6.358911514282227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42201340198516846,
      "logits/rejected": 0.48083484172821045,
      "logps/chosen": -575.83642578125,
      "logps/rejected": -518.6596069335938,
      "loss": 0.6084,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4036830961704254,
      "rewards/margins": 0.3965940773487091,
      "rewards/rejected": -0.8002771139144897,
      "step": 721
    },
    {
      "epoch": 0.09447943076797252,
      "grad_norm": 5.93036413192749,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24424347281455994,
      "logits/rejected": 0.2827465832233429,
      "logps/chosen": -393.9485168457031,
      "logps/rejected": -477.8402099609375,
      "loss": 0.5436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.541002631187439,
      "rewards/margins": 0.48609912395477295,
      "rewards/rejected": -1.0271018743515015,
      "step": 722
    },
    {
      "epoch": 0.09461028870532429,
      "grad_norm": 5.719703197479248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40970340371131897,
      "logits/rejected": 0.49472662806510925,
      "logps/chosen": -411.48883056640625,
      "logps/rejected": -474.2031555175781,
      "loss": 0.689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5473560690879822,
      "rewards/margins": 0.2157333344221115,
      "rewards/rejected": -0.7630893588066101,
      "step": 723
    },
    {
      "epoch": 0.09474114664267605,
      "grad_norm": 35.26607131958008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24475917220115662,
      "logits/rejected": 0.27574530243873596,
      "logps/chosen": -467.6706848144531,
      "logps/rejected": -596.1629028320312,
      "loss": 0.9774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09422071278095245,
      "rewards/margins": 0.12025946378707886,
      "rewards/rejected": -0.2144802361726761,
      "step": 724
    },
    {
      "epoch": 0.09487200458002781,
      "grad_norm": 7.00619649887085,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4162127673625946,
      "logits/rejected": 0.3132724463939667,
      "logps/chosen": -399.76971435546875,
      "logps/rejected": -601.70458984375,
      "loss": 0.6621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6222696304321289,
      "rewards/margins": 0.09848631918430328,
      "rewards/rejected": -0.720755934715271,
      "step": 725
    },
    {
      "epoch": 0.09500286251737956,
      "grad_norm": 5.1745476722717285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.455182820558548,
      "logits/rejected": 0.3177340626716614,
      "logps/chosen": -318.18878173828125,
      "logps/rejected": -421.3782043457031,
      "loss": 0.6211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.578618586063385,
      "rewards/margins": 0.2212318480014801,
      "rewards/rejected": -0.7998504638671875,
      "step": 726
    },
    {
      "epoch": 0.09513372045473133,
      "grad_norm": 5.8577165603637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0006784405559301376,
      "logits/rejected": 0.03923714533448219,
      "logps/chosen": -426.30413818359375,
      "logps/rejected": -377.482421875,
      "loss": 0.5642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7115553021430969,
      "rewards/margins": 0.4560468792915344,
      "rewards/rejected": -1.1676021814346313,
      "step": 727
    },
    {
      "epoch": 0.09526457839208309,
      "grad_norm": 7.495192527770996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47538426518440247,
      "logits/rejected": 0.3402881622314453,
      "logps/chosen": -664.6676635742188,
      "logps/rejected": -511.05926513671875,
      "loss": 0.6362,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.457567036151886,
      "rewards/margins": 0.37214046716690063,
      "rewards/rejected": -0.8297076225280762,
      "step": 728
    },
    {
      "epoch": 0.09539543632943485,
      "grad_norm": 5.183895111083984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7880237102508545,
      "logits/rejected": 0.7579085826873779,
      "logps/chosen": -449.4179382324219,
      "logps/rejected": -544.6536865234375,
      "loss": 0.5756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7120449542999268,
      "rewards/margins": 0.3772873282432556,
      "rewards/rejected": -1.0893322229385376,
      "step": 729
    },
    {
      "epoch": 0.09552629426678662,
      "grad_norm": 7.168574810028076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3563985526561737,
      "logits/rejected": 0.252963125705719,
      "logps/chosen": -418.9479675292969,
      "logps/rejected": -511.24896240234375,
      "loss": 0.6879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6935625672340393,
      "rewards/margins": 0.3186931312084198,
      "rewards/rejected": -1.0122556686401367,
      "step": 730
    },
    {
      "epoch": 0.09565715220413838,
      "grad_norm": 5.376876354217529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3913496434688568,
      "logits/rejected": 0.5066252946853638,
      "logps/chosen": -477.46527099609375,
      "logps/rejected": -468.25872802734375,
      "loss": 0.5909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3211313486099243,
      "rewards/margins": 0.4859548509120941,
      "rewards/rejected": -0.807086169719696,
      "step": 731
    },
    {
      "epoch": 0.09578801014149015,
      "grad_norm": 4.892544269561768,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2845458984375,
      "logits/rejected": 0.251335471868515,
      "logps/chosen": -601.7391357421875,
      "logps/rejected": -489.1092529296875,
      "loss": 0.5224,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27409762144088745,
      "rewards/margins": 0.6617116332054138,
      "rewards/rejected": -0.935809314250946,
      "step": 732
    },
    {
      "epoch": 0.09591886807884191,
      "grad_norm": 6.289251327514648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25762611627578735,
      "logits/rejected": 0.27690982818603516,
      "logps/chosen": -542.6485595703125,
      "logps/rejected": -379.2081298828125,
      "loss": 0.634,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12061843276023865,
      "rewards/margins": 0.4022018313407898,
      "rewards/rejected": -0.5228202939033508,
      "step": 733
    },
    {
      "epoch": 0.09604972601619367,
      "grad_norm": 8.416934967041016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4046989679336548,
      "logits/rejected": 0.3348032832145691,
      "logps/chosen": -423.3009948730469,
      "logps/rejected": -450.2203063964844,
      "loss": 0.9994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.387486457824707,
      "rewards/margins": -0.07973003387451172,
      "rewards/rejected": -1.3077564239501953,
      "step": 734
    },
    {
      "epoch": 0.09618058395354544,
      "grad_norm": 5.38561487197876,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4302466809749603,
      "logits/rejected": 0.3014068603515625,
      "logps/chosen": -591.8191528320312,
      "logps/rejected": -574.873779296875,
      "loss": 0.5134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2495395690202713,
      "rewards/margins": 0.6425756812095642,
      "rewards/rejected": -0.8921152353286743,
      "step": 735
    },
    {
      "epoch": 0.0963114418908972,
      "grad_norm": 6.409686088562012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.572711706161499,
      "logits/rejected": 0.4219312071800232,
      "logps/chosen": -420.46063232421875,
      "logps/rejected": -585.4873046875,
      "loss": 0.6797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8553317189216614,
      "rewards/margins": 0.24923019111156464,
      "rewards/rejected": -1.1045618057250977,
      "step": 736
    },
    {
      "epoch": 0.09644229982824895,
      "grad_norm": 7.7154107093811035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3622705340385437,
      "logits/rejected": 0.08279823511838913,
      "logps/chosen": -695.7171630859375,
      "logps/rejected": -585.0291137695312,
      "loss": 0.5222,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7435142397880554,
      "rewards/margins": 0.5920761823654175,
      "rewards/rejected": -1.3355903625488281,
      "step": 737
    },
    {
      "epoch": 0.09657315776560071,
      "grad_norm": 5.817506790161133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34370243549346924,
      "logits/rejected": 0.3484077453613281,
      "logps/chosen": -629.9061889648438,
      "logps/rejected": -635.1419677734375,
      "loss": 0.6074,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7665817737579346,
      "rewards/margins": 0.45024940371513367,
      "rewards/rejected": -1.2168312072753906,
      "step": 738
    },
    {
      "epoch": 0.09670401570295248,
      "grad_norm": 4.012712001800537,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30169931054115295,
      "logits/rejected": 0.0887298434972763,
      "logps/chosen": -331.7363586425781,
      "logps/rejected": -512.1900634765625,
      "loss": 0.3005,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3988227844238281,
      "rewards/margins": 1.3877284526824951,
      "rewards/rejected": -1.7865512371063232,
      "step": 739
    },
    {
      "epoch": 0.09683487364030424,
      "grad_norm": 7.168163776397705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24476656317710876,
      "logits/rejected": 0.07939161360263824,
      "logps/chosen": -433.1288757324219,
      "logps/rejected": -363.2433166503906,
      "loss": 0.7868,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6422609686851501,
      "rewards/margins": 0.07038476318120956,
      "rewards/rejected": -0.7126456499099731,
      "step": 740
    },
    {
      "epoch": 0.096965731577656,
      "grad_norm": 4.821505546569824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6480488181114197,
      "logits/rejected": 0.652218222618103,
      "logps/chosen": -506.64617919921875,
      "logps/rejected": -453.8896484375,
      "loss": 0.4162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7364563345909119,
      "rewards/margins": 1.2410509586334229,
      "rewards/rejected": -1.9775071144104004,
      "step": 741
    },
    {
      "epoch": 0.09709658951500777,
      "grad_norm": 4.984346866607666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2722756862640381,
      "logits/rejected": -0.10644717514514923,
      "logps/chosen": -671.54345703125,
      "logps/rejected": -502.6934509277344,
      "loss": 0.3961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20221573114395142,
      "rewards/margins": 1.026736855506897,
      "rewards/rejected": -0.8245211243629456,
      "step": 742
    },
    {
      "epoch": 0.09722744745235953,
      "grad_norm": 6.984718322753906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6856338977813721,
      "logits/rejected": 0.7038280963897705,
      "logps/chosen": -622.9219970703125,
      "logps/rejected": -625.745849609375,
      "loss": 0.5658,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24274294078350067,
      "rewards/margins": 0.5907387733459473,
      "rewards/rejected": -0.8334816098213196,
      "step": 743
    },
    {
      "epoch": 0.0973583053897113,
      "grad_norm": 6.77297306060791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2076246738433838,
      "logits/rejected": 0.010534979403018951,
      "logps/chosen": -542.9813232421875,
      "logps/rejected": -528.2823486328125,
      "loss": 0.566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7399309277534485,
      "rewards/margins": 0.5207563638687134,
      "rewards/rejected": -1.260687232017517,
      "step": 744
    },
    {
      "epoch": 0.09748916332706306,
      "grad_norm": 6.118258953094482,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39510583877563477,
      "logits/rejected": 0.05695011094212532,
      "logps/chosen": -395.60089111328125,
      "logps/rejected": -334.6087341308594,
      "loss": 0.5752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5834883451461792,
      "rewards/margins": 0.4727184772491455,
      "rewards/rejected": -1.0562067031860352,
      "step": 745
    },
    {
      "epoch": 0.09762002126441482,
      "grad_norm": 5.314411640167236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3352448046207428,
      "logits/rejected": 0.32690879702568054,
      "logps/chosen": -551.06494140625,
      "logps/rejected": -412.1207275390625,
      "loss": 0.5485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7951112985610962,
      "rewards/margins": 0.4076307415962219,
      "rewards/rejected": -1.202742099761963,
      "step": 746
    },
    {
      "epoch": 0.09775087920176659,
      "grad_norm": 5.238245487213135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2843049168586731,
      "logits/rejected": 0.28803980350494385,
      "logps/chosen": -385.2016906738281,
      "logps/rejected": -413.3121337890625,
      "loss": 0.4383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5163307189941406,
      "rewards/margins": 0.7260928750038147,
      "rewards/rejected": -1.2424235343933105,
      "step": 747
    },
    {
      "epoch": 0.09788173713911834,
      "grad_norm": 8.189615249633789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.341664582490921,
      "logits/rejected": 0.3382870554924011,
      "logps/chosen": -620.43115234375,
      "logps/rejected": -522.9760131835938,
      "loss": 0.9264,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.47908347845077515,
      "rewards/margins": -0.008194521069526672,
      "rewards/rejected": -0.4708889424800873,
      "step": 748
    },
    {
      "epoch": 0.0980125950764701,
      "grad_norm": 6.881958484649658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2747799754142761,
      "logits/rejected": 0.23536628484725952,
      "logps/chosen": -386.7587585449219,
      "logps/rejected": -397.20269775390625,
      "loss": 0.5882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6694033145904541,
      "rewards/margins": 0.6424713134765625,
      "rewards/rejected": -1.3118746280670166,
      "step": 749
    },
    {
      "epoch": 0.09814345301382187,
      "grad_norm": 10.033164978027344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14282411336898804,
      "logits/rejected": 0.12617981433868408,
      "logps/chosen": -430.48046875,
      "logps/rejected": -543.062744140625,
      "loss": 0.8811,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8861360549926758,
      "rewards/margins": 0.3007749915122986,
      "rewards/rejected": -2.186911106109619,
      "step": 750
    },
    {
      "epoch": 0.09827431095117363,
      "grad_norm": 6.23040771484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1229141429066658,
      "logits/rejected": 0.10783844441175461,
      "logps/chosen": -376.1332702636719,
      "logps/rejected": -495.6929626464844,
      "loss": 0.635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6869258880615234,
      "rewards/margins": 0.2944602966308594,
      "rewards/rejected": -0.9813861846923828,
      "step": 751
    },
    {
      "epoch": 0.0984051688885254,
      "grad_norm": 5.600058078765869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5279306173324585,
      "logits/rejected": 0.38659951090812683,
      "logps/chosen": -497.5005187988281,
      "logps/rejected": -508.467041015625,
      "loss": 0.673,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6567050814628601,
      "rewards/margins": 0.22152023017406464,
      "rewards/rejected": -0.8782253265380859,
      "step": 752
    },
    {
      "epoch": 0.09853602682587716,
      "grad_norm": 5.544954299926758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47117605805397034,
      "logits/rejected": 0.5399964451789856,
      "logps/chosen": -439.234375,
      "logps/rejected": -344.13397216796875,
      "loss": 0.6885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7269167304039001,
      "rewards/margins": 0.232464998960495,
      "rewards/rejected": -0.959381639957428,
      "step": 753
    },
    {
      "epoch": 0.09866688476322892,
      "grad_norm": 5.215688705444336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6901030540466309,
      "logits/rejected": 0.5746413469314575,
      "logps/chosen": -370.62896728515625,
      "logps/rejected": -390.2222900390625,
      "loss": 0.6092,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4469871520996094,
      "rewards/margins": 0.3194984793663025,
      "rewards/rejected": -0.7664856314659119,
      "step": 754
    },
    {
      "epoch": 0.09879774270058068,
      "grad_norm": 5.4214301109313965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19518765807151794,
      "logits/rejected": 0.2898768186569214,
      "logps/chosen": -400.16033935546875,
      "logps/rejected": -456.77044677734375,
      "loss": 0.662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7116226553916931,
      "rewards/margins": 0.22403183579444885,
      "rewards/rejected": -0.9356545209884644,
      "step": 755
    },
    {
      "epoch": 0.09892860063793245,
      "grad_norm": 4.205509662628174,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5736894607543945,
      "logits/rejected": 0.4200748801231384,
      "logps/chosen": -495.0681457519531,
      "logps/rejected": -349.94256591796875,
      "loss": 0.4752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49937573075294495,
      "rewards/margins": 0.5885734558105469,
      "rewards/rejected": -1.087949275970459,
      "step": 756
    },
    {
      "epoch": 0.09905945857528421,
      "grad_norm": 5.840367317199707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37269434332847595,
      "logits/rejected": 0.35486531257629395,
      "logps/chosen": -546.1258544921875,
      "logps/rejected": -660.5294189453125,
      "loss": 0.5414,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7491308450698853,
      "rewards/margins": 0.5783014297485352,
      "rewards/rejected": -1.3274322748184204,
      "step": 757
    },
    {
      "epoch": 0.09919031651263598,
      "grad_norm": 6.800898551940918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46448636054992676,
      "logits/rejected": 0.44211840629577637,
      "logps/chosen": -458.805419921875,
      "logps/rejected": -464.3771057128906,
      "loss": 0.6835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06836432963609695,
      "rewards/margins": 0.28143519163131714,
      "rewards/rejected": -0.2130708545446396,
      "step": 758
    },
    {
      "epoch": 0.09932117444998773,
      "grad_norm": 5.750589847564697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4237598776817322,
      "logits/rejected": 0.3915828466415405,
      "logps/chosen": -384.5552062988281,
      "logps/rejected": -460.7048034667969,
      "loss": 0.4604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.625289797782898,
      "rewards/margins": 0.689963161945343,
      "rewards/rejected": -1.3152529001235962,
      "step": 759
    },
    {
      "epoch": 0.09945203238733949,
      "grad_norm": 7.797125816345215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4517671465873718,
      "logits/rejected": 0.4950829744338989,
      "logps/chosen": -510.16162109375,
      "logps/rejected": -431.1383972167969,
      "loss": 0.8825,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5008888244628906,
      "rewards/margins": 0.05769234895706177,
      "rewards/rejected": -0.5585811734199524,
      "step": 760
    },
    {
      "epoch": 0.09958289032469125,
      "grad_norm": 10.06049919128418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3902912437915802,
      "logits/rejected": 0.428231805562973,
      "logps/chosen": -636.9656372070312,
      "logps/rejected": -383.3055419921875,
      "loss": 0.743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9311472177505493,
      "rewards/margins": 0.928984522819519,
      "rewards/rejected": -1.860131859779358,
      "step": 761
    },
    {
      "epoch": 0.09971374826204302,
      "grad_norm": 4.629703998565674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2109987437725067,
      "logits/rejected": 0.12412016093730927,
      "logps/chosen": -325.5423278808594,
      "logps/rejected": -336.07867431640625,
      "loss": 0.5451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8182058334350586,
      "rewards/margins": 0.4848289489746094,
      "rewards/rejected": -1.303034782409668,
      "step": 762
    },
    {
      "epoch": 0.09984460619939478,
      "grad_norm": 5.291471481323242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5739395618438721,
      "logits/rejected": 0.3545410633087158,
      "logps/chosen": -621.913818359375,
      "logps/rejected": -498.19256591796875,
      "loss": 0.4264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45912647247314453,
      "rewards/margins": 0.7982765436172485,
      "rewards/rejected": -1.257403016090393,
      "step": 763
    },
    {
      "epoch": 0.09997546413674654,
      "grad_norm": 3.8248555660247803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43426352739334106,
      "logits/rejected": 0.5681473016738892,
      "logps/chosen": -403.57781982421875,
      "logps/rejected": -391.4416198730469,
      "loss": 0.3634,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09058970957994461,
      "rewards/margins": 1.1817400455474854,
      "rewards/rejected": -1.2723298072814941,
      "step": 764
    },
    {
      "epoch": 0.10010632207409831,
      "grad_norm": 5.202819347381592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44390565156936646,
      "logits/rejected": 0.565950870513916,
      "logps/chosen": -229.19271850585938,
      "logps/rejected": -276.37725830078125,
      "loss": 0.7363,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7530701160430908,
      "rewards/margins": -0.039661016315221786,
      "rewards/rejected": -0.7134090662002563,
      "step": 765
    },
    {
      "epoch": 0.10023718001145007,
      "grad_norm": 5.713364601135254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4942231774330139,
      "logits/rejected": 0.5025256872177124,
      "logps/chosen": -630.1362915039062,
      "logps/rejected": -648.3995361328125,
      "loss": 0.5447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.303410142660141,
      "rewards/margins": 0.4511316418647766,
      "rewards/rejected": -0.75454181432724,
      "step": 766
    },
    {
      "epoch": 0.10036803794880184,
      "grad_norm": 7.364444732666016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29137569665908813,
      "logits/rejected": 0.26287397742271423,
      "logps/chosen": -494.147216796875,
      "logps/rejected": -487.21575927734375,
      "loss": 0.6075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7027714252471924,
      "rewards/margins": 0.3035537600517273,
      "rewards/rejected": -1.006325125694275,
      "step": 767
    },
    {
      "epoch": 0.1004988958861536,
      "grad_norm": 5.110191822052002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32367056608200073,
      "logits/rejected": 0.3241129517555237,
      "logps/chosen": -527.70068359375,
      "logps/rejected": -395.0962829589844,
      "loss": 0.479,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22317923605442047,
      "rewards/margins": 0.7132276296615601,
      "rewards/rejected": -0.9364069104194641,
      "step": 768
    },
    {
      "epoch": 0.10062975382350536,
      "grad_norm": 6.7914533615112305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16201823949813843,
      "logits/rejected": 0.3092207610607147,
      "logps/chosen": -441.9441223144531,
      "logps/rejected": -408.6866455078125,
      "loss": 0.5092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9431089758872986,
      "rewards/margins": 0.7534643411636353,
      "rewards/rejected": -1.6965731382369995,
      "step": 769
    },
    {
      "epoch": 0.10076061176085711,
      "grad_norm": 5.093097686767578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2790907621383667,
      "logits/rejected": 0.1318950355052948,
      "logps/chosen": -518.2168579101562,
      "logps/rejected": -328.3145446777344,
      "loss": 0.4965,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4591871201992035,
      "rewards/margins": 0.4932359755039215,
      "rewards/rejected": -0.952423095703125,
      "step": 770
    },
    {
      "epoch": 0.10089146969820888,
      "grad_norm": 6.025127410888672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.524517297744751,
      "logits/rejected": 0.5594715476036072,
      "logps/chosen": -627.1339721679688,
      "logps/rejected": -542.3154907226562,
      "loss": 0.6173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32420769333839417,
      "rewards/margins": 0.5415031909942627,
      "rewards/rejected": -0.8657108545303345,
      "step": 771
    },
    {
      "epoch": 0.10102232763556064,
      "grad_norm": 5.401389122009277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4397751986980438,
      "logits/rejected": 0.3330690860748291,
      "logps/chosen": -259.1911926269531,
      "logps/rejected": -359.97113037109375,
      "loss": 0.5811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.444668173789978,
      "rewards/margins": 0.43578413128852844,
      "rewards/rejected": -0.8804523944854736,
      "step": 772
    },
    {
      "epoch": 0.1011531855729124,
      "grad_norm": 5.85255241394043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3380261957645416,
      "logits/rejected": 0.331535667181015,
      "logps/chosen": -746.4715576171875,
      "logps/rejected": -470.125244140625,
      "loss": 0.5837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9079002141952515,
      "rewards/margins": 0.46256256103515625,
      "rewards/rejected": -1.3704627752304077,
      "step": 773
    },
    {
      "epoch": 0.10128404351026417,
      "grad_norm": 8.853219985961914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31773051619529724,
      "logits/rejected": 0.2583145499229431,
      "logps/chosen": -514.1084594726562,
      "logps/rejected": -371.86029052734375,
      "loss": 0.9901,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6122369766235352,
      "rewards/margins": -0.2688913345336914,
      "rewards/rejected": -0.34334564208984375,
      "step": 774
    },
    {
      "epoch": 0.10141490144761593,
      "grad_norm": 5.791942596435547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5240508317947388,
      "logits/rejected": 0.18819883465766907,
      "logps/chosen": -382.034912109375,
      "logps/rejected": -374.39013671875,
      "loss": 0.6214,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4466831684112549,
      "rewards/margins": 0.5607240796089172,
      "rewards/rejected": -1.0074071884155273,
      "step": 775
    },
    {
      "epoch": 0.1015457593849677,
      "grad_norm": 4.36548376083374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2635425329208374,
      "logits/rejected": 0.19240650534629822,
      "logps/chosen": -498.6607666015625,
      "logps/rejected": -436.4168395996094,
      "loss": 0.4732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5721357464790344,
      "rewards/margins": 0.6587162017822266,
      "rewards/rejected": -1.2308520078659058,
      "step": 776
    },
    {
      "epoch": 0.10167661732231946,
      "grad_norm": 5.036925315856934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43399783968925476,
      "logits/rejected": 0.7241575717926025,
      "logps/chosen": -474.7344055175781,
      "logps/rejected": -402.51910400390625,
      "loss": 0.4902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8839539289474487,
      "rewards/margins": 0.5228700041770935,
      "rewards/rejected": -1.4068238735198975,
      "step": 777
    },
    {
      "epoch": 0.10180747525967122,
      "grad_norm": 6.098311901092529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24024096131324768,
      "logits/rejected": 0.5282402038574219,
      "logps/chosen": -471.1260681152344,
      "logps/rejected": -397.3375244140625,
      "loss": 0.5813,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5693528652191162,
      "rewards/margins": 0.5368784666061401,
      "rewards/rejected": -1.106231451034546,
      "step": 778
    },
    {
      "epoch": 0.10193833319702299,
      "grad_norm": 9.31104564666748,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24309520423412323,
      "logits/rejected": 0.19092151522636414,
      "logps/chosen": -386.6473083496094,
      "logps/rejected": -451.51422119140625,
      "loss": 0.8968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9231780767440796,
      "rewards/margins": -0.27920129895210266,
      "rewards/rejected": -0.6439768075942993,
      "step": 779
    },
    {
      "epoch": 0.10206919113437475,
      "grad_norm": 7.225132465362549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.528052031993866,
      "logits/rejected": 0.11994505673646927,
      "logps/chosen": -503.22125244140625,
      "logps/rejected": -348.0010070800781,
      "loss": 0.6523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.691636323928833,
      "rewards/margins": 0.19712258875370026,
      "rewards/rejected": -0.8887588381767273,
      "step": 780
    },
    {
      "epoch": 0.1022000490717265,
      "grad_norm": 5.311466217041016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25881969928741455,
      "logits/rejected": 0.3210188150405884,
      "logps/chosen": -323.103759765625,
      "logps/rejected": -401.819091796875,
      "loss": 0.6143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6104715466499329,
      "rewards/margins": 0.49213582277297974,
      "rewards/rejected": -1.1026073694229126,
      "step": 781
    },
    {
      "epoch": 0.10233090700907826,
      "grad_norm": 6.872526168823242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06805442273616791,
      "logits/rejected": 0.07661402970552444,
      "logps/chosen": -424.7983703613281,
      "logps/rejected": -367.9195556640625,
      "loss": 0.7004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8384552597999573,
      "rewards/margins": 0.2353181540966034,
      "rewards/rejected": -1.0737733840942383,
      "step": 782
    },
    {
      "epoch": 0.10246176494643003,
      "grad_norm": 3.5892786979675293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19518065452575684,
      "logits/rejected": 0.22249533236026764,
      "logps/chosen": -360.6571960449219,
      "logps/rejected": -409.6820068359375,
      "loss": 0.4635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5858217477798462,
      "rewards/margins": 0.6682665348052979,
      "rewards/rejected": -1.254088282585144,
      "step": 783
    },
    {
      "epoch": 0.10259262288378179,
      "grad_norm": 5.334415435791016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24685940146446228,
      "logits/rejected": 0.13880404829978943,
      "logps/chosen": -375.8072814941406,
      "logps/rejected": -423.06298828125,
      "loss": 0.6576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.08547043800354,
      "rewards/margins": 0.2029293328523636,
      "rewards/rejected": -1.288399577140808,
      "step": 784
    },
    {
      "epoch": 0.10272348082113356,
      "grad_norm": 6.893562316894531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2358655035495758,
      "logits/rejected": 0.4493864178657532,
      "logps/chosen": -419.2173156738281,
      "logps/rejected": -400.8935852050781,
      "loss": 0.7432,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8511743545532227,
      "rewards/margins": 0.05779438465833664,
      "rewards/rejected": -0.9089687466621399,
      "step": 785
    },
    {
      "epoch": 0.10285433875848532,
      "grad_norm": 6.552959442138672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4997107684612274,
      "logits/rejected": 0.5101848840713501,
      "logps/chosen": -415.7711486816406,
      "logps/rejected": -422.6083679199219,
      "loss": 0.8047,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9771319031715393,
      "rewards/margins": -0.15114061534404755,
      "rewards/rejected": -0.8259912729263306,
      "step": 786
    },
    {
      "epoch": 0.10298519669583708,
      "grad_norm": 6.65531587600708,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5176591277122498,
      "logits/rejected": 0.30253446102142334,
      "logps/chosen": -460.35345458984375,
      "logps/rejected": -426.0553283691406,
      "loss": 0.7199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7210161089897156,
      "rewards/margins": 0.23372671008110046,
      "rewards/rejected": -0.9547428488731384,
      "step": 787
    },
    {
      "epoch": 0.10311605463318885,
      "grad_norm": 5.372630596160889,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1706445813179016,
      "logits/rejected": 0.2761213779449463,
      "logps/chosen": -515.8637084960938,
      "logps/rejected": -385.42095947265625,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9968992471694946,
      "rewards/margins": 0.5661934018135071,
      "rewards/rejected": -1.563092589378357,
      "step": 788
    },
    {
      "epoch": 0.10324691257054061,
      "grad_norm": 4.18715238571167,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3471287190914154,
      "logits/rejected": 0.3232196271419525,
      "logps/chosen": -459.1716003417969,
      "logps/rejected": -397.3865051269531,
      "loss": 0.471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.518655002117157,
      "rewards/margins": 0.6480159759521484,
      "rewards/rejected": -1.1666710376739502,
      "step": 789
    },
    {
      "epoch": 0.10337777050789237,
      "grad_norm": 5.3941426277160645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32413411140441895,
      "logits/rejected": 0.4794948399066925,
      "logps/chosen": -372.6148376464844,
      "logps/rejected": -341.2162170410156,
      "loss": 0.5471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42099955677986145,
      "rewards/margins": 0.684269368648529,
      "rewards/rejected": -1.1052688360214233,
      "step": 790
    },
    {
      "epoch": 0.10350862844524414,
      "grad_norm": 4.227010726928711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42895275354385376,
      "logits/rejected": 0.3319123685359955,
      "logps/chosen": -418.18792724609375,
      "logps/rejected": -349.6480712890625,
      "loss": 0.5839,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15334013104438782,
      "rewards/margins": 0.47328415513038635,
      "rewards/rejected": -0.31994399428367615,
      "step": 791
    },
    {
      "epoch": 0.10363948638259589,
      "grad_norm": 5.358755111694336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5356914401054382,
      "logits/rejected": 0.2551209032535553,
      "logps/chosen": -484.9924011230469,
      "logps/rejected": -488.58038330078125,
      "loss": 0.55,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5475936532020569,
      "rewards/margins": 0.3628818690776825,
      "rewards/rejected": -0.9104755520820618,
      "step": 792
    },
    {
      "epoch": 0.10377034431994765,
      "grad_norm": 6.493375301361084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2689949572086334,
      "logits/rejected": 0.3104904592037201,
      "logps/chosen": -563.9481201171875,
      "logps/rejected": -462.1107177734375,
      "loss": 0.4868,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6241010427474976,
      "rewards/margins": 0.7135146856307983,
      "rewards/rejected": -1.337615728378296,
      "step": 793
    },
    {
      "epoch": 0.10390120225729942,
      "grad_norm": 5.4449944496154785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4204469919204712,
      "logits/rejected": 0.355338454246521,
      "logps/chosen": -402.3786315917969,
      "logps/rejected": -305.5943298339844,
      "loss": 0.4676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7806946039199829,
      "rewards/margins": 0.7637684345245361,
      "rewards/rejected": -1.5444629192352295,
      "step": 794
    },
    {
      "epoch": 0.10403206019465118,
      "grad_norm": 4.293424606323242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37868422269821167,
      "logits/rejected": 0.20221690833568573,
      "logps/chosen": -570.0875854492188,
      "logps/rejected": -415.6699523925781,
      "loss": 0.4021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3159387707710266,
      "rewards/margins": 0.8857710361480713,
      "rewards/rejected": -1.2017097473144531,
      "step": 795
    },
    {
      "epoch": 0.10416291813200294,
      "grad_norm": 5.1812872886657715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26536673307418823,
      "logits/rejected": 0.05727935582399368,
      "logps/chosen": -311.9419250488281,
      "logps/rejected": -317.1072998046875,
      "loss": 0.5211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7864099740982056,
      "rewards/margins": 0.6125985980033875,
      "rewards/rejected": -1.3990086317062378,
      "step": 796
    },
    {
      "epoch": 0.10429377606935471,
      "grad_norm": 5.346696853637695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3161201477050781,
      "logits/rejected": 0.2586881220340729,
      "logps/chosen": -430.92303466796875,
      "logps/rejected": -487.6055603027344,
      "loss": 0.4788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41837769746780396,
      "rewards/margins": 0.7365854382514954,
      "rewards/rejected": -1.1549631357192993,
      "step": 797
    },
    {
      "epoch": 0.10442463400670647,
      "grad_norm": 6.3725996017456055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5583187937736511,
      "logits/rejected": 0.3376363217830658,
      "logps/chosen": -420.2054443359375,
      "logps/rejected": -412.2725830078125,
      "loss": 0.7321,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6350002288818359,
      "rewards/margins": 0.2959440350532532,
      "rewards/rejected": -0.9309442639350891,
      "step": 798
    },
    {
      "epoch": 0.10455549194405823,
      "grad_norm": 5.246599197387695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09224733710289001,
      "logits/rejected": -0.00344228558242321,
      "logps/chosen": -514.0303955078125,
      "logps/rejected": -480.4706726074219,
      "loss": 0.5291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1685236096382141,
      "rewards/margins": 0.4292701780796051,
      "rewards/rejected": -0.5977937579154968,
      "step": 799
    },
    {
      "epoch": 0.10468634988141,
      "grad_norm": 5.926941871643066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2228212207555771,
      "logits/rejected": 0.36651962995529175,
      "logps/chosen": -633.6323852539062,
      "logps/rejected": -612.0349731445312,
      "loss": 0.442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3926987051963806,
      "rewards/margins": 0.7307052612304688,
      "rewards/rejected": -1.1234039068222046,
      "step": 800
    },
    {
      "epoch": 0.10481720781876176,
      "grad_norm": 5.049549102783203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5592835545539856,
      "logits/rejected": 0.36913296580314636,
      "logps/chosen": -443.35076904296875,
      "logps/rejected": -425.351318359375,
      "loss": 0.4627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08487701416015625,
      "rewards/margins": 0.731311023235321,
      "rewards/rejected": -0.8161880373954773,
      "step": 801
    },
    {
      "epoch": 0.10494806575611353,
      "grad_norm": 7.569244861602783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2932763993740082,
      "logits/rejected": 0.2294701337814331,
      "logps/chosen": -673.4625854492188,
      "logps/rejected": -478.09881591796875,
      "loss": 0.6981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1258891820907593,
      "rewards/margins": 0.2726619839668274,
      "rewards/rejected": -1.3985512256622314,
      "step": 802
    },
    {
      "epoch": 0.10507892369346528,
      "grad_norm": 5.30280065536499,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29409950971603394,
      "logits/rejected": 0.34419217705726624,
      "logps/chosen": -307.22955322265625,
      "logps/rejected": -253.4936981201172,
      "loss": 0.6287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7691642642021179,
      "rewards/margins": 0.2962358593940735,
      "rewards/rejected": -1.0654001235961914,
      "step": 803
    },
    {
      "epoch": 0.10520978163081704,
      "grad_norm": 4.943577766418457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2927398085594177,
      "logits/rejected": 0.28448250889778137,
      "logps/chosen": -555.0213623046875,
      "logps/rejected": -669.9964599609375,
      "loss": 0.3967,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15799503028392792,
      "rewards/margins": 1.1565287113189697,
      "rewards/rejected": -1.314523696899414,
      "step": 804
    },
    {
      "epoch": 0.1053406395681688,
      "grad_norm": 8.052886009216309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4414512813091278,
      "logits/rejected": 0.42219144105911255,
      "logps/chosen": -407.8262939453125,
      "logps/rejected": -479.7161560058594,
      "loss": 0.7614,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6050186157226562,
      "rewards/margins": 0.07253246009349823,
      "rewards/rejected": -0.6775510907173157,
      "step": 805
    },
    {
      "epoch": 0.10547149750552057,
      "grad_norm": 3.8574209213256836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.389375776052475,
      "logits/rejected": 0.5058029294013977,
      "logps/chosen": -407.59210205078125,
      "logps/rejected": -425.5368347167969,
      "loss": 0.4744,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4431014060974121,
      "rewards/margins": 0.641707718372345,
      "rewards/rejected": -1.0848091840744019,
      "step": 806
    },
    {
      "epoch": 0.10560235544287233,
      "grad_norm": 9.554423332214355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1525876820087433,
      "logits/rejected": 0.1538713574409485,
      "logps/chosen": -617.0785522460938,
      "logps/rejected": -631.8154907226562,
      "loss": 0.5671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4551418423652649,
      "rewards/margins": 0.4039674699306488,
      "rewards/rejected": -0.8591092824935913,
      "step": 807
    },
    {
      "epoch": 0.1057332133802241,
      "grad_norm": 4.203765869140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44993332028388977,
      "logits/rejected": 0.1521991342306137,
      "logps/chosen": -232.40809631347656,
      "logps/rejected": -410.3840637207031,
      "loss": 0.4077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41747477650642395,
      "rewards/margins": 0.9289470911026001,
      "rewards/rejected": -1.3464218378067017,
      "step": 808
    },
    {
      "epoch": 0.10586407131757586,
      "grad_norm": 5.278911113739014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3219839632511139,
      "logits/rejected": 0.38244691491127014,
      "logps/chosen": -410.6929931640625,
      "logps/rejected": -453.1525573730469,
      "loss": 0.5606,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3775026202201843,
      "rewards/margins": 0.5104848742485046,
      "rewards/rejected": -0.8879875540733337,
      "step": 809
    },
    {
      "epoch": 0.10599492925492762,
      "grad_norm": 4.170354843139648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6074292659759521,
      "logits/rejected": 0.5722739696502686,
      "logps/chosen": -326.48876953125,
      "logps/rejected": -326.6288146972656,
      "loss": 0.4541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6943596601486206,
      "rewards/margins": 0.7490053176879883,
      "rewards/rejected": -1.4433648586273193,
      "step": 810
    },
    {
      "epoch": 0.10612578719227939,
      "grad_norm": 8.00566577911377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01374208927154541,
      "logits/rejected": 0.15728014707565308,
      "logps/chosen": -379.78314208984375,
      "logps/rejected": -538.756103515625,
      "loss": 0.65,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0781396627426147,
      "rewards/margins": 0.34576529264450073,
      "rewards/rejected": -1.4239050149917603,
      "step": 811
    },
    {
      "epoch": 0.10625664512963115,
      "grad_norm": 5.685384273529053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38880836963653564,
      "logits/rejected": 0.27242782711982727,
      "logps/chosen": -546.3648071289062,
      "logps/rejected": -544.7181396484375,
      "loss": 0.5514,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3346751928329468,
      "rewards/margins": 0.5494747161865234,
      "rewards/rejected": -0.8841498494148254,
      "step": 812
    },
    {
      "epoch": 0.10638750306698291,
      "grad_norm": 5.721697807312012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.554778516292572,
      "logits/rejected": 0.5469962954521179,
      "logps/chosen": -488.512939453125,
      "logps/rejected": -357.43701171875,
      "loss": 0.6536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5888665914535522,
      "rewards/margins": 0.23166486620903015,
      "rewards/rejected": -0.8205314874649048,
      "step": 813
    },
    {
      "epoch": 0.10651836100433466,
      "grad_norm": 5.889228343963623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3866979479789734,
      "logits/rejected": 0.48281392455101013,
      "logps/chosen": -717.6029052734375,
      "logps/rejected": -369.4361877441406,
      "loss": 0.576,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46156045794487,
      "rewards/margins": 0.40360015630722046,
      "rewards/rejected": -0.8651605844497681,
      "step": 814
    },
    {
      "epoch": 0.10664921894168643,
      "grad_norm": 5.620748043060303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6437865495681763,
      "logits/rejected": 0.4367891848087311,
      "logps/chosen": -547.76806640625,
      "logps/rejected": -483.60595703125,
      "loss": 0.5545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7617505788803101,
      "rewards/margins": 0.4099021553993225,
      "rewards/rejected": -1.171652913093567,
      "step": 815
    },
    {
      "epoch": 0.10678007687903819,
      "grad_norm": 6.057722091674805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26810792088508606,
      "logits/rejected": 0.2120940387248993,
      "logps/chosen": -614.2009887695312,
      "logps/rejected": -649.0098876953125,
      "loss": 0.5987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44716089963912964,
      "rewards/margins": 0.4530373215675354,
      "rewards/rejected": -0.900198221206665,
      "step": 816
    },
    {
      "epoch": 0.10691093481638995,
      "grad_norm": 4.757554531097412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.324847012758255,
      "logits/rejected": 0.23889335989952087,
      "logps/chosen": -430.03924560546875,
      "logps/rejected": -415.1385803222656,
      "loss": 0.5052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3400379419326782,
      "rewards/margins": 0.48973730206489563,
      "rewards/rejected": -0.8297752141952515,
      "step": 817
    },
    {
      "epoch": 0.10704179275374172,
      "grad_norm": 6.284666538238525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19693417847156525,
      "logits/rejected": 0.20557764172554016,
      "logps/chosen": -608.2255249023438,
      "logps/rejected": -405.2546081542969,
      "loss": 0.5979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6292009353637695,
      "rewards/margins": 0.4511297345161438,
      "rewards/rejected": -1.080330729484558,
      "step": 818
    },
    {
      "epoch": 0.10717265069109348,
      "grad_norm": 4.9075541496276855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4693737328052521,
      "logits/rejected": 0.7385989427566528,
      "logps/chosen": -330.6434326171875,
      "logps/rejected": -239.07310485839844,
      "loss": 0.5955,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6035186648368835,
      "rewards/margins": 0.519883394241333,
      "rewards/rejected": -1.1234019994735718,
      "step": 819
    },
    {
      "epoch": 0.10730350862844525,
      "grad_norm": 4.3886308670043945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5784168243408203,
      "logits/rejected": 0.5279878973960876,
      "logps/chosen": -336.34564208984375,
      "logps/rejected": -327.3858337402344,
      "loss": 0.5169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5976988077163696,
      "rewards/margins": 0.6131391525268555,
      "rewards/rejected": -1.210837960243225,
      "step": 820
    },
    {
      "epoch": 0.10743436656579701,
      "grad_norm": 5.524917125701904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4967363476753235,
      "logits/rejected": 0.4222244620323181,
      "logps/chosen": -512.9578857421875,
      "logps/rejected": -483.0869140625,
      "loss": 0.6484,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6141996383666992,
      "rewards/margins": 0.2717767357826233,
      "rewards/rejected": -0.8859763741493225,
      "step": 821
    },
    {
      "epoch": 0.10756522450314877,
      "grad_norm": 6.6455583572387695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29902976751327515,
      "logits/rejected": -0.005328770726919174,
      "logps/chosen": -511.28997802734375,
      "logps/rejected": -449.5905456542969,
      "loss": 0.6724,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5132589340209961,
      "rewards/margins": 0.27093827724456787,
      "rewards/rejected": -0.784197211265564,
      "step": 822
    },
    {
      "epoch": 0.10769608244050054,
      "grad_norm": 4.512702465057373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34734463691711426,
      "logits/rejected": 0.3846398591995239,
      "logps/chosen": -431.9305114746094,
      "logps/rejected": -390.58233642578125,
      "loss": 0.4618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28390368819236755,
      "rewards/margins": 0.7849154472351074,
      "rewards/rejected": -1.0688190460205078,
      "step": 823
    },
    {
      "epoch": 0.10782694037785229,
      "grad_norm": 4.742830753326416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2966124713420868,
      "logits/rejected": 0.2748314440250397,
      "logps/chosen": -442.1011047363281,
      "logps/rejected": -374.3941345214844,
      "loss": 0.5,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3055526614189148,
      "rewards/margins": 0.6641031503677368,
      "rewards/rejected": -0.9696558713912964,
      "step": 824
    },
    {
      "epoch": 0.10795779831520405,
      "grad_norm": 6.994813442230225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5119159817695618,
      "logits/rejected": 0.7123091220855713,
      "logps/chosen": -677.8743896484375,
      "logps/rejected": -745.8353881835938,
      "loss": 0.5902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47045424580574036,
      "rewards/margins": 0.3736448287963867,
      "rewards/rejected": -0.8440990447998047,
      "step": 825
    },
    {
      "epoch": 0.10808865625255581,
      "grad_norm": 5.653128623962402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35362380743026733,
      "logits/rejected": 0.17454183101654053,
      "logps/chosen": -608.4725341796875,
      "logps/rejected": -424.87420654296875,
      "loss": 0.4623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5130689740180969,
      "rewards/margins": 0.6465206146240234,
      "rewards/rejected": -1.1595896482467651,
      "step": 826
    },
    {
      "epoch": 0.10821951418990758,
      "grad_norm": 5.154376029968262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35823696851730347,
      "logits/rejected": 0.46174749732017517,
      "logps/chosen": -390.78076171875,
      "logps/rejected": -419.38458251953125,
      "loss": 0.5461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5662204623222351,
      "rewards/margins": 0.5957533121109009,
      "rewards/rejected": -1.1619737148284912,
      "step": 827
    },
    {
      "epoch": 0.10835037212725934,
      "grad_norm": 4.918712615966797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16772226989269257,
      "logits/rejected": 0.3559948801994324,
      "logps/chosen": -430.633544921875,
      "logps/rejected": -391.21722412109375,
      "loss": 0.4353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3986038267612457,
      "rewards/margins": 1.098966121673584,
      "rewards/rejected": -1.4975699186325073,
      "step": 828
    },
    {
      "epoch": 0.1084812300646111,
      "grad_norm": 5.530383586883545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10031689703464508,
      "logits/rejected": 0.01928618550300598,
      "logps/chosen": -434.033935546875,
      "logps/rejected": -398.29248046875,
      "loss": 0.6056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8858020901679993,
      "rewards/margins": 0.3679181933403015,
      "rewards/rejected": -1.2537202835083008,
      "step": 829
    },
    {
      "epoch": 0.10861208800196287,
      "grad_norm": 8.42459487915039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40966832637786865,
      "logits/rejected": 0.16055947542190552,
      "logps/chosen": -462.75396728515625,
      "logps/rejected": -557.3153076171875,
      "loss": 0.9029,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7622587084770203,
      "rewards/margins": -0.258745014667511,
      "rewards/rejected": -0.5035136938095093,
      "step": 830
    },
    {
      "epoch": 0.10874294593931463,
      "grad_norm": 7.784180641174316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3790072798728943,
      "logits/rejected": 0.3676035702228546,
      "logps/chosen": -437.673583984375,
      "logps/rejected": -370.8141174316406,
      "loss": 0.7479,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7402610778808594,
      "rewards/margins": 0.18951037526130676,
      "rewards/rejected": -0.9297714233398438,
      "step": 831
    },
    {
      "epoch": 0.1088738038766664,
      "grad_norm": 5.739704132080078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3440052568912506,
      "logits/rejected": 0.34569427371025085,
      "logps/chosen": -295.7879943847656,
      "logps/rejected": -409.22265625,
      "loss": 0.538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3054910898208618,
      "rewards/margins": 0.6155887842178345,
      "rewards/rejected": -1.9210797548294067,
      "step": 832
    },
    {
      "epoch": 0.10900466181401816,
      "grad_norm": 4.5881171226501465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.9384287595748901,
      "logits/rejected": 0.9585997462272644,
      "logps/chosen": -436.01885986328125,
      "logps/rejected": -386.5590515136719,
      "loss": 0.4993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5200441479682922,
      "rewards/margins": 0.7470373511314392,
      "rewards/rejected": -1.2670814990997314,
      "step": 833
    },
    {
      "epoch": 0.10913551975136992,
      "grad_norm": 4.721193790435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20875667035579681,
      "logits/rejected": 0.25820598006248474,
      "logps/chosen": -520.3756103515625,
      "logps/rejected": -464.1244812011719,
      "loss": 0.4738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7090217471122742,
      "rewards/margins": 0.5427740216255188,
      "rewards/rejected": -1.251795768737793,
      "step": 834
    },
    {
      "epoch": 0.10926637768872167,
      "grad_norm": 6.742298603057861,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29847386479377747,
      "logits/rejected": 0.0784335732460022,
      "logps/chosen": -726.5267333984375,
      "logps/rejected": -436.1566162109375,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8841279745101929,
      "rewards/margins": 0.08928205072879791,
      "rewards/rejected": -0.9734100103378296,
      "step": 835
    },
    {
      "epoch": 0.10939723562607344,
      "grad_norm": 3.6094908714294434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22053490579128265,
      "logits/rejected": 0.10184275358915329,
      "logps/chosen": -369.50604248046875,
      "logps/rejected": -299.24896240234375,
      "loss": 0.4292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46671807765960693,
      "rewards/margins": 0.7257221341133118,
      "rewards/rejected": -1.1924402713775635,
      "step": 836
    },
    {
      "epoch": 0.1095280935634252,
      "grad_norm": 7.0742950439453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40489619970321655,
      "logits/rejected": 0.2347404509782791,
      "logps/chosen": -496.82550048828125,
      "logps/rejected": -585.90673828125,
      "loss": 0.5717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7339145541191101,
      "rewards/margins": 0.3059062659740448,
      "rewards/rejected": -1.0398207902908325,
      "step": 837
    },
    {
      "epoch": 0.10965895150077697,
      "grad_norm": 10.122419357299805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06659922003746033,
      "logits/rejected": 0.18715724349021912,
      "logps/chosen": -482.87933349609375,
      "logps/rejected": -305.0291748046875,
      "loss": 0.8176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0287799835205078,
      "rewards/margins": 0.2700977325439453,
      "rewards/rejected": -1.298877477645874,
      "step": 838
    },
    {
      "epoch": 0.10978980943812873,
      "grad_norm": 4.550763130187988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33458438515663147,
      "logits/rejected": 0.3983002305030823,
      "logps/chosen": -482.6382751464844,
      "logps/rejected": -478.08074951171875,
      "loss": 0.3432,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.552674412727356,
      "rewards/margins": 1.31635582447052,
      "rewards/rejected": -1.869030237197876,
      "step": 839
    },
    {
      "epoch": 0.1099206673754805,
      "grad_norm": 4.680232048034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.654535174369812,
      "logits/rejected": 0.6299415230751038,
      "logps/chosen": -586.3633422851562,
      "logps/rejected": -480.54718017578125,
      "loss": 0.3961,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6203567385673523,
      "rewards/margins": 0.8458923101425171,
      "rewards/rejected": -1.4662489891052246,
      "step": 840
    },
    {
      "epoch": 0.11005152531283226,
      "grad_norm": 5.205569267272949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24258333444595337,
      "logits/rejected": 0.25040963292121887,
      "logps/chosen": -421.9167175292969,
      "logps/rejected": -458.2255859375,
      "loss": 0.434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6402168273925781,
      "rewards/margins": 0.840161919593811,
      "rewards/rejected": -1.4803786277770996,
      "step": 841
    },
    {
      "epoch": 0.11018238325018402,
      "grad_norm": 5.008338451385498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4799945056438446,
      "logits/rejected": 0.36087873578071594,
      "logps/chosen": -616.4524536132812,
      "logps/rejected": -553.6436157226562,
      "loss": 0.4921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23768481612205505,
      "rewards/margins": 0.6430097222328186,
      "rewards/rejected": -0.8806946277618408,
      "step": 842
    },
    {
      "epoch": 0.11031324118753578,
      "grad_norm": 6.363810062408447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0579523965716362,
      "logits/rejected": 0.5352542996406555,
      "logps/chosen": -411.9973449707031,
      "logps/rejected": -498.4246826171875,
      "loss": 0.5047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8420547246932983,
      "rewards/margins": 0.7330948114395142,
      "rewards/rejected": -1.575149655342102,
      "step": 843
    },
    {
      "epoch": 0.11044409912488755,
      "grad_norm": 4.975373268127441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10578759014606476,
      "logits/rejected": 0.3538111746311188,
      "logps/chosen": -335.5250244140625,
      "logps/rejected": -357.22967529296875,
      "loss": 0.5438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7799096703529358,
      "rewards/margins": 0.522068440914154,
      "rewards/rejected": -1.3019781112670898,
      "step": 844
    },
    {
      "epoch": 0.11057495706223931,
      "grad_norm": 4.187541484832764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26875799894332886,
      "logits/rejected": 0.2537512481212616,
      "logps/chosen": -493.5310363769531,
      "logps/rejected": -465.01409912109375,
      "loss": 0.391,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36362743377685547,
      "rewards/margins": 0.8772950172424316,
      "rewards/rejected": -1.240922451019287,
      "step": 845
    },
    {
      "epoch": 0.11070581499959106,
      "grad_norm": 4.664307594299316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18635624647140503,
      "logits/rejected": 0.2715798020362854,
      "logps/chosen": -477.9554443359375,
      "logps/rejected": -427.5113220214844,
      "loss": 0.4497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6499618291854858,
      "rewards/margins": 0.6553922891616821,
      "rewards/rejected": -1.305354118347168,
      "step": 846
    },
    {
      "epoch": 0.11083667293694283,
      "grad_norm": 5.550420761108398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34127986431121826,
      "logits/rejected": 0.5083394646644592,
      "logps/chosen": -502.445068359375,
      "logps/rejected": -615.4642333984375,
      "loss": 0.5088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8072729110717773,
      "rewards/margins": 0.8196377158164978,
      "rewards/rejected": -1.62691068649292,
      "step": 847
    },
    {
      "epoch": 0.11096753087429459,
      "grad_norm": 5.23015022277832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021311461925506592,
      "logits/rejected": 0.1667921394109726,
      "logps/chosen": -364.0929870605469,
      "logps/rejected": -325.0736083984375,
      "loss": 0.6674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7610016465187073,
      "rewards/margins": 0.3533000946044922,
      "rewards/rejected": -1.1143016815185547,
      "step": 848
    },
    {
      "epoch": 0.11109838881164635,
      "grad_norm": 5.768561840057373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25233644247055054,
      "logits/rejected": 0.44310763478279114,
      "logps/chosen": -491.5770568847656,
      "logps/rejected": -447.87200927734375,
      "loss": 0.6641,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.990277111530304,
      "rewards/margins": 0.13805045187473297,
      "rewards/rejected": -1.1283276081085205,
      "step": 849
    },
    {
      "epoch": 0.11122924674899812,
      "grad_norm": 6.204668998718262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.054232511669397354,
      "logits/rejected": 0.09705398231744766,
      "logps/chosen": -331.67474365234375,
      "logps/rejected": -372.81500244140625,
      "loss": 0.7469,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0376044511795044,
      "rewards/margins": 0.0582151859998703,
      "rewards/rejected": -1.0958197116851807,
      "step": 850
    },
    {
      "epoch": 0.11136010468634988,
      "grad_norm": 8.507284164428711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010394349694252014,
      "logits/rejected": -0.2077840268611908,
      "logps/chosen": -496.8857116699219,
      "logps/rejected": -578.5689086914062,
      "loss": 0.8358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7534943222999573,
      "rewards/margins": -0.055787086486816406,
      "rewards/rejected": -0.6977071762084961,
      "step": 851
    },
    {
      "epoch": 0.11149096262370164,
      "grad_norm": 8.99571418762207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20702697336673737,
      "logits/rejected": 0.3595193326473236,
      "logps/chosen": -561.7128295898438,
      "logps/rejected": -395.94940185546875,
      "loss": 0.7496,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4086233079433441,
      "rewards/margins": 0.3242513835430145,
      "rewards/rejected": -0.7328747510910034,
      "step": 852
    },
    {
      "epoch": 0.11162182056105341,
      "grad_norm": 5.398214340209961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5511203408241272,
      "logits/rejected": 0.3218974471092224,
      "logps/chosen": -421.83782958984375,
      "logps/rejected": -453.59527587890625,
      "loss": 0.4639,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8388305902481079,
      "rewards/margins": 0.6267030835151672,
      "rewards/rejected": -1.46553373336792,
      "step": 853
    },
    {
      "epoch": 0.11175267849840517,
      "grad_norm": 6.048295974731445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05137949436903,
      "logits/rejected": 0.2935093641281128,
      "logps/chosen": -399.00030517578125,
      "logps/rejected": -449.3959655761719,
      "loss": 0.5299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6701934933662415,
      "rewards/margins": 0.7885631322860718,
      "rewards/rejected": -1.458756685256958,
      "step": 854
    },
    {
      "epoch": 0.11188353643575694,
      "grad_norm": 5.740461349487305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06006694212555885,
      "logits/rejected": 0.0561065599322319,
      "logps/chosen": -433.5227966308594,
      "logps/rejected": -562.5149536132812,
      "loss": 0.4661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.769199788570404,
      "rewards/margins": 1.0364277362823486,
      "rewards/rejected": -1.805627465248108,
      "step": 855
    },
    {
      "epoch": 0.1120143943731087,
      "grad_norm": 4.945367813110352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2680344581604004,
      "logits/rejected": 0.1650383174419403,
      "logps/chosen": -365.1053161621094,
      "logps/rejected": -425.3182067871094,
      "loss": 0.4127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5953825116157532,
      "rewards/margins": 1.0503687858581543,
      "rewards/rejected": -1.6457514762878418,
      "step": 856
    },
    {
      "epoch": 0.11214525231046045,
      "grad_norm": 5.031427383422852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44731149077415466,
      "logits/rejected": 0.4499232769012451,
      "logps/chosen": -326.587158203125,
      "logps/rejected": -258.5126647949219,
      "loss": 0.5093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0173828601837158,
      "rewards/margins": 0.5003801584243774,
      "rewards/rejected": -1.5177628993988037,
      "step": 857
    },
    {
      "epoch": 0.11227611024781221,
      "grad_norm": 9.049826622009277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22712351381778717,
      "logits/rejected": 0.13591091334819794,
      "logps/chosen": -442.83465576171875,
      "logps/rejected": -463.2509765625,
      "loss": 0.7203,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8660076260566711,
      "rewards/margins": 0.7476606369018555,
      "rewards/rejected": -1.6136682033538818,
      "step": 858
    },
    {
      "epoch": 0.11240696818516398,
      "grad_norm": 4.838373184204102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2701056897640228,
      "logits/rejected": 0.3160448968410492,
      "logps/chosen": -475.133056640625,
      "logps/rejected": -463.77496337890625,
      "loss": 0.4053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8229553699493408,
      "rewards/margins": 1.0597801208496094,
      "rewards/rejected": -1.8827354907989502,
      "step": 859
    },
    {
      "epoch": 0.11253782612251574,
      "grad_norm": 5.750983238220215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18825121223926544,
      "logits/rejected": 0.2248075306415558,
      "logps/chosen": -678.2138061523438,
      "logps/rejected": -472.881103515625,
      "loss": 0.5607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17188243567943573,
      "rewards/margins": 0.6448390483856201,
      "rewards/rejected": -0.8167215585708618,
      "step": 860
    },
    {
      "epoch": 0.1126686840598675,
      "grad_norm": 5.424793720245361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19448557496070862,
      "logits/rejected": 0.2222394198179245,
      "logps/chosen": -468.4676208496094,
      "logps/rejected": -520.0569458007812,
      "loss": 0.4601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.025882355868816376,
      "rewards/margins": 0.6959545612335205,
      "rewards/rejected": -0.7218368649482727,
      "step": 861
    },
    {
      "epoch": 0.11279954199721927,
      "grad_norm": 6.13049840927124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36989134550094604,
      "logits/rejected": 0.5617847442626953,
      "logps/chosen": -385.56146240234375,
      "logps/rejected": -348.7109680175781,
      "loss": 0.7449,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3589966595172882,
      "rewards/margins": 0.27845343947410583,
      "rewards/rejected": -0.6374500393867493,
      "step": 862
    },
    {
      "epoch": 0.11293039993457103,
      "grad_norm": 6.274487018585205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2744051218032837,
      "logits/rejected": 0.19986233115196228,
      "logps/chosen": -596.9049682617188,
      "logps/rejected": -699.7832641601562,
      "loss": 0.6333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1109318733215332,
      "rewards/margins": 0.23952332139015198,
      "rewards/rejected": -1.3504551649093628,
      "step": 863
    },
    {
      "epoch": 0.1130612578719228,
      "grad_norm": 5.764925003051758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09070532768964767,
      "logits/rejected": 0.2535328269004822,
      "logps/chosen": -501.1924743652344,
      "logps/rejected": -392.9209289550781,
      "loss": 0.5845,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.759168267250061,
      "rewards/margins": 0.5506246089935303,
      "rewards/rejected": -1.3097929954528809,
      "step": 864
    },
    {
      "epoch": 0.11319211580927456,
      "grad_norm": 8.786309242248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29567205905914307,
      "logits/rejected": 0.5330418944358826,
      "logps/chosen": -357.00750732421875,
      "logps/rejected": -367.564208984375,
      "loss": 0.6832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9776020646095276,
      "rewards/margins": 0.36591875553131104,
      "rewards/rejected": -1.3435207605361938,
      "step": 865
    },
    {
      "epoch": 0.11332297374662632,
      "grad_norm": 6.2362060546875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5808522701263428,
      "logits/rejected": 0.13430289924144745,
      "logps/chosen": -424.90313720703125,
      "logps/rejected": -424.7942810058594,
      "loss": 0.5078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1718015670776367,
      "rewards/margins": 0.7114551663398743,
      "rewards/rejected": -1.8832569122314453,
      "step": 866
    },
    {
      "epoch": 0.11345383168397809,
      "grad_norm": 5.5603814125061035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47849512100219727,
      "logits/rejected": 0.5905318856239319,
      "logps/chosen": -374.9496154785156,
      "logps/rejected": -436.2245178222656,
      "loss": 0.5632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5394808053970337,
      "rewards/margins": 0.6364332437515259,
      "rewards/rejected": -1.1759140491485596,
      "step": 867
    },
    {
      "epoch": 0.11358468962132984,
      "grad_norm": 5.352482318878174,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15444499254226685,
      "logits/rejected": 0.21691212058067322,
      "logps/chosen": -413.9619140625,
      "logps/rejected": -413.091796875,
      "loss": 0.4832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45163196325302124,
      "rewards/margins": 0.6083078384399414,
      "rewards/rejected": -1.0599397420883179,
      "step": 868
    },
    {
      "epoch": 0.1137155475586816,
      "grad_norm": 6.759423732757568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1257888674736023,
      "logits/rejected": -0.06304923444986343,
      "logps/chosen": -598.613037109375,
      "logps/rejected": -592.3701171875,
      "loss": 0.5482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21018430590629578,
      "rewards/margins": 1.00485360622406,
      "rewards/rejected": -1.2150379419326782,
      "step": 869
    },
    {
      "epoch": 0.11384640549603336,
      "grad_norm": 7.210513591766357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3669978976249695,
      "logits/rejected": 0.3889007866382599,
      "logps/chosen": -586.5531616210938,
      "logps/rejected": -360.9434814453125,
      "loss": 0.6312,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7187673449516296,
      "rewards/margins": 0.30528315901756287,
      "rewards/rejected": -1.0240505933761597,
      "step": 870
    },
    {
      "epoch": 0.11397726343338513,
      "grad_norm": 6.852779388427734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02094864472746849,
      "logits/rejected": 0.09766271710395813,
      "logps/chosen": -356.27716064453125,
      "logps/rejected": -348.07745361328125,
      "loss": 0.6315,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6631879806518555,
      "rewards/margins": 0.20058727264404297,
      "rewards/rejected": -0.8637752532958984,
      "step": 871
    },
    {
      "epoch": 0.11410812137073689,
      "grad_norm": 4.078627586364746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38701605796813965,
      "logits/rejected": 0.3705693185329437,
      "logps/chosen": -586.1931762695312,
      "logps/rejected": -418.15753173828125,
      "loss": 0.3926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2313966155052185,
      "rewards/margins": 1.173054575920105,
      "rewards/rejected": -0.9416579008102417,
      "step": 872
    },
    {
      "epoch": 0.11423897930808866,
      "grad_norm": 5.596363067626953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03995883837342262,
      "logits/rejected": -0.16279348731040955,
      "logps/chosen": -364.5232849121094,
      "logps/rejected": -408.1069641113281,
      "loss": 0.576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6764446496963501,
      "rewards/margins": 0.3753545880317688,
      "rewards/rejected": -1.0517991781234741,
      "step": 873
    },
    {
      "epoch": 0.11436983724544042,
      "grad_norm": 4.362244606018066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.367165207862854,
      "logits/rejected": 0.35882335901260376,
      "logps/chosen": -274.9637451171875,
      "logps/rejected": -357.0531921386719,
      "loss": 0.431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5064264535903931,
      "rewards/margins": 1.1031068563461304,
      "rewards/rejected": -1.6095331907272339,
      "step": 874
    },
    {
      "epoch": 0.11450069518279218,
      "grad_norm": 5.665684223175049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4353468418121338,
      "logits/rejected": 0.5389131903648376,
      "logps/chosen": -533.9715576171875,
      "logps/rejected": -600.7987670898438,
      "loss": 0.4861,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8014976382255554,
      "rewards/margins": 0.5285770893096924,
      "rewards/rejected": -1.330074667930603,
      "step": 875
    },
    {
      "epoch": 0.11463155312014395,
      "grad_norm": 5.865130424499512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.493729829788208,
      "logits/rejected": 0.5419198274612427,
      "logps/chosen": -550.2811889648438,
      "logps/rejected": -465.3130187988281,
      "loss": 0.4871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08784674853086472,
      "rewards/margins": 0.9027194976806641,
      "rewards/rejected": -0.9905662536621094,
      "step": 876
    },
    {
      "epoch": 0.11476241105749571,
      "grad_norm": 5.389120101928711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44242385029792786,
      "logits/rejected": 0.21040065586566925,
      "logps/chosen": -556.1556396484375,
      "logps/rejected": -498.22161865234375,
      "loss": 0.4214,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5823234915733337,
      "rewards/margins": 0.950461208820343,
      "rewards/rejected": -1.5327845811843872,
      "step": 877
    },
    {
      "epoch": 0.11489326899484748,
      "grad_norm": 5.977161884307861,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2579685151576996,
      "logits/rejected": 0.31714656949043274,
      "logps/chosen": -428.6722106933594,
      "logps/rejected": -320.1172180175781,
      "loss": 0.6266,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29977738857269287,
      "rewards/margins": 0.32676735520362854,
      "rewards/rejected": -0.6265447735786438,
      "step": 878
    },
    {
      "epoch": 0.11502412693219922,
      "grad_norm": 6.3652567863464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0930185616016388,
      "logits/rejected": -0.17624859511852264,
      "logps/chosen": -690.1708984375,
      "logps/rejected": -636.9344482421875,
      "loss": 0.5433,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8991897702217102,
      "rewards/margins": 0.9627849459648132,
      "rewards/rejected": -1.8619747161865234,
      "step": 879
    },
    {
      "epoch": 0.11515498486955099,
      "grad_norm": 4.302181243896484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28441017866134644,
      "logits/rejected": 0.4915646016597748,
      "logps/chosen": -268.44525146484375,
      "logps/rejected": -388.63580322265625,
      "loss": 0.4769,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5016781091690063,
      "rewards/margins": 0.6881374716758728,
      "rewards/rejected": -1.1898155212402344,
      "step": 880
    },
    {
      "epoch": 0.11528584280690275,
      "grad_norm": 4.565494537353516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21284252405166626,
      "logits/rejected": 0.29874366521835327,
      "logps/chosen": -432.58685302734375,
      "logps/rejected": -315.25909423828125,
      "loss": 0.398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5903688669204712,
      "rewards/margins": 0.8236898183822632,
      "rewards/rejected": -1.4140586853027344,
      "step": 881
    },
    {
      "epoch": 0.11541670074425452,
      "grad_norm": 7.661503791809082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13000154495239258,
      "logits/rejected": 0.38715237379074097,
      "logps/chosen": -604.8644409179688,
      "logps/rejected": -438.9541931152344,
      "loss": 0.7177,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9728395938873291,
      "rewards/margins": 0.2902509272098541,
      "rewards/rejected": -1.2630906105041504,
      "step": 882
    },
    {
      "epoch": 0.11554755868160628,
      "grad_norm": 6.709864616394043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4067714810371399,
      "logits/rejected": 0.3433372378349304,
      "logps/chosen": -488.2435302734375,
      "logps/rejected": -409.26300048828125,
      "loss": 0.5646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9152869582176208,
      "rewards/margins": 0.6885578632354736,
      "rewards/rejected": -1.6038447618484497,
      "step": 883
    },
    {
      "epoch": 0.11567841661895804,
      "grad_norm": 4.506185054779053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41922223567962646,
      "logits/rejected": 0.4795446991920471,
      "logps/chosen": -412.9285583496094,
      "logps/rejected": -312.2352294921875,
      "loss": 0.4857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08732166886329651,
      "rewards/margins": 0.5609728097915649,
      "rewards/rejected": -0.4736511707305908,
      "step": 884
    },
    {
      "epoch": 0.11580927455630981,
      "grad_norm": 8.078036308288574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22525112330913544,
      "logits/rejected": 0.4651859402656555,
      "logps/chosen": -499.2409973144531,
      "logps/rejected": -598.4593505859375,
      "loss": 0.645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6460167169570923,
      "rewards/margins": 0.3517600893974304,
      "rewards/rejected": -0.9977768659591675,
      "step": 885
    },
    {
      "epoch": 0.11594013249366157,
      "grad_norm": 6.484405040740967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4037385582923889,
      "logits/rejected": 0.16425122320652008,
      "logps/chosen": -572.4058837890625,
      "logps/rejected": -632.8954467773438,
      "loss": 0.541,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.636204719543457,
      "rewards/margins": 0.5232910513877869,
      "rewards/rejected": -1.1594957113265991,
      "step": 886
    },
    {
      "epoch": 0.11607099043101334,
      "grad_norm": 5.992774963378906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3495398759841919,
      "logits/rejected": 0.216516375541687,
      "logps/chosen": -582.4830932617188,
      "logps/rejected": -372.9735107421875,
      "loss": 0.4235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3064633905887604,
      "rewards/margins": 0.9954912066459656,
      "rewards/rejected": -1.3019546270370483,
      "step": 887
    },
    {
      "epoch": 0.1162018483683651,
      "grad_norm": 4.750809192657471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5530473589897156,
      "logits/rejected": 0.41738763451576233,
      "logps/chosen": -545.2443237304688,
      "logps/rejected": -598.979248046875,
      "loss": 0.4632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5315422415733337,
      "rewards/margins": 0.6911224722862244,
      "rewards/rejected": -1.222664713859558,
      "step": 888
    },
    {
      "epoch": 0.11633270630571686,
      "grad_norm": 4.810063362121582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2756572961807251,
      "logits/rejected": 0.564177930355072,
      "logps/chosen": -346.93865966796875,
      "logps/rejected": -398.04376220703125,
      "loss": 0.4586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6995742917060852,
      "rewards/margins": 0.8175007104873657,
      "rewards/rejected": -1.5170749425888062,
      "step": 889
    },
    {
      "epoch": 0.11646356424306861,
      "grad_norm": 6.78872537612915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.390410840511322,
      "logits/rejected": 0.313638299703598,
      "logps/chosen": -444.2767028808594,
      "logps/rejected": -478.06048583984375,
      "loss": 0.6234,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8226485252380371,
      "rewards/margins": 0.3668974041938782,
      "rewards/rejected": -1.18954598903656,
      "step": 890
    },
    {
      "epoch": 0.11659442218042038,
      "grad_norm": 6.676537990570068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2511456310749054,
      "logits/rejected": -0.02103346586227417,
      "logps/chosen": -523.0083618164062,
      "logps/rejected": -599.6345825195312,
      "loss": 0.571,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.001143455505371,
      "rewards/margins": 0.51247638463974,
      "rewards/rejected": -1.5136198997497559,
      "step": 891
    },
    {
      "epoch": 0.11672528011777214,
      "grad_norm": 9.018078804016113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5326060652732849,
      "logits/rejected": 0.46745067834854126,
      "logps/chosen": -389.348876953125,
      "logps/rejected": -374.9736633300781,
      "loss": 0.8019,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9149473309516907,
      "rewards/margins": 0.31223490834236145,
      "rewards/rejected": -1.227182149887085,
      "step": 892
    },
    {
      "epoch": 0.1168561380551239,
      "grad_norm": 5.235430717468262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3660978078842163,
      "logits/rejected": 0.16353639960289001,
      "logps/chosen": -361.01287841796875,
      "logps/rejected": -283.5741882324219,
      "loss": 0.496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4013151526451111,
      "rewards/margins": 0.6337608098983765,
      "rewards/rejected": -1.0350760221481323,
      "step": 893
    },
    {
      "epoch": 0.11698699599247567,
      "grad_norm": 5.229407787322998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30385640263557434,
      "logits/rejected": 0.26116806268692017,
      "logps/chosen": -645.533203125,
      "logps/rejected": -700.8955078125,
      "loss": 0.4091,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20954591035842896,
      "rewards/margins": 1.136351227760315,
      "rewards/rejected": -0.926805317401886,
      "step": 894
    },
    {
      "epoch": 0.11711785392982743,
      "grad_norm": 6.624841690063477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48803481459617615,
      "logits/rejected": 0.35960230231285095,
      "logps/chosen": -623.2972412109375,
      "logps/rejected": -440.0205078125,
      "loss": 0.4571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8556040525436401,
      "rewards/margins": 0.6896567940711975,
      "rewards/rejected": -1.5452607870101929,
      "step": 895
    },
    {
      "epoch": 0.1172487118671792,
      "grad_norm": 5.823755741119385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1800316572189331,
      "logits/rejected": 0.25594261288642883,
      "logps/chosen": -378.44476318359375,
      "logps/rejected": -432.614013671875,
      "loss": 0.4323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9558877944946289,
      "rewards/margins": 0.7870477437973022,
      "rewards/rejected": -1.7429354190826416,
      "step": 896
    },
    {
      "epoch": 0.11737956980453096,
      "grad_norm": 8.09742259979248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49576592445373535,
      "logits/rejected": 0.48067349195480347,
      "logps/chosen": -504.4131774902344,
      "logps/rejected": -463.05145263671875,
      "loss": 0.9014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9869574308395386,
      "rewards/margins": 0.007301881909370422,
      "rewards/rejected": -0.9942592978477478,
      "step": 897
    },
    {
      "epoch": 0.11751042774188272,
      "grad_norm": 3.2915079593658447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24911321699619293,
      "logits/rejected": 0.24377809464931488,
      "logps/chosen": -569.69580078125,
      "logps/rejected": -484.3837890625,
      "loss": 0.285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6111027002334595,
      "rewards/margins": 1.6046698093414307,
      "rewards/rejected": -2.2157726287841797,
      "step": 898
    },
    {
      "epoch": 0.11764128567923449,
      "grad_norm": 8.312788963317871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3543381989002228,
      "logits/rejected": 0.45084646344184875,
      "logps/chosen": -414.910888671875,
      "logps/rejected": -481.6664123535156,
      "loss": 0.7399,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.144717812538147,
      "rewards/margins": 0.2213985174894333,
      "rewards/rejected": -1.3661162853240967,
      "step": 899
    },
    {
      "epoch": 0.11777214361658625,
      "grad_norm": 7.598565578460693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19956941902637482,
      "logits/rejected": 0.26926884055137634,
      "logps/chosen": -505.60443115234375,
      "logps/rejected": -436.04669189453125,
      "loss": 0.9008,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.15657377243042,
      "rewards/margins": -0.19984807074069977,
      "rewards/rejected": -0.9567256569862366,
      "step": 900
    },
    {
      "epoch": 0.117903001553938,
      "grad_norm": 6.154417514801025,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24655723571777344,
      "logits/rejected": 0.5192055106163025,
      "logps/chosen": -420.0001220703125,
      "logps/rejected": -425.8867492675781,
      "loss": 0.6654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8360832929611206,
      "rewards/margins": 0.22512894868850708,
      "rewards/rejected": -1.061212182044983,
      "step": 901
    },
    {
      "epoch": 0.11803385949128976,
      "grad_norm": 6.789346694946289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37321794033050537,
      "logits/rejected": 0.271905779838562,
      "logps/chosen": -473.9968566894531,
      "logps/rejected": -392.333984375,
      "loss": 0.7093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7735702991485596,
      "rewards/margins": 0.3787580728530884,
      "rewards/rejected": -1.152328372001648,
      "step": 902
    },
    {
      "epoch": 0.11816471742864153,
      "grad_norm": 9.978965759277344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1256456971168518,
      "logits/rejected": 0.17152318358421326,
      "logps/chosen": -534.4208984375,
      "logps/rejected": -580.8660278320312,
      "loss": 1.0697,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1515538692474365,
      "rewards/margins": -0.3806079924106598,
      "rewards/rejected": -0.7709457278251648,
      "step": 903
    },
    {
      "epoch": 0.11829557536599329,
      "grad_norm": 8.355548858642578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3575800955295563,
      "logits/rejected": 0.44638773798942566,
      "logps/chosen": -552.0428466796875,
      "logps/rejected": -454.0060119628906,
      "loss": 0.8285,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34024325013160706,
      "rewards/margins": 0.3956776559352875,
      "rewards/rejected": -0.7359209060668945,
      "step": 904
    },
    {
      "epoch": 0.11842643330334506,
      "grad_norm": 7.720804691314697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5284213423728943,
      "logits/rejected": 0.5219791531562805,
      "logps/chosen": -546.8360595703125,
      "logps/rejected": -393.690673828125,
      "loss": 0.6748,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8758301734924316,
      "rewards/margins": 0.4224415123462677,
      "rewards/rejected": -1.298271656036377,
      "step": 905
    },
    {
      "epoch": 0.11855729124069682,
      "grad_norm": 5.570131301879883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3882042169570923,
      "logits/rejected": 0.3910590708255768,
      "logps/chosen": -388.33184814453125,
      "logps/rejected": -455.248046875,
      "loss": 0.5958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6781486868858337,
      "rewards/margins": 0.9668917059898376,
      "rewards/rejected": -1.6450403928756714,
      "step": 906
    },
    {
      "epoch": 0.11868814917804858,
      "grad_norm": 7.305237293243408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.8009530305862427,
      "logits/rejected": 0.5464763641357422,
      "logps/chosen": -470.268310546875,
      "logps/rejected": -412.14349365234375,
      "loss": 0.7406,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.04302978515625,
      "rewards/margins": 0.14600394666194916,
      "rewards/rejected": -1.1890337467193604,
      "step": 907
    },
    {
      "epoch": 0.11881900711540035,
      "grad_norm": 5.6059746742248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1990164816379547,
      "logits/rejected": 0.4320182204246521,
      "logps/chosen": -626.4769287109375,
      "logps/rejected": -620.38818359375,
      "loss": 0.41,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17013874650001526,
      "rewards/margins": 0.9780920147895813,
      "rewards/rejected": -1.148230791091919,
      "step": 908
    },
    {
      "epoch": 0.11894986505275211,
      "grad_norm": 6.561068058013916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.421471506357193,
      "logits/rejected": 0.34909310936927795,
      "logps/chosen": -521.764404296875,
      "logps/rejected": -478.9140625,
      "loss": 0.6662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27925771474838257,
      "rewards/margins": 0.3903476595878601,
      "rewards/rejected": -0.6696054339408875,
      "step": 909
    },
    {
      "epoch": 0.11908072299010387,
      "grad_norm": 7.1937408447265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02834625542163849,
      "logits/rejected": 0.15861469507217407,
      "logps/chosen": -548.1945190429688,
      "logps/rejected": -543.3731689453125,
      "loss": 0.5807,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6406877636909485,
      "rewards/margins": 0.6645536422729492,
      "rewards/rejected": -1.305241346359253,
      "step": 910
    },
    {
      "epoch": 0.11921158092745564,
      "grad_norm": 4.168766975402832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4267480969429016,
      "logits/rejected": 0.3709794878959656,
      "logps/chosen": -421.51934814453125,
      "logps/rejected": -420.3378601074219,
      "loss": 0.4367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2654912769794464,
      "rewards/margins": 1.0059480667114258,
      "rewards/rejected": -1.2714394330978394,
      "step": 911
    },
    {
      "epoch": 0.11934243886480739,
      "grad_norm": 4.3005781173706055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2658211588859558,
      "logits/rejected": 0.2992108166217804,
      "logps/chosen": -357.664306640625,
      "logps/rejected": -366.5072326660156,
      "loss": 0.3668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41463032364845276,
      "rewards/margins": 1.0201886892318726,
      "rewards/rejected": -1.4348191022872925,
      "step": 912
    },
    {
      "epoch": 0.11947329680215915,
      "grad_norm": 5.696707725524902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6667912602424622,
      "logits/rejected": 0.4976036548614502,
      "logps/chosen": -400.7867126464844,
      "logps/rejected": -417.189453125,
      "loss": 0.5379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9502981305122375,
      "rewards/margins": 0.39834707975387573,
      "rewards/rejected": -1.3486452102661133,
      "step": 913
    },
    {
      "epoch": 0.11960415473951092,
      "grad_norm": 4.845549583435059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4267340898513794,
      "logits/rejected": 0.5218257308006287,
      "logps/chosen": -424.8563232421875,
      "logps/rejected": -312.2115478515625,
      "loss": 0.3747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6622831225395203,
      "rewards/margins": 0.9048737287521362,
      "rewards/rejected": -1.5671567916870117,
      "step": 914
    },
    {
      "epoch": 0.11973501267686268,
      "grad_norm": 5.541475296020508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36795133352279663,
      "logits/rejected": 0.22532904148101807,
      "logps/chosen": -289.43377685546875,
      "logps/rejected": -332.23858642578125,
      "loss": 0.6211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.31191885471344,
      "rewards/margins": 0.32175102829933167,
      "rewards/rejected": -1.6336698532104492,
      "step": 915
    },
    {
      "epoch": 0.11986587061421444,
      "grad_norm": 5.475308418273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23716503381729126,
      "logits/rejected": 0.06738603860139847,
      "logps/chosen": -383.4229431152344,
      "logps/rejected": -498.75628662109375,
      "loss": 0.4342,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7345011234283447,
      "rewards/margins": 1.2257097959518433,
      "rewards/rejected": -1.9602108001708984,
      "step": 916
    },
    {
      "epoch": 0.1199967285515662,
      "grad_norm": 6.503938674926758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46259957551956177,
      "logits/rejected": 0.45904746651649475,
      "logps/chosen": -445.76007080078125,
      "logps/rejected": -372.8926696777344,
      "loss": 0.705,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7780061364173889,
      "rewards/margins": 0.15021812915802002,
      "rewards/rejected": -0.9282242059707642,
      "step": 917
    },
    {
      "epoch": 0.12012758648891797,
      "grad_norm": 6.170630931854248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4400131106376648,
      "logits/rejected": 0.35561662912368774,
      "logps/chosen": -442.29718017578125,
      "logps/rejected": -367.0413513183594,
      "loss": 0.498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7091802358627319,
      "rewards/margins": 0.6235570907592773,
      "rewards/rejected": -1.3327373266220093,
      "step": 918
    },
    {
      "epoch": 0.12025844442626973,
      "grad_norm": 4.9579243659973145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6789141297340393,
      "logits/rejected": 0.6590065956115723,
      "logps/chosen": -456.7152099609375,
      "logps/rejected": -379.6198425292969,
      "loss": 0.4886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4439247250556946,
      "rewards/margins": 0.7824448943138123,
      "rewards/rejected": -1.2263695001602173,
      "step": 919
    },
    {
      "epoch": 0.1203893023636215,
      "grad_norm": 6.75512170791626,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24689078330993652,
      "logits/rejected": 0.36766326427459717,
      "logps/chosen": -571.3815307617188,
      "logps/rejected": -536.032470703125,
      "loss": 0.5154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5132652521133423,
      "rewards/margins": 0.5290511846542358,
      "rewards/rejected": -1.0423165559768677,
      "step": 920
    },
    {
      "epoch": 0.12052016030097326,
      "grad_norm": 8.481575012207031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35391563177108765,
      "logits/rejected": 0.35889580845832825,
      "logps/chosen": -496.63055419921875,
      "logps/rejected": -417.9285583496094,
      "loss": 0.6378,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9588807821273804,
      "rewards/margins": 0.7593370676040649,
      "rewards/rejected": -1.7182180881500244,
      "step": 921
    },
    {
      "epoch": 0.12065101823832503,
      "grad_norm": 6.070562839508057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2827538549900055,
      "logits/rejected": 0.2773073613643646,
      "logps/chosen": -672.70458984375,
      "logps/rejected": -519.416259765625,
      "loss": 0.4764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6025413274765015,
      "rewards/margins": 0.7734304666519165,
      "rewards/rejected": -1.375971794128418,
      "step": 922
    },
    {
      "epoch": 0.12078187617567678,
      "grad_norm": 5.8401899337768555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.595574676990509,
      "logits/rejected": 0.601962149143219,
      "logps/chosen": -392.7971496582031,
      "logps/rejected": -436.0509338378906,
      "loss": 0.5723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9399133324623108,
      "rewards/margins": 0.4324125349521637,
      "rewards/rejected": -1.3723258972167969,
      "step": 923
    },
    {
      "epoch": 0.12091273411302854,
      "grad_norm": 4.556703090667725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36596009135246277,
      "logits/rejected": 0.3112567067146301,
      "logps/chosen": -553.7994384765625,
      "logps/rejected": -439.2347412109375,
      "loss": 0.4752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15350855886936188,
      "rewards/margins": 1.3603260517120361,
      "rewards/rejected": -1.5138345956802368,
      "step": 924
    },
    {
      "epoch": 0.1210435920503803,
      "grad_norm": 5.279415130615234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13779020309448242,
      "logits/rejected": 0.289930522441864,
      "logps/chosen": -372.9906005859375,
      "logps/rejected": -397.39996337890625,
      "loss": 0.4536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3174625039100647,
      "rewards/margins": 0.681065022945404,
      "rewards/rejected": -0.9985275268554688,
      "step": 925
    },
    {
      "epoch": 0.12117444998773207,
      "grad_norm": 4.5431599617004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4761233925819397,
      "logits/rejected": 0.38420820236206055,
      "logps/chosen": -558.12744140625,
      "logps/rejected": -393.280029296875,
      "loss": 0.3514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6077839136123657,
      "rewards/margins": 1.1772226095199585,
      "rewards/rejected": -1.7850066423416138,
      "step": 926
    },
    {
      "epoch": 0.12130530792508383,
      "grad_norm": 5.801321983337402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5947443246841431,
      "logits/rejected": 0.44670262932777405,
      "logps/chosen": -542.6265258789062,
      "logps/rejected": -429.4884033203125,
      "loss": 0.497,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6838486790657043,
      "rewards/margins": 0.746900200843811,
      "rewards/rejected": -1.4307489395141602,
      "step": 927
    },
    {
      "epoch": 0.1214361658624356,
      "grad_norm": 5.952662944793701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20475932955741882,
      "logits/rejected": 0.1154291182756424,
      "logps/chosen": -555.9732666015625,
      "logps/rejected": -352.2993469238281,
      "loss": 0.6283,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.689452588558197,
      "rewards/margins": 0.6089437007904053,
      "rewards/rejected": -1.298396348953247,
      "step": 928
    },
    {
      "epoch": 0.12156702379978736,
      "grad_norm": 5.167762756347656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14777342975139618,
      "logits/rejected": 0.19398868083953857,
      "logps/chosen": -476.87158203125,
      "logps/rejected": -354.716796875,
      "loss": 0.4638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42926520109176636,
      "rewards/margins": 0.6651262640953064,
      "rewards/rejected": -1.0943914651870728,
      "step": 929
    },
    {
      "epoch": 0.12169788173713912,
      "grad_norm": 6.72353982925415,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4026697874069214,
      "logits/rejected": 0.3923761546611786,
      "logps/chosen": -468.8904724121094,
      "logps/rejected": -529.3204956054688,
      "loss": 0.6207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8955281376838684,
      "rewards/margins": 0.36736613512039185,
      "rewards/rejected": -1.2628942728042603,
      "step": 930
    },
    {
      "epoch": 0.12182873967449089,
      "grad_norm": 8.429191589355469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4279451370239258,
      "logits/rejected": 0.17145663499832153,
      "logps/chosen": -561.0809936523438,
      "logps/rejected": -608.36328125,
      "loss": 0.5489,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7537504434585571,
      "rewards/margins": 0.4780994653701782,
      "rewards/rejected": -1.2318499088287354,
      "step": 931
    },
    {
      "epoch": 0.12195959761184265,
      "grad_norm": 6.090408802032471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14478690922260284,
      "logits/rejected": 0.12229539453983307,
      "logps/chosen": -302.7214660644531,
      "logps/rejected": -396.4291687011719,
      "loss": 0.5875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9111449718475342,
      "rewards/margins": 0.5346128344535828,
      "rewards/rejected": -1.4457578659057617,
      "step": 932
    },
    {
      "epoch": 0.12209045554919441,
      "grad_norm": 8.637927055358887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2825557589530945,
      "logits/rejected": 0.5047904253005981,
      "logps/chosen": -474.7752685546875,
      "logps/rejected": -486.78289794921875,
      "loss": 0.552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.192674994468689,
      "rewards/margins": 0.5697935819625854,
      "rewards/rejected": -1.7624685764312744,
      "step": 933
    },
    {
      "epoch": 0.12222131348654616,
      "grad_norm": 4.984228134155273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34335601329803467,
      "logits/rejected": 0.5594561696052551,
      "logps/chosen": -385.40594482421875,
      "logps/rejected": -473.0711364746094,
      "loss": 0.4667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2849024534225464,
      "rewards/margins": 0.744271457195282,
      "rewards/rejected": -2.0291738510131836,
      "step": 934
    },
    {
      "epoch": 0.12235217142389793,
      "grad_norm": 6.953893184661865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5749280452728271,
      "logits/rejected": 0.4662952721118927,
      "logps/chosen": -605.919189453125,
      "logps/rejected": -535.251953125,
      "loss": 0.5434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6248285174369812,
      "rewards/margins": 0.5452585220336914,
      "rewards/rejected": -1.1700870990753174,
      "step": 935
    },
    {
      "epoch": 0.12248302936124969,
      "grad_norm": 9.206121444702148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4603883922100067,
      "logits/rejected": 0.47162818908691406,
      "logps/chosen": -441.5382995605469,
      "logps/rejected": -434.5299377441406,
      "loss": 0.8138,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.28917396068573,
      "rewards/margins": 0.031800463795661926,
      "rewards/rejected": -1.320974349975586,
      "step": 936
    },
    {
      "epoch": 0.12261388729860145,
      "grad_norm": 6.062788009643555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08865029364824295,
      "logits/rejected": 0.3367513120174408,
      "logps/chosen": -440.9075927734375,
      "logps/rejected": -359.02105712890625,
      "loss": 0.5073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0643339157104492,
      "rewards/margins": 0.7909203171730042,
      "rewards/rejected": -1.8552541732788086,
      "step": 937
    },
    {
      "epoch": 0.12274474523595322,
      "grad_norm": 5.240674018859863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40106555819511414,
      "logits/rejected": 0.38264840841293335,
      "logps/chosen": -555.2310180664062,
      "logps/rejected": -619.1748046875,
      "loss": 0.4663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5977010726928711,
      "rewards/margins": 0.6967591643333435,
      "rewards/rejected": -1.2944602966308594,
      "step": 938
    },
    {
      "epoch": 0.12287560317330498,
      "grad_norm": 5.343712329864502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2720458209514618,
      "logits/rejected": 0.3699706792831421,
      "logps/chosen": -485.21697998046875,
      "logps/rejected": -336.239990234375,
      "loss": 0.5062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0626153945922852,
      "rewards/margins": 0.5729441046714783,
      "rewards/rejected": -1.6355595588684082,
      "step": 939
    },
    {
      "epoch": 0.12300646111065675,
      "grad_norm": 11.430858612060547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22239306569099426,
      "logits/rejected": 0.40165916085243225,
      "logps/chosen": -481.94696044921875,
      "logps/rejected": -261.7507019042969,
      "loss": 0.8755,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3974406719207764,
      "rewards/margins": -0.17830944061279297,
      "rewards/rejected": -1.2191312313079834,
      "step": 940
    },
    {
      "epoch": 0.12313731904800851,
      "grad_norm": 5.745604038238525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2909781336784363,
      "logits/rejected": 0.15087935328483582,
      "logps/chosen": -436.3047180175781,
      "logps/rejected": -355.3074645996094,
      "loss": 0.6478,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4204263687133789,
      "rewards/margins": 0.16836397349834442,
      "rewards/rejected": -0.5887903571128845,
      "step": 941
    },
    {
      "epoch": 0.12326817698536027,
      "grad_norm": 5.804140090942383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5744646191596985,
      "logits/rejected": 0.6003060340881348,
      "logps/chosen": -664.011474609375,
      "logps/rejected": -650.8877563476562,
      "loss": 0.5621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3094492256641388,
      "rewards/margins": 0.5643938183784485,
      "rewards/rejected": -0.8738430142402649,
      "step": 942
    },
    {
      "epoch": 0.12339903492271204,
      "grad_norm": 8.349814414978027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2342434823513031,
      "logits/rejected": 0.24641326069831848,
      "logps/chosen": -437.74847412109375,
      "logps/rejected": -551.4112548828125,
      "loss": 0.853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1101335659623146,
      "rewards/margins": -0.02508583664894104,
      "rewards/rejected": -0.08504773676395416,
      "step": 943
    },
    {
      "epoch": 0.12352989286006379,
      "grad_norm": 6.819198131561279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26377108693122864,
      "logits/rejected": 0.1371789574623108,
      "logps/chosen": -382.8796691894531,
      "logps/rejected": -485.18359375,
      "loss": 0.5968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9403585195541382,
      "rewards/margins": 0.4845474362373352,
      "rewards/rejected": -1.4249058961868286,
      "step": 944
    },
    {
      "epoch": 0.12366075079741555,
      "grad_norm": 7.866349220275879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3062565326690674,
      "logits/rejected": 0.19999194145202637,
      "logps/chosen": -556.6314086914062,
      "logps/rejected": -424.78741455078125,
      "loss": 0.6495,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0244554281234741,
      "rewards/margins": 0.3407849669456482,
      "rewards/rejected": -1.3652405738830566,
      "step": 945
    },
    {
      "epoch": 0.12379160873476731,
      "grad_norm": 6.900474548339844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4008490741252899,
      "logits/rejected": 0.5156485438346863,
      "logps/chosen": -434.7265319824219,
      "logps/rejected": -365.9407958984375,
      "loss": 0.5882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6759992837905884,
      "rewards/margins": 0.3259859085083008,
      "rewards/rejected": -1.0019851922988892,
      "step": 946
    },
    {
      "epoch": 0.12392246667211908,
      "grad_norm": 7.849832534790039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48028135299682617,
      "logits/rejected": 0.43609827756881714,
      "logps/chosen": -338.6278381347656,
      "logps/rejected": -399.10870361328125,
      "loss": 0.6175,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2125071287155151,
      "rewards/margins": 0.4970388114452362,
      "rewards/rejected": -1.7095459699630737,
      "step": 947
    },
    {
      "epoch": 0.12405332460947084,
      "grad_norm": 6.406095504760742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3257363736629486,
      "logits/rejected": 0.2162856161594391,
      "logps/chosen": -553.0769653320312,
      "logps/rejected": -495.5730895996094,
      "loss": 0.5513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6656745672225952,
      "rewards/margins": 0.5405634641647339,
      "rewards/rejected": -1.206238031387329,
      "step": 948
    },
    {
      "epoch": 0.1241841825468226,
      "grad_norm": 3.4160289764404297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47242480516433716,
      "logits/rejected": 0.5519857406616211,
      "logps/chosen": -540.0386962890625,
      "logps/rejected": -470.09783935546875,
      "loss": 0.2808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05499802157282829,
      "rewards/margins": 1.3742480278015137,
      "rewards/rejected": -1.429245948791504,
      "step": 949
    },
    {
      "epoch": 0.12431504048417437,
      "grad_norm": 8.39553451538086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35863521695137024,
      "logits/rejected": 0.3345405161380768,
      "logps/chosen": -457.138916015625,
      "logps/rejected": -443.03607177734375,
      "loss": 0.674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8186912536621094,
      "rewards/margins": 0.6336410641670227,
      "rewards/rejected": -1.4523323774337769,
      "step": 950
    },
    {
      "epoch": 0.12444589842152613,
      "grad_norm": 4.822265148162842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27384135127067566,
      "logits/rejected": 0.3994259238243103,
      "logps/chosen": -313.6097106933594,
      "logps/rejected": -400.7183837890625,
      "loss": 0.3292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5092742443084717,
      "rewards/margins": 1.1382801532745361,
      "rewards/rejected": -1.6475545167922974,
      "step": 951
    },
    {
      "epoch": 0.1245767563588779,
      "grad_norm": 9.764653205871582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08398668467998505,
      "logits/rejected": -0.03424740582704544,
      "logps/chosen": -580.6981811523438,
      "logps/rejected": -395.64227294921875,
      "loss": 0.9824,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4446420669555664,
      "rewards/margins": -0.26204490661621094,
      "rewards/rejected": -1.1825971603393555,
      "step": 952
    },
    {
      "epoch": 0.12470761429622966,
      "grad_norm": 5.428713798522949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4309752583503723,
      "logits/rejected": 0.19462905824184418,
      "logps/chosen": -580.003662109375,
      "logps/rejected": -572.9896240234375,
      "loss": 0.4644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.613412082195282,
      "rewards/margins": 0.8950804471969604,
      "rewards/rejected": -1.5084925889968872,
      "step": 953
    },
    {
      "epoch": 0.12483847223358142,
      "grad_norm": 7.363371849060059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06191156804561615,
      "logits/rejected": 0.2657619118690491,
      "logps/chosen": -622.9961547851562,
      "logps/rejected": -450.28741455078125,
      "loss": 0.5898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3460472524166107,
      "rewards/margins": 0.3309711515903473,
      "rewards/rejected": -0.677018404006958,
      "step": 954
    },
    {
      "epoch": 0.12496933017093317,
      "grad_norm": 6.174604892730713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3541105389595032,
      "logits/rejected": 0.4173390865325928,
      "logps/chosen": -405.5657653808594,
      "logps/rejected": -413.7557067871094,
      "loss": 0.5246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6498956680297852,
      "rewards/margins": 0.5855957269668579,
      "rewards/rejected": -1.235491394996643,
      "step": 955
    },
    {
      "epoch": 0.12510018810828494,
      "grad_norm": 6.9405035972595215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45109236240386963,
      "logits/rejected": 0.46927905082702637,
      "logps/chosen": -310.0649719238281,
      "logps/rejected": -353.36798095703125,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.842392086982727,
      "rewards/margins": 0.19178682565689087,
      "rewards/rejected": -1.0341789722442627,
      "step": 956
    },
    {
      "epoch": 0.1252310460456367,
      "grad_norm": 5.940856456756592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011123431846499443,
      "logits/rejected": 0.02902516908943653,
      "logps/chosen": -345.0038757324219,
      "logps/rejected": -313.4781799316406,
      "loss": 0.576,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.929182231426239,
      "rewards/margins": 0.5839742422103882,
      "rewards/rejected": -1.5131566524505615,
      "step": 957
    },
    {
      "epoch": 0.12536190398298847,
      "grad_norm": 6.067348480224609,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21760748326778412,
      "logits/rejected": 0.15945793688297272,
      "logps/chosen": -496.0253601074219,
      "logps/rejected": -493.9806823730469,
      "loss": 0.5594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11625918745994568,
      "rewards/margins": 0.47122713923454285,
      "rewards/rejected": -0.3549678921699524,
      "step": 958
    },
    {
      "epoch": 0.12549276192034023,
      "grad_norm": 6.161136627197266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5429046750068665,
      "logits/rejected": 0.7133273482322693,
      "logps/chosen": -413.1007995605469,
      "logps/rejected": -509.2440185546875,
      "loss": 0.5679,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8323566317558289,
      "rewards/margins": 0.45179611444473267,
      "rewards/rejected": -1.284152865409851,
      "step": 959
    },
    {
      "epoch": 0.125623619857692,
      "grad_norm": 5.418319225311279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09406471252441406,
      "logits/rejected": 0.2789316773414612,
      "logps/chosen": -456.1820373535156,
      "logps/rejected": -586.8197021484375,
      "loss": 0.4589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4163965582847595,
      "rewards/margins": 0.9161064028739929,
      "rewards/rejected": -1.332502841949463,
      "step": 960
    },
    {
      "epoch": 0.12575447779504376,
      "grad_norm": 8.8511381149292,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2937075197696686,
      "logits/rejected": 0.2538214325904846,
      "logps/chosen": -528.917236328125,
      "logps/rejected": -485.60906982421875,
      "loss": 0.6869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.36521780490875244,
      "rewards/margins": 0.3226930499076843,
      "rewards/rejected": -0.6879109144210815,
      "step": 961
    },
    {
      "epoch": 0.12588533573239552,
      "grad_norm": 6.295707702636719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3790960907936096,
      "logits/rejected": 0.3313724994659424,
      "logps/chosen": -587.9446411132812,
      "logps/rejected": -590.6953735351562,
      "loss": 0.4903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3133842647075653,
      "rewards/margins": 0.9026243686676025,
      "rewards/rejected": -1.2160085439682007,
      "step": 962
    },
    {
      "epoch": 0.12601619366974728,
      "grad_norm": 6.091015815734863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3064703047275543,
      "logits/rejected": 0.38121405243873596,
      "logps/chosen": -503.39892578125,
      "logps/rejected": -435.693603515625,
      "loss": 0.5404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5407602190971375,
      "rewards/margins": 0.5596472024917603,
      "rewards/rejected": -1.1004074811935425,
      "step": 963
    },
    {
      "epoch": 0.12614705160709905,
      "grad_norm": 4.936882495880127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4226933419704437,
      "logits/rejected": 0.3772403597831726,
      "logps/chosen": -454.49200439453125,
      "logps/rejected": -464.0226745605469,
      "loss": 0.4474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002018935978412628,
      "rewards/margins": 1.0238101482391357,
      "rewards/rejected": -1.0258290767669678,
      "step": 964
    },
    {
      "epoch": 0.1262779095444508,
      "grad_norm": 6.704470634460449,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3177357316017151,
      "logits/rejected": 0.27976667881011963,
      "logps/chosen": -523.5711669921875,
      "logps/rejected": -393.8192138671875,
      "loss": 0.6321,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2608911395072937,
      "rewards/margins": 0.5192380547523499,
      "rewards/rejected": -0.7801292538642883,
      "step": 965
    },
    {
      "epoch": 0.12640876748180258,
      "grad_norm": 8.268489837646484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1877700686454773,
      "logits/rejected": 0.07016178965568542,
      "logps/chosen": -540.3603515625,
      "logps/rejected": -346.4853515625,
      "loss": 0.7481,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0078601837158203,
      "rewards/margins": -0.0783476009964943,
      "rewards/rejected": -0.929512619972229,
      "step": 966
    },
    {
      "epoch": 0.12653962541915434,
      "grad_norm": 6.925821781158447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5750939846038818,
      "logits/rejected": 0.39683324098587036,
      "logps/chosen": -384.1754150390625,
      "logps/rejected": -355.7662353515625,
      "loss": 0.7284,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5927845239639282,
      "rewards/margins": 0.07213859260082245,
      "rewards/rejected": -0.6649231314659119,
      "step": 967
    },
    {
      "epoch": 0.1266704833565061,
      "grad_norm": 5.696268081665039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1726110279560089,
      "logits/rejected": 0.25957125425338745,
      "logps/chosen": -472.640869140625,
      "logps/rejected": -377.6659240722656,
      "loss": 0.6567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04358328506350517,
      "rewards/margins": 0.5028911828994751,
      "rewards/rejected": -0.5464744567871094,
      "step": 968
    },
    {
      "epoch": 0.12680134129385787,
      "grad_norm": 7.066081523895264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.178463414311409,
      "logits/rejected": 0.035286396741867065,
      "logps/chosen": -438.4370422363281,
      "logps/rejected": -329.1421203613281,
      "loss": 0.4771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47650662064552307,
      "rewards/margins": 0.8468199372291565,
      "rewards/rejected": -1.323326587677002,
      "step": 969
    },
    {
      "epoch": 0.12693219923120963,
      "grad_norm": 6.25821590423584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31224167346954346,
      "logits/rejected": 0.32712066173553467,
      "logps/chosen": -547.9844970703125,
      "logps/rejected": -494.2738037109375,
      "loss": 0.5861,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16261884570121765,
      "rewards/margins": 0.5618950128555298,
      "rewards/rejected": -0.39927616715431213,
      "step": 970
    },
    {
      "epoch": 0.1270630571685614,
      "grad_norm": 4.626386642456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11140843480825424,
      "logits/rejected": -0.12299302220344543,
      "logps/chosen": -344.0912780761719,
      "logps/rejected": -408.05462646484375,
      "loss": 0.5748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16181087493896484,
      "rewards/margins": 0.6186309456825256,
      "rewards/rejected": -0.7804418802261353,
      "step": 971
    },
    {
      "epoch": 0.12719391510591313,
      "grad_norm": 6.793534278869629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10569670796394348,
      "logits/rejected": 0.4285798966884613,
      "logps/chosen": -485.8283386230469,
      "logps/rejected": -522.380615234375,
      "loss": 0.58,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4308170676231384,
      "rewards/margins": 0.4551391899585724,
      "rewards/rejected": -0.8859562277793884,
      "step": 972
    },
    {
      "epoch": 0.1273247730432649,
      "grad_norm": 4.937919616699219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2850506603717804,
      "logits/rejected": 0.2605257034301758,
      "logps/chosen": -484.2558288574219,
      "logps/rejected": -425.7637023925781,
      "loss": 0.3968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.026807982474565506,
      "rewards/margins": 0.8555076122283936,
      "rewards/rejected": -0.8286997675895691,
      "step": 973
    },
    {
      "epoch": 0.12745563098061666,
      "grad_norm": 9.393157958984375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5140948295593262,
      "logits/rejected": 0.49116894602775574,
      "logps/chosen": -806.7579956054688,
      "logps/rejected": -471.2535400390625,
      "loss": 0.7066,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3171358108520508,
      "rewards/margins": 0.16027987003326416,
      "rewards/rejected": -0.47741568088531494,
      "step": 974
    },
    {
      "epoch": 0.12758648891796842,
      "grad_norm": 7.080035209655762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08880466222763062,
      "logits/rejected": 0.1052064374089241,
      "logps/chosen": -456.7549133300781,
      "logps/rejected": -528.2907104492188,
      "loss": 0.513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7791614532470703,
      "rewards/margins": 0.4604211747646332,
      "rewards/rejected": -1.2395826578140259,
      "step": 975
    },
    {
      "epoch": 0.12771734685532019,
      "grad_norm": 6.375178337097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45785415172576904,
      "logits/rejected": 0.5054827928543091,
      "logps/chosen": -321.233154296875,
      "logps/rejected": -359.26470947265625,
      "loss": 0.5932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6087266802787781,
      "rewards/margins": 0.6547015905380249,
      "rewards/rejected": -1.2634282112121582,
      "step": 976
    },
    {
      "epoch": 0.12784820479267195,
      "grad_norm": 5.845597743988037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3341589868068695,
      "logits/rejected": 0.2612766623497009,
      "logps/chosen": -615.0374755859375,
      "logps/rejected": -566.5733032226562,
      "loss": 0.3418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10910147428512573,
      "rewards/margins": 1.2006454467773438,
      "rewards/rejected": -1.3097469806671143,
      "step": 977
    },
    {
      "epoch": 0.1279790627300237,
      "grad_norm": 6.505733966827393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4210808575153351,
      "logits/rejected": 0.4270290732383728,
      "logps/chosen": -336.74761962890625,
      "logps/rejected": -406.70147705078125,
      "loss": 0.8218,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6842037439346313,
      "rewards/margins": -0.0657043382525444,
      "rewards/rejected": -0.6184993386268616,
      "step": 978
    },
    {
      "epoch": 0.12810992066737548,
      "grad_norm": 6.173058032989502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19769932329654694,
      "logits/rejected": 0.00931251049041748,
      "logps/chosen": -490.08154296875,
      "logps/rejected": -435.62493896484375,
      "loss": 0.5142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48473262786865234,
      "rewards/margins": 0.4880892336368561,
      "rewards/rejected": -0.9728217720985413,
      "step": 979
    },
    {
      "epoch": 0.12824077860472724,
      "grad_norm": 6.484613418579102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3046185374259949,
      "logits/rejected": 0.22550469636917114,
      "logps/chosen": -439.8515930175781,
      "logps/rejected": -390.69537353515625,
      "loss": 0.6169,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1294536590576172,
      "rewards/margins": 0.2427000105381012,
      "rewards/rejected": -0.3721536695957184,
      "step": 980
    },
    {
      "epoch": 0.128371636542079,
      "grad_norm": 11.958264350891113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.322191059589386,
      "logits/rejected": 0.2462863028049469,
      "logps/chosen": -498.7013854980469,
      "logps/rejected": -425.3600769042969,
      "loss": 1.0592,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8642156720161438,
      "rewards/margins": -0.45412737131118774,
      "rewards/rejected": -0.41008836030960083,
      "step": 981
    },
    {
      "epoch": 0.12850249447943077,
      "grad_norm": 4.81798791885376,
      "learning_rate": 5e-06,
      "logits/chosen": 0.386093407869339,
      "logits/rejected": 0.0770428255200386,
      "logps/chosen": -511.70245361328125,
      "logps/rejected": -406.8500061035156,
      "loss": 0.4661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26477450132369995,
      "rewards/margins": 0.7813065052032471,
      "rewards/rejected": -1.0460809469223022,
      "step": 982
    },
    {
      "epoch": 0.12863335241678253,
      "grad_norm": 4.921051979064941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5798825621604919,
      "logits/rejected": 0.4642309844493866,
      "logps/chosen": -456.8643798828125,
      "logps/rejected": -433.58392333984375,
      "loss": 0.3918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08066482841968536,
      "rewards/margins": 0.8784298896789551,
      "rewards/rejected": -0.9590946435928345,
      "step": 983
    },
    {
      "epoch": 0.1287642103541343,
      "grad_norm": 6.7087578773498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3149535655975342,
      "logits/rejected": 0.4901692271232605,
      "logps/chosen": -536.8867797851562,
      "logps/rejected": -598.5159301757812,
      "loss": 0.6006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6885811686515808,
      "rewards/margins": 0.4034256935119629,
      "rewards/rejected": -1.0920069217681885,
      "step": 984
    },
    {
      "epoch": 0.12889506829148606,
      "grad_norm": 6.204934597015381,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2523830235004425,
      "logits/rejected": 0.2185574322938919,
      "logps/chosen": -307.8053283691406,
      "logps/rejected": -277.3731689453125,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6651796698570251,
      "rewards/margins": 0.196851909160614,
      "rewards/rejected": -0.8620315194129944,
      "step": 985
    },
    {
      "epoch": 0.12902592622883782,
      "grad_norm": 5.065591812133789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10049018263816833,
      "logits/rejected": 0.3631308674812317,
      "logps/chosen": -248.7509307861328,
      "logps/rejected": -418.5736083984375,
      "loss": 0.4536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8528190851211548,
      "rewards/margins": 1.0519134998321533,
      "rewards/rejected": -1.9047324657440186,
      "step": 986
    },
    {
      "epoch": 0.1291567841661896,
      "grad_norm": 8.201048851013184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29477962851524353,
      "logits/rejected": 0.33775001764297485,
      "logps/chosen": -710.532470703125,
      "logps/rejected": -603.2080688476562,
      "loss": 0.9191,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2050367295742035,
      "rewards/margins": -0.07428265362977982,
      "rewards/rejected": -0.13075409829616547,
      "step": 987
    },
    {
      "epoch": 0.12928764210354135,
      "grad_norm": 4.521463871002197,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16106387972831726,
      "logits/rejected": 0.2753216624259949,
      "logps/chosen": -436.274658203125,
      "logps/rejected": -345.75537109375,
      "loss": 0.5085,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3548148274421692,
      "rewards/margins": 0.5778161287307739,
      "rewards/rejected": -0.9326308965682983,
      "step": 988
    },
    {
      "epoch": 0.12941850004089311,
      "grad_norm": 4.537938117980957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2997731566429138,
      "logits/rejected": 0.17209257185459137,
      "logps/chosen": -717.0152587890625,
      "logps/rejected": -432.3104553222656,
      "loss": 0.4464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.032545484602451324,
      "rewards/margins": 0.8191030025482178,
      "rewards/rejected": -0.7865575551986694,
      "step": 989
    },
    {
      "epoch": 0.12954935797824488,
      "grad_norm": 7.932755470275879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3901892304420471,
      "logits/rejected": 0.4399184286594391,
      "logps/chosen": -541.339111328125,
      "logps/rejected": -471.79833984375,
      "loss": 1.0427,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.3664281964302063,
      "rewards/margins": -0.4607732892036438,
      "rewards/rejected": 0.0943451002240181,
      "step": 990
    },
    {
      "epoch": 0.12968021591559664,
      "grad_norm": 5.431867599487305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4895409345626831,
      "logits/rejected": 0.4255991578102112,
      "logps/chosen": -305.0446472167969,
      "logps/rejected": -281.6188659667969,
      "loss": 0.7316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8028948307037354,
      "rewards/margins": 0.0805431380867958,
      "rewards/rejected": -0.8834379315376282,
      "step": 991
    },
    {
      "epoch": 0.1298110738529484,
      "grad_norm": 3.444575786590576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17037013173103333,
      "logits/rejected": 0.23469938337802887,
      "logps/chosen": -428.0037841796875,
      "logps/rejected": -451.1077880859375,
      "loss": 0.24,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3201051950454712,
      "rewards/margins": 1.4583054780960083,
      "rewards/rejected": -1.7784109115600586,
      "step": 992
    },
    {
      "epoch": 0.12994193179030017,
      "grad_norm": 4.294405937194824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49249622225761414,
      "logits/rejected": 0.3322319984436035,
      "logps/chosen": -371.062744140625,
      "logps/rejected": -425.5702209472656,
      "loss": 0.3742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5996180176734924,
      "rewards/margins": 0.9538572430610657,
      "rewards/rejected": -1.553475260734558,
      "step": 993
    },
    {
      "epoch": 0.1300727897276519,
      "grad_norm": 7.194321155548096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24430444836616516,
      "logits/rejected": 0.35034167766571045,
      "logps/chosen": -403.5347595214844,
      "logps/rejected": -409.22332763671875,
      "loss": 0.5993,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8691303133964539,
      "rewards/margins": 0.5879074335098267,
      "rewards/rejected": -1.4570379257202148,
      "step": 994
    },
    {
      "epoch": 0.13020364766500367,
      "grad_norm": 9.17737102508545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29231852293014526,
      "logits/rejected": 0.24528053402900696,
      "logps/chosen": -577.5948486328125,
      "logps/rejected": -462.21923828125,
      "loss": 0.9003,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.53173828125,
      "rewards/margins": -0.24385645985603333,
      "rewards/rejected": -0.28788185119628906,
      "step": 995
    },
    {
      "epoch": 0.13033450560235543,
      "grad_norm": 5.878252983093262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1839415729045868,
      "logits/rejected": 0.2765008509159088,
      "logps/chosen": -487.21484375,
      "logps/rejected": -413.6318359375,
      "loss": 0.4894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07381019741296768,
      "rewards/margins": 0.5184189081192017,
      "rewards/rejected": -0.5922290682792664,
      "step": 996
    },
    {
      "epoch": 0.1304653635397072,
      "grad_norm": 4.340980052947998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023503374308347702,
      "logits/rejected": -0.12372665107250214,
      "logps/chosen": -412.4071044921875,
      "logps/rejected": -294.3893737792969,
      "loss": 0.464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.016790397465229034,
      "rewards/margins": 0.7534511089324951,
      "rewards/rejected": -0.7702415585517883,
      "step": 997
    },
    {
      "epoch": 0.13059622147705896,
      "grad_norm": 4.032830715179443,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5154745578765869,
      "logits/rejected": 0.3178352415561676,
      "logps/chosen": -521.56201171875,
      "logps/rejected": -549.928955078125,
      "loss": 0.2839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3642786145210266,
      "rewards/margins": 1.237084150314331,
      "rewards/rejected": -1.601362705230713,
      "step": 998
    },
    {
      "epoch": 0.13072707941441072,
      "grad_norm": 10.683083534240723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006241991184651852,
      "logits/rejected": 0.2193288505077362,
      "logps/chosen": -560.305419921875,
      "logps/rejected": -277.5995788574219,
      "loss": 0.9686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8806349039077759,
      "rewards/margins": -0.02730569988489151,
      "rewards/rejected": -0.8533291816711426,
      "step": 999
    },
    {
      "epoch": 0.1308579373517625,
      "grad_norm": 6.30381965637207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45223599672317505,
      "logits/rejected": 0.2725021243095398,
      "logps/chosen": -547.160400390625,
      "logps/rejected": -406.62994384765625,
      "loss": 0.586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14931412041187286,
      "rewards/margins": 0.33841437101364136,
      "rewards/rejected": -0.18910028040409088,
      "step": 1000
    },
    {
      "epoch": 0.13098879528911425,
      "grad_norm": 4.519503116607666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10815072059631348,
      "logits/rejected": -0.05610961467027664,
      "logps/chosen": -463.67071533203125,
      "logps/rejected": -503.3813781738281,
      "loss": 0.3936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.049438659101724625,
      "rewards/margins": 0.8705776333808899,
      "rewards/rejected": -0.9200162887573242,
      "step": 1001
    },
    {
      "epoch": 0.13111965322646602,
      "grad_norm": 6.590030670166016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29064375162124634,
      "logits/rejected": 0.3849104642868042,
      "logps/chosen": -389.8515625,
      "logps/rejected": -453.82843017578125,
      "loss": 0.6646,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1598726212978363,
      "rewards/margins": 0.24396666884422302,
      "rewards/rejected": -0.4038393199443817,
      "step": 1002
    },
    {
      "epoch": 0.13125051116381778,
      "grad_norm": 6.391528606414795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35833731293678284,
      "logits/rejected": 0.30426695942878723,
      "logps/chosen": -786.97119140625,
      "logps/rejected": -583.7837524414062,
      "loss": 0.499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.33061471581459045,
      "rewards/margins": 0.9218519926071167,
      "rewards/rejected": -0.5912372469902039,
      "step": 1003
    },
    {
      "epoch": 0.13138136910116954,
      "grad_norm": 7.801018714904785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2958104908466339,
      "logits/rejected": 0.34546464681625366,
      "logps/chosen": -403.5703125,
      "logps/rejected": -428.282958984375,
      "loss": 0.6966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30604860186576843,
      "rewards/margins": 0.10972326993942261,
      "rewards/rejected": -0.4157719016075134,
      "step": 1004
    },
    {
      "epoch": 0.1315122270385213,
      "grad_norm": 7.867246627807617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2140325903892517,
      "logits/rejected": 0.15097923576831818,
      "logps/chosen": -403.6993713378906,
      "logps/rejected": -423.985107421875,
      "loss": 0.6315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38945120573043823,
      "rewards/margins": 1.0639358758926392,
      "rewards/rejected": -1.4533872604370117,
      "step": 1005
    },
    {
      "epoch": 0.13164308497587307,
      "grad_norm": 6.390652656555176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03016297146677971,
      "logits/rejected": 0.06898090988397598,
      "logps/chosen": -922.7549438476562,
      "logps/rejected": -506.84686279296875,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10612297058105469,
      "rewards/margins": 1.023171305656433,
      "rewards/rejected": -0.9170482754707336,
      "step": 1006
    },
    {
      "epoch": 0.13177394291322483,
      "grad_norm": 7.019920825958252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16167715191841125,
      "logits/rejected": 0.08808178454637527,
      "logps/chosen": -395.5838623046875,
      "logps/rejected": -386.9559020996094,
      "loss": 0.6523,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.426675021648407,
      "rewards/margins": 0.1792827844619751,
      "rewards/rejected": -0.6059578061103821,
      "step": 1007
    },
    {
      "epoch": 0.1319048008505766,
      "grad_norm": 7.187270641326904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4380807876586914,
      "logits/rejected": 0.4421035945415497,
      "logps/chosen": -487.42132568359375,
      "logps/rejected": -460.57489013671875,
      "loss": 0.716,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13293209671974182,
      "rewards/margins": 0.3258456885814667,
      "rewards/rejected": -0.4587777554988861,
      "step": 1008
    },
    {
      "epoch": 0.13203565878792836,
      "grad_norm": 4.2782721519470215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6027994155883789,
      "logits/rejected": 0.4973145127296448,
      "logps/chosen": -292.662841796875,
      "logps/rejected": -354.8939208984375,
      "loss": 0.473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6922809481620789,
      "rewards/margins": 0.849077582359314,
      "rewards/rejected": -1.5413585901260376,
      "step": 1009
    },
    {
      "epoch": 0.13216651672528013,
      "grad_norm": 5.232116222381592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5187036395072937,
      "logits/rejected": 0.2678436040878296,
      "logps/chosen": -425.01593017578125,
      "logps/rejected": -333.6350402832031,
      "loss": 0.5507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.238690584897995,
      "rewards/margins": 0.6322135925292969,
      "rewards/rejected": -0.8709041476249695,
      "step": 1010
    },
    {
      "epoch": 0.1322973746626319,
      "grad_norm": 4.974579334259033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18623857200145721,
      "logits/rejected": 0.07466969639062881,
      "logps/chosen": -465.5102233886719,
      "logps/rejected": -455.2747497558594,
      "loss": 0.4466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03579273819923401,
      "rewards/margins": 0.6956016421318054,
      "rewards/rejected": -0.7313944101333618,
      "step": 1011
    },
    {
      "epoch": 0.13242823259998365,
      "grad_norm": 6.123441696166992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025860439985990524,
      "logits/rejected": 0.20706796646118164,
      "logps/chosen": -480.666259765625,
      "logps/rejected": -593.56103515625,
      "loss": 0.5023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0700405091047287,
      "rewards/margins": 0.6259823441505432,
      "rewards/rejected": -0.5559417605400085,
      "step": 1012
    },
    {
      "epoch": 0.13255909053733542,
      "grad_norm": 7.049070358276367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2211543619632721,
      "logits/rejected": 0.2161274403333664,
      "logps/chosen": -574.3280029296875,
      "logps/rejected": -483.73406982421875,
      "loss": 0.4311,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05908507853746414,
      "rewards/margins": 0.9095897674560547,
      "rewards/rejected": -0.968674898147583,
      "step": 1013
    },
    {
      "epoch": 0.13268994847468718,
      "grad_norm": 5.759273529052734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5048056840896606,
      "logits/rejected": 0.531347393989563,
      "logps/chosen": -370.224853515625,
      "logps/rejected": -433.0459289550781,
      "loss": 0.4724,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24582690000534058,
      "rewards/margins": 1.0327277183532715,
      "rewards/rejected": -1.2785545587539673,
      "step": 1014
    },
    {
      "epoch": 0.13282080641203892,
      "grad_norm": 5.094720363616943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01858433708548546,
      "logits/rejected": 0.18497499823570251,
      "logps/chosen": -578.894775390625,
      "logps/rejected": -738.7767333984375,
      "loss": 0.4734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44212228059768677,
      "rewards/margins": 0.8232400417327881,
      "rewards/rejected": -1.2653623819351196,
      "step": 1015
    },
    {
      "epoch": 0.13295166434939068,
      "grad_norm": 8.157928466796875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45595934987068176,
      "logits/rejected": 0.36067327857017517,
      "logps/chosen": -537.673583984375,
      "logps/rejected": -490.8145751953125,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3096025288105011,
      "rewards/margins": 0.19969522953033447,
      "rewards/rejected": -0.5092977285385132,
      "step": 1016
    },
    {
      "epoch": 0.13308252228674244,
      "grad_norm": 11.885706901550293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3413608968257904,
      "logits/rejected": 0.2098240703344345,
      "logps/chosen": -549.29833984375,
      "logps/rejected": -299.61749267578125,
      "loss": 0.9603,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6634445190429688,
      "rewards/margins": 0.08494052290916443,
      "rewards/rejected": -0.7483850717544556,
      "step": 1017
    },
    {
      "epoch": 0.1332133802240942,
      "grad_norm": 5.507351875305176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.221379816532135,
      "logits/rejected": 0.36595845222473145,
      "logps/chosen": -402.6578369140625,
      "logps/rejected": -326.6196594238281,
      "loss": 0.4979,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.036567091941833496,
      "rewards/margins": 0.7434830069541931,
      "rewards/rejected": -0.7800500988960266,
      "step": 1018
    },
    {
      "epoch": 0.13334423816144597,
      "grad_norm": 7.302731037139893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16457104682922363,
      "logits/rejected": 0.2721419930458069,
      "logps/chosen": -551.3851318359375,
      "logps/rejected": -407.4789123535156,
      "loss": 0.5848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6669866442680359,
      "rewards/margins": 0.6711680293083191,
      "rewards/rejected": -1.3381547927856445,
      "step": 1019
    },
    {
      "epoch": 0.13347509609879774,
      "grad_norm": 5.561334609985352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4339597821235657,
      "logits/rejected": 0.4367326498031616,
      "logps/chosen": -280.84375,
      "logps/rejected": -446.45068359375,
      "loss": 0.5281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.161603182554245,
      "rewards/margins": 0.7457729578018188,
      "rewards/rejected": -0.9073761701583862,
      "step": 1020
    },
    {
      "epoch": 0.1336059540361495,
      "grad_norm": 6.980013370513916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4764598608016968,
      "logits/rejected": 0.5245504975318909,
      "logps/chosen": -395.3486022949219,
      "logps/rejected": -595.9710083007812,
      "loss": 0.5669,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2162969559431076,
      "rewards/margins": 0.34628623723983765,
      "rewards/rejected": -0.5625832080841064,
      "step": 1021
    },
    {
      "epoch": 0.13373681197350126,
      "grad_norm": 8.355899810791016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21516531705856323,
      "logits/rejected": -0.04860570281744003,
      "logps/chosen": -679.6773071289062,
      "logps/rejected": -401.8369445800781,
      "loss": 0.6778,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9679551720619202,
      "rewards/margins": 0.1973474621772766,
      "rewards/rejected": -1.1653026342391968,
      "step": 1022
    },
    {
      "epoch": 0.13386766991085303,
      "grad_norm": 8.180948257446289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6490461230278015,
      "logits/rejected": 0.6674215793609619,
      "logps/chosen": -612.2195434570312,
      "logps/rejected": -576.6028442382812,
      "loss": 0.642,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1996818631887436,
      "rewards/margins": 0.2721084654331207,
      "rewards/rejected": -0.4717903137207031,
      "step": 1023
    },
    {
      "epoch": 0.1339985278482048,
      "grad_norm": 3.8175361156463623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20976272225379944,
      "logits/rejected": 0.27362334728240967,
      "logps/chosen": -369.7750549316406,
      "logps/rejected": -284.8096008300781,
      "loss": 0.5116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11024989187717438,
      "rewards/margins": 0.5785887241363525,
      "rewards/rejected": -0.6888386011123657,
      "step": 1024
    },
    {
      "epoch": 0.13412938578555655,
      "grad_norm": 6.502624034881592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.446519136428833,
      "logits/rejected": 0.23835206031799316,
      "logps/chosen": -443.7042236328125,
      "logps/rejected": -501.2143249511719,
      "loss": 0.6519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14082831144332886,
      "rewards/margins": 0.40399208664894104,
      "rewards/rejected": -0.5448204278945923,
      "step": 1025
    },
    {
      "epoch": 0.13426024372290832,
      "grad_norm": 5.225977897644043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3316604495048523,
      "logits/rejected": 0.3413436710834503,
      "logps/chosen": -564.6347045898438,
      "logps/rejected": -542.3981323242188,
      "loss": 0.4988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12384701520204544,
      "rewards/margins": 0.5573827624320984,
      "rewards/rejected": -0.681229829788208,
      "step": 1026
    },
    {
      "epoch": 0.13439110166026008,
      "grad_norm": 6.676679611206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5435327887535095,
      "logits/rejected": 0.3338817358016968,
      "logps/chosen": -406.310791015625,
      "logps/rejected": -400.0215148925781,
      "loss": 0.6448,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5335623025894165,
      "rewards/margins": 0.6055422425270081,
      "rewards/rejected": -1.1391044855117798,
      "step": 1027
    },
    {
      "epoch": 0.13452195959761185,
      "grad_norm": 5.364424705505371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33548495173454285,
      "logits/rejected": 0.27131012082099915,
      "logps/chosen": -375.4910888671875,
      "logps/rejected": -482.8286437988281,
      "loss": 0.5909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11166077107191086,
      "rewards/margins": 0.4783535301685333,
      "rewards/rejected": -0.3666927218437195,
      "step": 1028
    },
    {
      "epoch": 0.1346528175349636,
      "grad_norm": 8.264115333557129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06778745353221893,
      "logits/rejected": 0.22384865581989288,
      "logps/chosen": -534.2788696289062,
      "logps/rejected": -622.7301025390625,
      "loss": 0.8387,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47973138093948364,
      "rewards/margins": -0.18183176219463348,
      "rewards/rejected": -0.29789960384368896,
      "step": 1029
    },
    {
      "epoch": 0.13478367547231537,
      "grad_norm": 5.42825174331665,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025680501013994217,
      "logits/rejected": 0.3818921446800232,
      "logps/chosen": -416.6944580078125,
      "logps/rejected": -368.0368957519531,
      "loss": 0.5163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4799309968948364,
      "rewards/margins": 0.6719897985458374,
      "rewards/rejected": -1.1519206762313843,
      "step": 1030
    },
    {
      "epoch": 0.13491453340966714,
      "grad_norm": 4.0384907722473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3058356046676636,
      "logits/rejected": 0.3822767436504364,
      "logps/chosen": -282.5701599121094,
      "logps/rejected": -490.8961181640625,
      "loss": 0.3051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05499936640262604,
      "rewards/margins": 1.9648678302764893,
      "rewards/rejected": -2.019867181777954,
      "step": 1031
    },
    {
      "epoch": 0.1350453913470189,
      "grad_norm": 6.883047103881836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42585331201553345,
      "logits/rejected": 0.47101759910583496,
      "logps/chosen": -458.22528076171875,
      "logps/rejected": -413.84564208984375,
      "loss": 0.6661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44926032423973083,
      "rewards/margins": 0.14897535741329193,
      "rewards/rejected": -0.5982357263565063,
      "step": 1032
    },
    {
      "epoch": 0.13517624928437066,
      "grad_norm": 4.8774027824401855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48312148451805115,
      "logits/rejected": 0.4086567759513855,
      "logps/chosen": -454.22637939453125,
      "logps/rejected": -434.08984375,
      "loss": 0.38,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4685913324356079,
      "rewards/margins": 1.041420340538025,
      "rewards/rejected": -1.5100115537643433,
      "step": 1033
    },
    {
      "epoch": 0.13530710722172243,
      "grad_norm": 11.208763122558594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4589633643627167,
      "logits/rejected": 0.32001572847366333,
      "logps/chosen": -445.4408264160156,
      "logps/rejected": -578.06591796875,
      "loss": 0.8966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8578090667724609,
      "rewards/margins": -0.28524816036224365,
      "rewards/rejected": -0.5725609064102173,
      "step": 1034
    },
    {
      "epoch": 0.1354379651590742,
      "grad_norm": 5.839941501617432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31931063532829285,
      "logits/rejected": 0.4282611906528473,
      "logps/chosen": -465.06817626953125,
      "logps/rejected": -614.4853515625,
      "loss": 0.431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6076998114585876,
      "rewards/margins": 0.7815031409263611,
      "rewards/rejected": -1.3892029523849487,
      "step": 1035
    },
    {
      "epoch": 0.13556882309642596,
      "grad_norm": 6.870021820068359,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6636332273483276,
      "logits/rejected": 0.6715163588523865,
      "logps/chosen": -351.26361083984375,
      "logps/rejected": -398.3832092285156,
      "loss": 0.7285,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.56588214635849,
      "rewards/margins": 0.09695012867450714,
      "rewards/rejected": -0.6628322601318359,
      "step": 1036
    },
    {
      "epoch": 0.1356996810337777,
      "grad_norm": 6.225213527679443,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20731845498085022,
      "logits/rejected": 0.47967803478240967,
      "logps/chosen": -440.4134826660156,
      "logps/rejected": -425.8306884765625,
      "loss": 0.6093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9008241891860962,
      "rewards/margins": 0.3086234927177429,
      "rewards/rejected": -1.2094477415084839,
      "step": 1037
    },
    {
      "epoch": 0.13583053897112946,
      "grad_norm": 7.669069766998291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4527204632759094,
      "logits/rejected": 0.40682560205459595,
      "logps/chosen": -411.9756164550781,
      "logps/rejected": -469.3623962402344,
      "loss": 0.6552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5844415426254272,
      "rewards/margins": 0.3148067593574524,
      "rewards/rejected": -0.8992483615875244,
      "step": 1038
    },
    {
      "epoch": 0.13596139690848122,
      "grad_norm": 7.730799198150635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.320128858089447,
      "logits/rejected": 0.07107213884592056,
      "logps/chosen": -331.47393798828125,
      "logps/rejected": -331.99560546875,
      "loss": 0.7137,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6934659481048584,
      "rewards/margins": 0.3752107620239258,
      "rewards/rejected": -1.0686767101287842,
      "step": 1039
    },
    {
      "epoch": 0.13609225484583298,
      "grad_norm": 5.658441543579102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29940909147262573,
      "logits/rejected": 0.425545334815979,
      "logps/chosen": -303.0299987792969,
      "logps/rejected": -320.59600830078125,
      "loss": 0.5223,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34144920110702515,
      "rewards/margins": 0.647912859916687,
      "rewards/rejected": -0.9893620014190674,
      "step": 1040
    },
    {
      "epoch": 0.13622311278318475,
      "grad_norm": 5.507644176483154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24757015705108643,
      "logits/rejected": 0.19637157022953033,
      "logps/chosen": -343.6222229003906,
      "logps/rejected": -468.6717834472656,
      "loss": 0.5388,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8030328750610352,
      "rewards/margins": 0.5855039358139038,
      "rewards/rejected": -1.388536810874939,
      "step": 1041
    },
    {
      "epoch": 0.1363539707205365,
      "grad_norm": 5.605057716369629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4622645676136017,
      "logits/rejected": 0.10587498545646667,
      "logps/chosen": -540.0260620117188,
      "logps/rejected": -292.7799377441406,
      "loss": 0.4363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18011474609375,
      "rewards/margins": 0.8088892698287964,
      "rewards/rejected": -0.9890039563179016,
      "step": 1042
    },
    {
      "epoch": 0.13648482865788827,
      "grad_norm": 7.591958999633789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4616266191005707,
      "logits/rejected": 0.264892041683197,
      "logps/chosen": -495.81365966796875,
      "logps/rejected": -385.5213317871094,
      "loss": 0.7687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.25284308195114136,
      "rewards/margins": 0.01880323886871338,
      "rewards/rejected": -0.27164632081985474,
      "step": 1043
    },
    {
      "epoch": 0.13661568659524004,
      "grad_norm": 6.207342624664307,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2161259651184082,
      "logits/rejected": 0.3648305833339691,
      "logps/chosen": -511.1809387207031,
      "logps/rejected": -377.1685791015625,
      "loss": 0.6729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6732902526855469,
      "rewards/margins": 0.25169867277145386,
      "rewards/rejected": -0.924988865852356,
      "step": 1044
    },
    {
      "epoch": 0.1367465445325918,
      "grad_norm": 5.061804294586182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1414027214050293,
      "logits/rejected": 0.12549938261508942,
      "logps/chosen": -382.1319274902344,
      "logps/rejected": -362.2898254394531,
      "loss": 0.4178,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3655191659927368,
      "rewards/margins": 0.9588123559951782,
      "rewards/rejected": -1.3243314027786255,
      "step": 1045
    },
    {
      "epoch": 0.13687740246994357,
      "grad_norm": 10.001608848571777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23669499158859253,
      "logits/rejected": 0.420698344707489,
      "logps/chosen": -437.64715576171875,
      "logps/rejected": -424.762451171875,
      "loss": 0.7893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.277652382850647,
      "rewards/margins": 0.027196109294891357,
      "rewards/rejected": -1.3048484325408936,
      "step": 1046
    },
    {
      "epoch": 0.13700826040729533,
      "grad_norm": 7.959191799163818,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21084046363830566,
      "logits/rejected": 0.2171250730752945,
      "logps/chosen": -406.59375,
      "logps/rejected": -586.0860595703125,
      "loss": 0.8037,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12350043654441833,
      "rewards/margins": 0.2519164979457855,
      "rewards/rejected": -0.37541693449020386,
      "step": 1047
    },
    {
      "epoch": 0.1371391183446471,
      "grad_norm": 7.216464996337891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1564396470785141,
      "logits/rejected": 0.2709656059741974,
      "logps/chosen": -431.976318359375,
      "logps/rejected": -495.55914306640625,
      "loss": 0.5755,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23607882857322693,
      "rewards/margins": 0.43781355023384094,
      "rewards/rejected": -0.6738923788070679,
      "step": 1048
    },
    {
      "epoch": 0.13726997628199886,
      "grad_norm": 5.6917877197265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2517052888870239,
      "logits/rejected": 0.0513509064912796,
      "logps/chosen": -392.07025146484375,
      "logps/rejected": -366.0294494628906,
      "loss": 0.6602,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6590276956558228,
      "rewards/margins": 0.2239154726266861,
      "rewards/rejected": -0.8829432129859924,
      "step": 1049
    },
    {
      "epoch": 0.13740083421935062,
      "grad_norm": 6.122308731079102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.336104154586792,
      "logits/rejected": 0.3868963122367859,
      "logps/chosen": -463.1116027832031,
      "logps/rejected": -433.1105651855469,
      "loss": 0.6725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3318069875240326,
      "rewards/margins": 0.544712245464325,
      "rewards/rejected": -0.8765192031860352,
      "step": 1050
    },
    {
      "epoch": 0.13753169215670238,
      "grad_norm": 5.505475044250488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22396652400493622,
      "logits/rejected": 0.273159384727478,
      "logps/chosen": -596.9754638671875,
      "logps/rejected": -619.8345947265625,
      "loss": 0.4171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2778475880622864,
      "rewards/margins": 0.8784934282302856,
      "rewards/rejected": -1.1563409566879272,
      "step": 1051
    },
    {
      "epoch": 0.13766255009405415,
      "grad_norm": 9.080044746398926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13602182269096375,
      "logits/rejected": 0.1475566178560257,
      "logps/chosen": -464.1673583984375,
      "logps/rejected": -346.01947021484375,
      "loss": 0.7026,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6357040405273438,
      "rewards/margins": 0.3120207190513611,
      "rewards/rejected": -0.9477246999740601,
      "step": 1052
    },
    {
      "epoch": 0.1377934080314059,
      "grad_norm": 7.039789199829102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1171731948852539,
      "logits/rejected": -0.11391111463308334,
      "logps/chosen": -583.2913818359375,
      "logps/rejected": -377.4644775390625,
      "loss": 0.503,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19882260262966156,
      "rewards/margins": 0.9872177243232727,
      "rewards/rejected": -1.1860404014587402,
      "step": 1053
    },
    {
      "epoch": 0.13792426596875768,
      "grad_norm": 4.832516193389893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.145213782787323,
      "logits/rejected": 0.18033580482006073,
      "logps/chosen": -386.7517395019531,
      "logps/rejected": -475.4393005371094,
      "loss": 0.4056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5382890105247498,
      "rewards/margins": 0.8766720294952393,
      "rewards/rejected": -1.4149610996246338,
      "step": 1054
    },
    {
      "epoch": 0.13805512390610944,
      "grad_norm": 9.434450149536133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5821617245674133,
      "logits/rejected": 0.7919642925262451,
      "logps/chosen": -655.5682983398438,
      "logps/rejected": -392.5494384765625,
      "loss": 0.6403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.979103147983551,
      "rewards/margins": 0.24311676621437073,
      "rewards/rejected": -1.2222198247909546,
      "step": 1055
    },
    {
      "epoch": 0.1381859818434612,
      "grad_norm": 14.838889122009277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4000764489173889,
      "logits/rejected": 0.41338688135147095,
      "logps/chosen": -525.6398315429688,
      "logps/rejected": -432.1575622558594,
      "loss": 1.0522,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1957423686981201,
      "rewards/margins": -0.2377965748310089,
      "rewards/rejected": -0.9579458236694336,
      "step": 1056
    },
    {
      "epoch": 0.13831683978081297,
      "grad_norm": 9.569571495056152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42834973335266113,
      "logits/rejected": 0.3789622187614441,
      "logps/chosen": -469.9700927734375,
      "logps/rejected": -610.9347534179688,
      "loss": 0.9011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1448901891708374,
      "rewards/margins": -0.13124525547027588,
      "rewards/rejected": -1.013645052909851,
      "step": 1057
    },
    {
      "epoch": 0.13844769771816473,
      "grad_norm": 6.965477466583252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41342443227767944,
      "logits/rejected": 0.267085462808609,
      "logps/chosen": -452.1121520996094,
      "logps/rejected": -440.2910461425781,
      "loss": 0.489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4328310191631317,
      "rewards/margins": 0.9654868841171265,
      "rewards/rejected": -1.3983180522918701,
      "step": 1058
    },
    {
      "epoch": 0.13857855565551647,
      "grad_norm": 9.274901390075684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09766978770494461,
      "logits/rejected": 0.03163950517773628,
      "logps/chosen": -418.947998046875,
      "logps/rejected": -476.3730163574219,
      "loss": 0.7037,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1279371976852417,
      "rewards/margins": 0.1190175861120224,
      "rewards/rejected": -1.2469547986984253,
      "step": 1059
    },
    {
      "epoch": 0.13870941359286823,
      "grad_norm": 5.886720180511475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47901827096939087,
      "logits/rejected": 0.23577091097831726,
      "logps/chosen": -641.3441162109375,
      "logps/rejected": -468.01043701171875,
      "loss": 0.382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1807968020439148,
      "rewards/margins": 0.975422739982605,
      "rewards/rejected": -0.7946258783340454,
      "step": 1060
    },
    {
      "epoch": 0.13884027153022,
      "grad_norm": 12.881484031677246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36347877979278564,
      "logits/rejected": 0.25509709119796753,
      "logps/chosen": -527.22802734375,
      "logps/rejected": -542.061767578125,
      "loss": 1.1116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.35786494612693787,
      "rewards/margins": -0.377958208322525,
      "rewards/rejected": 0.0200931578874588,
      "step": 1061
    },
    {
      "epoch": 0.13897112946757176,
      "grad_norm": 5.526782989501953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04261451214551926,
      "logits/rejected": 0.21808120608329773,
      "logps/chosen": -698.5069580078125,
      "logps/rejected": -478.24688720703125,
      "loss": 0.4831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3837606608867645,
      "rewards/margins": 0.6655981540679932,
      "rewards/rejected": -1.04935884475708,
      "step": 1062
    },
    {
      "epoch": 0.13910198740492352,
      "grad_norm": 6.44744873046875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2446277141571045,
      "logits/rejected": 0.1864999681711197,
      "logps/chosen": -541.1040649414062,
      "logps/rejected": -484.2842102050781,
      "loss": 0.4775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.029189303517341614,
      "rewards/margins": 0.7204128503799438,
      "rewards/rejected": -0.749602198600769,
      "step": 1063
    },
    {
      "epoch": 0.13923284534227529,
      "grad_norm": 6.335524082183838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46084070205688477,
      "logits/rejected": 0.4303440451622009,
      "logps/chosen": -530.0802612304688,
      "logps/rejected": -466.4465637207031,
      "loss": 0.5611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09006690979003906,
      "rewards/margins": 0.43915218114852905,
      "rewards/rejected": -0.5292190909385681,
      "step": 1064
    },
    {
      "epoch": 0.13936370327962705,
      "grad_norm": 5.879575729370117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08489567786455154,
      "logits/rejected": 0.019606422632932663,
      "logps/chosen": -347.0702819824219,
      "logps/rejected": -496.2005310058594,
      "loss": 0.6115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29740768671035767,
      "rewards/margins": 0.18646582961082458,
      "rewards/rejected": -0.483873575925827,
      "step": 1065
    },
    {
      "epoch": 0.1394945612169788,
      "grad_norm": 2.7967710494995117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2874174416065216,
      "logits/rejected": 0.1342550367116928,
      "logps/chosen": -390.64117431640625,
      "logps/rejected": -306.80560302734375,
      "loss": 0.4242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2707153558731079,
      "rewards/margins": 0.9264938831329346,
      "rewards/rejected": -1.1972092390060425,
      "step": 1066
    },
    {
      "epoch": 0.13962541915433058,
      "grad_norm": 7.0528388023376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3322140574455261,
      "logits/rejected": 0.47228753566741943,
      "logps/chosen": -387.5663146972656,
      "logps/rejected": -425.1095275878906,
      "loss": 0.6319,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06606101989746094,
      "rewards/margins": 0.36351168155670166,
      "rewards/rejected": -0.29745063185691833,
      "step": 1067
    },
    {
      "epoch": 0.13975627709168234,
      "grad_norm": 4.416810512542725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35599568486213684,
      "logits/rejected": 0.3050270080566406,
      "logps/chosen": -326.0216979980469,
      "logps/rejected": -326.9020690917969,
      "loss": 0.5208,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48559898138046265,
      "rewards/margins": 0.6755044460296631,
      "rewards/rejected": -1.161103367805481,
      "step": 1068
    },
    {
      "epoch": 0.1398871350290341,
      "grad_norm": 4.269905090332031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5576736927032471,
      "logits/rejected": 0.5248836278915405,
      "logps/chosen": -717.6676025390625,
      "logps/rejected": -492.83233642578125,
      "loss": 0.4024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.039544105529785156,
      "rewards/margins": 0.8659597039222717,
      "rewards/rejected": -0.9055038690567017,
      "step": 1069
    },
    {
      "epoch": 0.14001799296638587,
      "grad_norm": 6.51361608505249,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30324721336364746,
      "logits/rejected": 0.19905827939510345,
      "logps/chosen": -550.4783935546875,
      "logps/rejected": -594.9337158203125,
      "loss": 0.5056,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5402050614356995,
      "rewards/margins": 0.7136474251747131,
      "rewards/rejected": -1.2538524866104126,
      "step": 1070
    },
    {
      "epoch": 0.14014885090373763,
      "grad_norm": 3.343520164489746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05941057205200195,
      "logits/rejected": -0.09287223219871521,
      "logps/chosen": -429.046142578125,
      "logps/rejected": -420.9762878417969,
      "loss": 0.3659,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2426902949810028,
      "rewards/margins": 0.9535072445869446,
      "rewards/rejected": -0.7108169794082642,
      "step": 1071
    },
    {
      "epoch": 0.1402797088410894,
      "grad_norm": 9.889904022216797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12104861438274384,
      "logits/rejected": -0.09231042861938477,
      "logps/chosen": -397.56402587890625,
      "logps/rejected": -445.6396789550781,
      "loss": 0.6825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.57955002784729,
      "rewards/margins": 0.6882009506225586,
      "rewards/rejected": -1.2677509784698486,
      "step": 1072
    },
    {
      "epoch": 0.14041056677844116,
      "grad_norm": 5.93983268737793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4381287395954132,
      "logits/rejected": 0.39194318652153015,
      "logps/chosen": -336.1798095703125,
      "logps/rejected": -397.32861328125,
      "loss": 0.6533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6271548867225647,
      "rewards/margins": 0.36312514543533325,
      "rewards/rejected": -0.9902799725532532,
      "step": 1073
    },
    {
      "epoch": 0.14054142471579292,
      "grad_norm": 8.804375648498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16957929730415344,
      "logits/rejected": 0.35056227445602417,
      "logps/chosen": -472.2464599609375,
      "logps/rejected": -518.9014892578125,
      "loss": 0.7742,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6283280849456787,
      "rewards/margins": 0.36422714591026306,
      "rewards/rejected": -0.9925553202629089,
      "step": 1074
    },
    {
      "epoch": 0.1406722826531447,
      "grad_norm": 6.496526718139648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35422345995903015,
      "logits/rejected": 0.5185878872871399,
      "logps/chosen": -328.735107421875,
      "logps/rejected": -552.4781494140625,
      "loss": 0.5813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6591997146606445,
      "rewards/margins": 0.32379329204559326,
      "rewards/rejected": -0.982992947101593,
      "step": 1075
    },
    {
      "epoch": 0.14080314059049645,
      "grad_norm": 6.630079746246338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4542819559574127,
      "logits/rejected": 0.33228734135627747,
      "logps/chosen": -394.12432861328125,
      "logps/rejected": -451.5872802734375,
      "loss": 0.6026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7164852023124695,
      "rewards/margins": 0.42197468876838684,
      "rewards/rejected": -1.1384600400924683,
      "step": 1076
    },
    {
      "epoch": 0.14093399852784821,
      "grad_norm": 5.7618937492370605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.045199695974588394,
      "logits/rejected": 0.14991649985313416,
      "logps/chosen": -487.34307861328125,
      "logps/rejected": -545.4525756835938,
      "loss": 0.4837,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6864204406738281,
      "rewards/margins": 0.6480274200439453,
      "rewards/rejected": -1.3344478607177734,
      "step": 1077
    },
    {
      "epoch": 0.14106485646519998,
      "grad_norm": 5.937338829040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04908367991447449,
      "logits/rejected": 0.021932117640972137,
      "logps/chosen": -702.686279296875,
      "logps/rejected": -540.2160034179688,
      "loss": 0.472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2224416583776474,
      "rewards/margins": 0.6704689264297485,
      "rewards/rejected": -0.44802722334861755,
      "step": 1078
    },
    {
      "epoch": 0.14119571440255174,
      "grad_norm": 6.166459083557129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5041360855102539,
      "logits/rejected": 0.33718907833099365,
      "logps/chosen": -328.1473388671875,
      "logps/rejected": -268.2967529296875,
      "loss": 0.7544,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7213295102119446,
      "rewards/margins": -0.05009383335709572,
      "rewards/rejected": -0.6712356209754944,
      "step": 1079
    },
    {
      "epoch": 0.1413265723399035,
      "grad_norm": 8.574816703796387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14180365204811096,
      "logits/rejected": 0.19209551811218262,
      "logps/chosen": -412.3734130859375,
      "logps/rejected": -413.3617858886719,
      "loss": 0.8701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4773533046245575,
      "rewards/margins": -0.07123470306396484,
      "rewards/rejected": -0.40611857175827026,
      "step": 1080
    },
    {
      "epoch": 0.14145743027725524,
      "grad_norm": 7.255353927612305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4978468120098114,
      "logits/rejected": 0.3856734335422516,
      "logps/chosen": -314.7862243652344,
      "logps/rejected": -309.47113037109375,
      "loss": 0.8046,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2067549228668213,
      "rewards/margins": 0.06600874662399292,
      "rewards/rejected": -1.2727636098861694,
      "step": 1081
    },
    {
      "epoch": 0.141588288214607,
      "grad_norm": 6.068937301635742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13167573511600494,
      "logits/rejected": 0.29870837926864624,
      "logps/chosen": -377.191162109375,
      "logps/rejected": -334.4132995605469,
      "loss": 0.6281,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7341560125350952,
      "rewards/margins": 0.25085848569869995,
      "rewards/rejected": -0.9850145578384399,
      "step": 1082
    },
    {
      "epoch": 0.14171914615195877,
      "grad_norm": 4.705585479736328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41650378704071045,
      "logits/rejected": 0.4826677143573761,
      "logps/chosen": -369.8996276855469,
      "logps/rejected": -324.91009521484375,
      "loss": 0.5738,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16220438480377197,
      "rewards/margins": 0.3914335370063782,
      "rewards/rejected": -0.5536379218101501,
      "step": 1083
    },
    {
      "epoch": 0.14185000408931053,
      "grad_norm": 6.871458053588867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5379002094268799,
      "logits/rejected": 0.4603026807308197,
      "logps/chosen": -373.36590576171875,
      "logps/rejected": -382.7330017089844,
      "loss": 0.6491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9430089592933655,
      "rewards/margins": 0.27146321535110474,
      "rewards/rejected": -1.2144721746444702,
      "step": 1084
    },
    {
      "epoch": 0.1419808620266623,
      "grad_norm": 5.776088714599609,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32246583700180054,
      "logits/rejected": -0.05821939930319786,
      "logps/chosen": -425.8218994140625,
      "logps/rejected": -553.761474609375,
      "loss": 0.4699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25564539432525635,
      "rewards/margins": 0.9437630772590637,
      "rewards/rejected": -1.1994085311889648,
      "step": 1085
    },
    {
      "epoch": 0.14211171996401406,
      "grad_norm": 5.640206336975098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4724649488925934,
      "logits/rejected": 0.3335367441177368,
      "logps/chosen": -343.1234130859375,
      "logps/rejected": -330.2069091796875,
      "loss": 0.6658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3119998872280121,
      "rewards/margins": 0.11961936205625534,
      "rewards/rejected": -0.43161922693252563,
      "step": 1086
    },
    {
      "epoch": 0.14224257790136582,
      "grad_norm": 6.049253463745117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6715950965881348,
      "logits/rejected": 0.40993383526802063,
      "logps/chosen": -476.55926513671875,
      "logps/rejected": -348.62286376953125,
      "loss": 0.5852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3690030872821808,
      "rewards/margins": 0.5242363214492798,
      "rewards/rejected": -0.893239438533783,
      "step": 1087
    },
    {
      "epoch": 0.1423734358387176,
      "grad_norm": 6.162786483764648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17366907000541687,
      "logits/rejected": 0.24361586570739746,
      "logps/chosen": -437.2167663574219,
      "logps/rejected": -443.89410400390625,
      "loss": 0.5917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48656392097473145,
      "rewards/margins": 0.4234886169433594,
      "rewards/rejected": -0.9100525379180908,
      "step": 1088
    },
    {
      "epoch": 0.14250429377606935,
      "grad_norm": 6.037134170532227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22425046563148499,
      "logits/rejected": 0.2751592993736267,
      "logps/chosen": -460.3141174316406,
      "logps/rejected": -439.747314453125,
      "loss": 0.6055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3339958190917969,
      "rewards/margins": 0.8066620230674744,
      "rewards/rejected": -1.1406577825546265,
      "step": 1089
    },
    {
      "epoch": 0.14263515171342112,
      "grad_norm": 7.964056015014648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33438584208488464,
      "logits/rejected": 0.3072170615196228,
      "logps/chosen": -374.8529357910156,
      "logps/rejected": -507.26318359375,
      "loss": 0.7881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4870895743370056,
      "rewards/margins": -0.004325106739997864,
      "rewards/rejected": -0.48276442289352417,
      "step": 1090
    },
    {
      "epoch": 0.14276600965077288,
      "grad_norm": 5.100627899169922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0924011617898941,
      "logits/rejected": -0.04468381032347679,
      "logps/chosen": -317.037353515625,
      "logps/rejected": -488.487060546875,
      "loss": 0.497,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19201087951660156,
      "rewards/margins": 0.500392735004425,
      "rewards/rejected": -0.6924036145210266,
      "step": 1091
    },
    {
      "epoch": 0.14289686758812464,
      "grad_norm": 5.301496505737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3047691583633423,
      "logits/rejected": 0.1649205982685089,
      "logps/chosen": -460.76318359375,
      "logps/rejected": -361.91815185546875,
      "loss": 0.381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10431118309497833,
      "rewards/margins": 0.9485442638397217,
      "rewards/rejected": -1.0528554916381836,
      "step": 1092
    },
    {
      "epoch": 0.1430277255254764,
      "grad_norm": 8.459578514099121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6274799704551697,
      "logits/rejected": 0.48704904317855835,
      "logps/chosen": -537.5741577148438,
      "logps/rejected": -515.8755493164062,
      "loss": 0.6445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4535539746284485,
      "rewards/margins": 0.4539526402950287,
      "rewards/rejected": -0.9075065851211548,
      "step": 1093
    },
    {
      "epoch": 0.14315858346282817,
      "grad_norm": 7.208016872406006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18275287747383118,
      "logits/rejected": 0.1410425305366516,
      "logps/chosen": -646.9564208984375,
      "logps/rejected": -432.97705078125,
      "loss": 0.6534,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2895427942276001,
      "rewards/margins": 0.2352140247821808,
      "rewards/rejected": -0.5247568488121033,
      "step": 1094
    },
    {
      "epoch": 0.14328944140017993,
      "grad_norm": 7.735657691955566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1708221584558487,
      "logits/rejected": 0.1160864606499672,
      "logps/chosen": -266.34490966796875,
      "logps/rejected": -293.3482360839844,
      "loss": 0.7596,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6919261813163757,
      "rewards/margins": 0.0033167824149131775,
      "rewards/rejected": -0.6952430605888367,
      "step": 1095
    },
    {
      "epoch": 0.1434202993375317,
      "grad_norm": 4.541666507720947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3047448992729187,
      "logits/rejected": 0.337686151266098,
      "logps/chosen": -452.3443603515625,
      "logps/rejected": -359.2469482421875,
      "loss": 0.3736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3557407855987549,
      "rewards/margins": 1.003090500831604,
      "rewards/rejected": -1.3588311672210693,
      "step": 1096
    },
    {
      "epoch": 0.14355115727488346,
      "grad_norm": 4.40247106552124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7283808588981628,
      "logits/rejected": 0.4057353436946869,
      "logps/chosen": -539.8699340820312,
      "logps/rejected": -682.6219482421875,
      "loss": 0.4561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5965874195098877,
      "rewards/margins": 0.7305576205253601,
      "rewards/rejected": -1.327144980430603,
      "step": 1097
    },
    {
      "epoch": 0.14368201521223523,
      "grad_norm": 7.938307285308838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28929340839385986,
      "logits/rejected": 0.14703349769115448,
      "logps/chosen": -428.3924560546875,
      "logps/rejected": -525.61279296875,
      "loss": 0.7201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5345237851142883,
      "rewards/margins": 0.5215229988098145,
      "rewards/rejected": -1.056046724319458,
      "step": 1098
    },
    {
      "epoch": 0.143812873149587,
      "grad_norm": 4.027404308319092,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42719727754592896,
      "logits/rejected": 0.5581768751144409,
      "logps/chosen": -486.7650451660156,
      "logps/rejected": -391.1658935546875,
      "loss": 0.347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06502647697925568,
      "rewards/margins": 1.1163090467453003,
      "rewards/rejected": -1.18133544921875,
      "step": 1099
    },
    {
      "epoch": 0.14394373108693875,
      "grad_norm": 11.950050354003906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45582619309425354,
      "logits/rejected": 0.18696527183055878,
      "logps/chosen": -408.8310546875,
      "logps/rejected": -546.0354614257812,
      "loss": 0.9283,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.8068235516548157,
      "rewards/margins": -0.2435096651315689,
      "rewards/rejected": -0.563313901424408,
      "step": 1100
    },
    {
      "epoch": 0.14407458902429052,
      "grad_norm": 6.852621078491211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16267287731170654,
      "logits/rejected": 0.03896098583936691,
      "logps/chosen": -374.68701171875,
      "logps/rejected": -302.9647521972656,
      "loss": 0.5871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.577480673789978,
      "rewards/margins": 0.3106920123100281,
      "rewards/rejected": -0.8881726861000061,
      "step": 1101
    },
    {
      "epoch": 0.14420544696164228,
      "grad_norm": 7.9101481437683105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17472583055496216,
      "logits/rejected": 0.10936081409454346,
      "logps/chosen": -412.5047912597656,
      "logps/rejected": -508.9553527832031,
      "loss": 0.8573,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6250606775283813,
      "rewards/margins": -0.20350611209869385,
      "rewards/rejected": -0.4215545952320099,
      "step": 1102
    },
    {
      "epoch": 0.14433630489899402,
      "grad_norm": 7.4676194190979,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2646508514881134,
      "logits/rejected": 0.12030449509620667,
      "logps/chosen": -440.7403259277344,
      "logps/rejected": -439.139404296875,
      "loss": 0.7648,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.20003661513328552,
      "rewards/margins": 0.2213839292526245,
      "rewards/rejected": -0.42142051458358765,
      "step": 1103
    },
    {
      "epoch": 0.14446716283634578,
      "grad_norm": 5.25122594833374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2577858567237854,
      "logits/rejected": 0.4655647575855255,
      "logps/chosen": -292.6732177734375,
      "logps/rejected": -311.791259765625,
      "loss": 0.4641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3694572448730469,
      "rewards/margins": 0.9367161989212036,
      "rewards/rejected": -1.3061734437942505,
      "step": 1104
    },
    {
      "epoch": 0.14459802077369754,
      "grad_norm": 7.045777320861816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4248558282852173,
      "logits/rejected": 0.3765396475791931,
      "logps/chosen": -467.1496276855469,
      "logps/rejected": -430.14349365234375,
      "loss": 0.5812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5111721158027649,
      "rewards/margins": 0.3712097108364105,
      "rewards/rejected": -0.8823818564414978,
      "step": 1105
    },
    {
      "epoch": 0.1447288787110493,
      "grad_norm": 4.919620513916016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1810876578092575,
      "logits/rejected": 0.17766739428043365,
      "logps/chosen": -417.52655029296875,
      "logps/rejected": -348.529052734375,
      "loss": 0.5467,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.34042128920555115,
      "rewards/margins": 0.6507591009140015,
      "rewards/rejected": -0.3103378415107727,
      "step": 1106
    },
    {
      "epoch": 0.14485973664840107,
      "grad_norm": 5.38695764541626,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1963479369878769,
      "logits/rejected": 0.25677645206451416,
      "logps/chosen": -516.59228515625,
      "logps/rejected": -381.23974609375,
      "loss": 0.397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.48845618963241577,
      "rewards/margins": 0.9055621027946472,
      "rewards/rejected": -0.4171058237552643,
      "step": 1107
    },
    {
      "epoch": 0.14499059458575284,
      "grad_norm": 6.394912242889404,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6450072526931763,
      "logits/rejected": 0.1775265634059906,
      "logps/chosen": -612.9984741210938,
      "logps/rejected": -496.57427978515625,
      "loss": 0.5433,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17147403955459595,
      "rewards/margins": 0.5462585687637329,
      "rewards/rejected": -0.7177326679229736,
      "step": 1108
    },
    {
      "epoch": 0.1451214525231046,
      "grad_norm": 6.995370388031006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028720099478960037,
      "logits/rejected": 0.16224654018878937,
      "logps/chosen": -437.3988342285156,
      "logps/rejected": -521.97509765625,
      "loss": 0.6606,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20086653530597687,
      "rewards/margins": 0.253989577293396,
      "rewards/rejected": -0.45485609769821167,
      "step": 1109
    },
    {
      "epoch": 0.14525231046045636,
      "grad_norm": 4.836155414581299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6008974313735962,
      "logits/rejected": 0.4370405077934265,
      "logps/chosen": -489.86920166015625,
      "logps/rejected": -406.152099609375,
      "loss": 0.4727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009754560887813568,
      "rewards/margins": 0.6998729705810547,
      "rewards/rejected": -0.6901184320449829,
      "step": 1110
    },
    {
      "epoch": 0.14538316839780813,
      "grad_norm": 3.5375149250030518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4569583833217621,
      "logits/rejected": 0.4989170432090759,
      "logps/chosen": -505.57275390625,
      "logps/rejected": -332.354736328125,
      "loss": 0.3312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5857284665107727,
      "rewards/margins": 1.2908482551574707,
      "rewards/rejected": -0.7051196694374084,
      "step": 1111
    },
    {
      "epoch": 0.1455140263351599,
      "grad_norm": 6.421303749084473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27641594409942627,
      "logits/rejected": 0.07454519718885422,
      "logps/chosen": -504.97528076171875,
      "logps/rejected": -490.291015625,
      "loss": 0.5348,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2246488630771637,
      "rewards/margins": 0.7013999819755554,
      "rewards/rejected": -0.4767511188983917,
      "step": 1112
    },
    {
      "epoch": 0.14564488427251165,
      "grad_norm": 4.316538333892822,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2568402588367462,
      "logits/rejected": 0.199843630194664,
      "logps/chosen": -372.15972900390625,
      "logps/rejected": -288.90484619140625,
      "loss": 0.495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24050980806350708,
      "rewards/margins": 0.5061113238334656,
      "rewards/rejected": -0.7466211318969727,
      "step": 1113
    },
    {
      "epoch": 0.14577574220986342,
      "grad_norm": 5.512701034545898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4073643386363983,
      "logits/rejected": 0.4091719090938568,
      "logps/chosen": -276.9488830566406,
      "logps/rejected": -234.0645294189453,
      "loss": 0.5525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3802194595336914,
      "rewards/margins": 0.4457881450653076,
      "rewards/rejected": -0.826007604598999,
      "step": 1114
    },
    {
      "epoch": 0.14590660014721518,
      "grad_norm": 7.183355808258057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5374265313148499,
      "logits/rejected": 0.4439234137535095,
      "logps/chosen": -538.8062744140625,
      "logps/rejected": -567.5424194335938,
      "loss": 0.5265,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08441543579101562,
      "rewards/margins": 0.5065155029296875,
      "rewards/rejected": -0.4221000671386719,
      "step": 1115
    },
    {
      "epoch": 0.14603745808456695,
      "grad_norm": 7.388915538787842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16383402049541473,
      "logits/rejected": -0.11839599162340164,
      "logps/chosen": -514.2415771484375,
      "logps/rejected": -362.2379455566406,
      "loss": 0.7638,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.02069767564535141,
      "rewards/margins": 0.08703671395778656,
      "rewards/rejected": -0.10773439705371857,
      "step": 1116
    },
    {
      "epoch": 0.1461683160219187,
      "grad_norm": 5.845030784606934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12910807132720947,
      "logits/rejected": 0.09657062590122223,
      "logps/chosen": -472.133544921875,
      "logps/rejected": -487.4727783203125,
      "loss": 0.5489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08988971263170242,
      "rewards/margins": 0.397041916847229,
      "rewards/rejected": -0.48693159222602844,
      "step": 1117
    },
    {
      "epoch": 0.14629917395927047,
      "grad_norm": 8.320662498474121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22352272272109985,
      "logits/rejected": 0.22728195786476135,
      "logps/chosen": -355.76904296875,
      "logps/rejected": -320.868896484375,
      "loss": 0.8118,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7441682815551758,
      "rewards/margins": 0.0004975385963916779,
      "rewards/rejected": -0.744665801525116,
      "step": 1118
    },
    {
      "epoch": 0.14643003189662224,
      "grad_norm": 4.1676154136657715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27464503049850464,
      "logits/rejected": 0.2891406714916229,
      "logps/chosen": -313.4901123046875,
      "logps/rejected": -359.57635498046875,
      "loss": 0.4452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.018092334270477295,
      "rewards/margins": 0.8621335029602051,
      "rewards/rejected": -0.8440411686897278,
      "step": 1119
    },
    {
      "epoch": 0.146560889833974,
      "grad_norm": 3.4647884368896484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44084399938583374,
      "logits/rejected": 0.5788986086845398,
      "logps/chosen": -340.7475891113281,
      "logps/rejected": -415.1082763671875,
      "loss": 0.3879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.024189569056034088,
      "rewards/margins": 0.8544561862945557,
      "rewards/rejected": -0.8786457777023315,
      "step": 1120
    },
    {
      "epoch": 0.14669174777132576,
      "grad_norm": 6.184832572937012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23334455490112305,
      "logits/rejected": 0.08746499568223953,
      "logps/chosen": -451.867431640625,
      "logps/rejected": -385.9119873046875,
      "loss": 0.6793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.33512747287750244,
      "rewards/margins": 0.20424193143844604,
      "rewards/rejected": 0.13088546693325043,
      "step": 1121
    },
    {
      "epoch": 0.14682260570867753,
      "grad_norm": 6.871128559112549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039532698690891266,
      "logits/rejected": -0.121932752430439,
      "logps/chosen": -284.05615234375,
      "logps/rejected": -446.5889587402344,
      "loss": 0.6015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48857438564300537,
      "rewards/margins": 0.4669761657714844,
      "rewards/rejected": -0.9555506110191345,
      "step": 1122
    },
    {
      "epoch": 0.1469534636460293,
      "grad_norm": 12.848406791687012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5194859504699707,
      "logits/rejected": 0.3902708888053894,
      "logps/chosen": -543.3807373046875,
      "logps/rejected": -495.82574462890625,
      "loss": 0.6292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11137445271015167,
      "rewards/margins": 0.7621399164199829,
      "rewards/rejected": -0.6507654190063477,
      "step": 1123
    },
    {
      "epoch": 0.14708432158338106,
      "grad_norm": 7.482705116271973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30846524238586426,
      "logits/rejected": 0.23823487758636475,
      "logps/chosen": -355.0257568359375,
      "logps/rejected": -356.0059814453125,
      "loss": 0.5463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3789033591747284,
      "rewards/margins": 0.48862117528915405,
      "rewards/rejected": -0.8675246238708496,
      "step": 1124
    },
    {
      "epoch": 0.1472151795207328,
      "grad_norm": 5.378259181976318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34241461753845215,
      "logits/rejected": 0.20211030542850494,
      "logps/chosen": -575.0925903320312,
      "logps/rejected": -465.9730529785156,
      "loss": 0.5342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18909092247486115,
      "rewards/margins": 0.534210205078125,
      "rewards/rejected": -0.34511926770210266,
      "step": 1125
    },
    {
      "epoch": 0.14734603745808456,
      "grad_norm": 5.1335368156433105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4727663993835449,
      "logits/rejected": 0.3208640217781067,
      "logps/chosen": -547.3991088867188,
      "logps/rejected": -434.8949279785156,
      "loss": 0.3442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17972202599048615,
      "rewards/margins": 1.0711342096328735,
      "rewards/rejected": -0.8914121985435486,
      "step": 1126
    },
    {
      "epoch": 0.14747689539543632,
      "grad_norm": 8.096014022827148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4522708058357239,
      "logits/rejected": 0.43576428294181824,
      "logps/chosen": -608.614013671875,
      "logps/rejected": -785.4381713867188,
      "loss": 0.753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03591117262840271,
      "rewards/margins": 0.027571486309170723,
      "rewards/rejected": 0.008339695632457733,
      "step": 1127
    },
    {
      "epoch": 0.14760775333278808,
      "grad_norm": 4.286124229431152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2526083290576935,
      "logits/rejected": 0.2596207857131958,
      "logps/chosen": -241.8556671142578,
      "logps/rejected": -347.9844970703125,
      "loss": 0.3386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13067969679832458,
      "rewards/margins": 1.1062320470809937,
      "rewards/rejected": -1.2369118928909302,
      "step": 1128
    },
    {
      "epoch": 0.14773861127013985,
      "grad_norm": 10.230550765991211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45595598220825195,
      "logits/rejected": 0.5446059107780457,
      "logps/chosen": -353.85162353515625,
      "logps/rejected": -557.5007934570312,
      "loss": 1.2586,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6489715576171875,
      "rewards/margins": -0.6963505148887634,
      "rewards/rejected": 0.04737892746925354,
      "step": 1129
    },
    {
      "epoch": 0.1478694692074916,
      "grad_norm": 6.742397308349609,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2225286215543747,
      "logits/rejected": 0.11286502331495285,
      "logps/chosen": -355.6346130371094,
      "logps/rejected": -318.2728271484375,
      "loss": 0.635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1103353500366211,
      "rewards/margins": 0.21853047609329224,
      "rewards/rejected": -0.10819512605667114,
      "step": 1130
    },
    {
      "epoch": 0.14800032714484337,
      "grad_norm": 5.09086275100708,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4712769389152527,
      "logits/rejected": 0.4314407706260681,
      "logps/chosen": -672.1015625,
      "logps/rejected": -475.1620178222656,
      "loss": 0.4309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8360443711280823,
      "rewards/margins": 1.0781333446502686,
      "rewards/rejected": -0.24208904802799225,
      "step": 1131
    },
    {
      "epoch": 0.14813118508219514,
      "grad_norm": 5.962331771850586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1254229098558426,
      "logits/rejected": 0.1909874826669693,
      "logps/chosen": -442.454345703125,
      "logps/rejected": -597.0338745117188,
      "loss": 0.3972,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.053662486374378204,
      "rewards/margins": 0.8186672329902649,
      "rewards/rejected": -0.7650046944618225,
      "step": 1132
    },
    {
      "epoch": 0.1482620430195469,
      "grad_norm": 5.16533088684082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2955223023891449,
      "logits/rejected": 0.36168521642684937,
      "logps/chosen": -372.3470153808594,
      "logps/rejected": -321.4659423828125,
      "loss": 0.4503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14428293704986572,
      "rewards/margins": 0.6377201676368713,
      "rewards/rejected": -0.7820031046867371,
      "step": 1133
    },
    {
      "epoch": 0.14839290095689867,
      "grad_norm": 5.947524547576904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5087000727653503,
      "logits/rejected": 0.35627132654190063,
      "logps/chosen": -576.2581176757812,
      "logps/rejected": -375.908203125,
      "loss": 0.4184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20758362114429474,
      "rewards/margins": 1.092203140258789,
      "rewards/rejected": -0.8846195936203003,
      "step": 1134
    },
    {
      "epoch": 0.14852375889425043,
      "grad_norm": 4.9001946449279785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22681951522827148,
      "logits/rejected": 0.02882864698767662,
      "logps/chosen": -435.82330322265625,
      "logps/rejected": -420.8673095703125,
      "loss": 0.515,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0006138086318969727,
      "rewards/margins": 1.16616952419281,
      "rewards/rejected": -1.165555715560913,
      "step": 1135
    },
    {
      "epoch": 0.1486546168316022,
      "grad_norm": 5.789614677429199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5509810447692871,
      "logits/rejected": 0.5040847659111023,
      "logps/chosen": -352.2806091308594,
      "logps/rejected": -282.9089660644531,
      "loss": 0.4517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32101574540138245,
      "rewards/margins": 0.7209230065345764,
      "rewards/rejected": -1.0419386625289917,
      "step": 1136
    },
    {
      "epoch": 0.14878547476895396,
      "grad_norm": 5.642823696136475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5576137900352478,
      "logits/rejected": 0.47280174493789673,
      "logps/chosen": -682.991455078125,
      "logps/rejected": -480.72369384765625,
      "loss": 0.489,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2484094649553299,
      "rewards/margins": 0.7888975739479065,
      "rewards/rejected": -0.5404881238937378,
      "step": 1137
    },
    {
      "epoch": 0.14891633270630572,
      "grad_norm": 7.128389358520508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33304351568222046,
      "logits/rejected": 0.2645586431026459,
      "logps/chosen": -463.65704345703125,
      "logps/rejected": -510.3169860839844,
      "loss": 0.7443,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07259026169776917,
      "rewards/margins": 0.13659249246120453,
      "rewards/rejected": -0.2091827392578125,
      "step": 1138
    },
    {
      "epoch": 0.14904719064365748,
      "grad_norm": 5.250916481018066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2796913683414459,
      "logits/rejected": 0.3200187683105469,
      "logps/chosen": -305.5198059082031,
      "logps/rejected": -349.337646484375,
      "loss": 0.5775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3051311671733856,
      "rewards/margins": 0.30485841631889343,
      "rewards/rejected": -0.6099895238876343,
      "step": 1139
    },
    {
      "epoch": 0.14917804858100925,
      "grad_norm": 6.166243076324463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14899109303951263,
      "logits/rejected": 0.47869873046875,
      "logps/chosen": -369.129638671875,
      "logps/rejected": -533.6571044921875,
      "loss": 0.5282,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3613489270210266,
      "rewards/margins": 0.6451517343521118,
      "rewards/rejected": -1.0065006017684937,
      "step": 1140
    },
    {
      "epoch": 0.149308906518361,
      "grad_norm": 9.025676727294922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09964225441217422,
      "logits/rejected": 0.3222353458404541,
      "logps/chosen": -687.125,
      "logps/rejected": -569.7823486328125,
      "loss": 0.6484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6307485699653625,
      "rewards/margins": 0.38468289375305176,
      "rewards/rejected": -1.0154314041137695,
      "step": 1141
    },
    {
      "epoch": 0.14943976445571278,
      "grad_norm": 12.499412536621094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33008790016174316,
      "logits/rejected": -0.13939644396305084,
      "logps/chosen": -444.1864013671875,
      "logps/rejected": -548.7120361328125,
      "loss": 0.7364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7315649390220642,
      "rewards/margins": 0.45454806089401245,
      "rewards/rejected": -1.1861129999160767,
      "step": 1142
    },
    {
      "epoch": 0.14957062239306454,
      "grad_norm": 7.35903263092041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5357350707054138,
      "logits/rejected": 0.5640150308609009,
      "logps/chosen": -757.8457641601562,
      "logps/rejected": -554.2803344726562,
      "loss": 0.6488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.033829860389232635,
      "rewards/margins": 0.2754989564418793,
      "rewards/rejected": -0.3093288242816925,
      "step": 1143
    },
    {
      "epoch": 0.1497014803304163,
      "grad_norm": 4.6425628662109375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4867744445800781,
      "logits/rejected": 0.4152134656906128,
      "logps/chosen": -350.0075988769531,
      "logps/rejected": -377.2970886230469,
      "loss": 0.39,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20067311823368073,
      "rewards/margins": 0.9700594544410706,
      "rewards/rejected": -1.1707326173782349,
      "step": 1144
    },
    {
      "epoch": 0.14983233826776807,
      "grad_norm": 6.581291675567627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17411747574806213,
      "logits/rejected": 0.24018941819667816,
      "logps/chosen": -485.70343017578125,
      "logps/rejected": -614.8404541015625,
      "loss": 0.5733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08587665855884552,
      "rewards/margins": 0.36726054549217224,
      "rewards/rejected": -0.2813839018344879,
      "step": 1145
    },
    {
      "epoch": 0.1499631962051198,
      "grad_norm": 7.774603843688965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6595529317855835,
      "logits/rejected": 0.5983150601387024,
      "logps/chosen": -462.2170715332031,
      "logps/rejected": -444.0718078613281,
      "loss": 0.6796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09136393666267395,
      "rewards/margins": 0.2415846437215805,
      "rewards/rejected": -0.15022072196006775,
      "step": 1146
    },
    {
      "epoch": 0.15009405414247157,
      "grad_norm": 4.2021284103393555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3382159471511841,
      "logits/rejected": 0.5549712181091309,
      "logps/chosen": -437.1804504394531,
      "logps/rejected": -485.8747863769531,
      "loss": 0.3966,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3759777247905731,
      "rewards/margins": 0.9978294968605042,
      "rewards/rejected": -0.6218518018722534,
      "step": 1147
    },
    {
      "epoch": 0.15022491207982333,
      "grad_norm": 4.702795028686523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6813784241676331,
      "logits/rejected": 0.39620280265808105,
      "logps/chosen": -490.4763488769531,
      "logps/rejected": -503.0699462890625,
      "loss": 0.3937,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10495568811893463,
      "rewards/margins": 0.9710474610328674,
      "rewards/rejected": -0.8660917282104492,
      "step": 1148
    },
    {
      "epoch": 0.1503557700171751,
      "grad_norm": 14.042829513549805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48421213030815125,
      "logits/rejected": 0.4170202910900116,
      "logps/chosen": -385.1295166015625,
      "logps/rejected": -420.0986328125,
      "loss": 0.7021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3801460564136505,
      "rewards/margins": 0.2811294496059418,
      "rewards/rejected": -0.6612755060195923,
      "step": 1149
    },
    {
      "epoch": 0.15048662795452686,
      "grad_norm": 5.837584018707275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16990326344966888,
      "logits/rejected": 0.22221443057060242,
      "logps/chosen": -413.1341552734375,
      "logps/rejected": -438.2785339355469,
      "loss": 0.4095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5552583932876587,
      "rewards/margins": 1.1618186235427856,
      "rewards/rejected": -1.7170770168304443,
      "step": 1150
    },
    {
      "epoch": 0.15061748589187862,
      "grad_norm": 9.01370620727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43771231174468994,
      "logits/rejected": 0.3215293884277344,
      "logps/chosen": -580.3466796875,
      "logps/rejected": -403.2593994140625,
      "loss": 0.838,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.49074745178222656,
      "rewards/margins": 0.009185045957565308,
      "rewards/rejected": -0.4999324679374695,
      "step": 1151
    },
    {
      "epoch": 0.15074834382923039,
      "grad_norm": 5.90054178237915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26108503341674805,
      "logits/rejected": -0.0029076822102069855,
      "logps/chosen": -457.0483703613281,
      "logps/rejected": -488.0186767578125,
      "loss": 0.5689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03456893563270569,
      "rewards/margins": 0.6119476556777954,
      "rewards/rejected": -0.6465166807174683,
      "step": 1152
    },
    {
      "epoch": 0.15087920176658215,
      "grad_norm": 6.574345588684082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6575735807418823,
      "logits/rejected": 0.38404321670532227,
      "logps/chosen": -434.3280334472656,
      "logps/rejected": -292.25048828125,
      "loss": 0.6193,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.04628334194421768,
      "rewards/margins": 0.24029645323753357,
      "rewards/rejected": -0.19401313364505768,
      "step": 1153
    },
    {
      "epoch": 0.1510100597039339,
      "grad_norm": 7.857641696929932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23320263624191284,
      "logits/rejected": 0.08875750005245209,
      "logps/chosen": -396.33880615234375,
      "logps/rejected": -332.78692626953125,
      "loss": 0.8219,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9634550213813782,
      "rewards/margins": -0.027149967849254608,
      "rewards/rejected": -0.9363051056861877,
      "step": 1154
    },
    {
      "epoch": 0.15114091764128568,
      "grad_norm": 4.010681629180908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3562624752521515,
      "logits/rejected": 0.1522468775510788,
      "logps/chosen": -302.9396057128906,
      "logps/rejected": -320.4288330078125,
      "loss": 0.4642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3723836839199066,
      "rewards/margins": 0.6371666193008423,
      "rewards/rejected": -1.0095503330230713,
      "step": 1155
    },
    {
      "epoch": 0.15127177557863744,
      "grad_norm": 7.4000067710876465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2602953314781189,
      "logits/rejected": 0.34147173166275024,
      "logps/chosen": -373.5447998046875,
      "logps/rejected": -425.99609375,
      "loss": 0.636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.009543895721435547,
      "rewards/margins": 0.3302115201950073,
      "rewards/rejected": -0.3206676244735718,
      "step": 1156
    },
    {
      "epoch": 0.1514026335159892,
      "grad_norm": 4.430837631225586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19911614060401917,
      "logits/rejected": 0.17092153429985046,
      "logps/chosen": -415.5804138183594,
      "logps/rejected": -293.9383544921875,
      "loss": 0.4117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1249362975358963,
      "rewards/margins": 0.9183370471000671,
      "rewards/rejected": -1.0432734489440918,
      "step": 1157
    },
    {
      "epoch": 0.15153349145334097,
      "grad_norm": 6.7430739402771,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06230998784303665,
      "logits/rejected": -0.032695259898900986,
      "logps/chosen": -548.7384643554688,
      "logps/rejected": -528.9866333007812,
      "loss": 0.5484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4905546307563782,
      "rewards/margins": 0.47456175088882446,
      "rewards/rejected": -0.9651163220405579,
      "step": 1158
    },
    {
      "epoch": 0.15166434939069273,
      "grad_norm": 5.237030506134033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37603867053985596,
      "logits/rejected": 0.1896160989999771,
      "logps/chosen": -656.2510986328125,
      "logps/rejected": -541.9931640625,
      "loss": 0.3747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.009834486991167068,
      "rewards/margins": 1.115564227104187,
      "rewards/rejected": -1.1253985166549683,
      "step": 1159
    },
    {
      "epoch": 0.1517952073280445,
      "grad_norm": 4.276772975921631,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41222766041755676,
      "logits/rejected": 0.1704103797674179,
      "logps/chosen": -409.9907531738281,
      "logps/rejected": -348.57781982421875,
      "loss": 0.3652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35279351472854614,
      "rewards/margins": 1.1346691846847534,
      "rewards/rejected": -0.7818756103515625,
      "step": 1160
    },
    {
      "epoch": 0.15192606526539626,
      "grad_norm": 4.59049654006958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3175559639930725,
      "logits/rejected": 0.3429361581802368,
      "logps/chosen": -376.9267272949219,
      "logps/rejected": -336.66680908203125,
      "loss": 0.4062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27874279022216797,
      "rewards/margins": 0.9127691984176636,
      "rewards/rejected": -0.6340264081954956,
      "step": 1161
    },
    {
      "epoch": 0.15205692320274802,
      "grad_norm": 8.526896476745605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.7788782715797424,
      "logits/rejected": 0.7106539011001587,
      "logps/chosen": -460.5459289550781,
      "logps/rejected": -584.2034912109375,
      "loss": 0.7533,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34351426362991333,
      "rewards/margins": 0.1844940334558487,
      "rewards/rejected": -0.5280082821846008,
      "step": 1162
    },
    {
      "epoch": 0.1521877811400998,
      "grad_norm": 7.560666084289551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19153612852096558,
      "logits/rejected": 0.11988342553377151,
      "logps/chosen": -396.659912109375,
      "logps/rejected": -385.9311218261719,
      "loss": 0.8434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.38021886348724365,
      "rewards/margins": 0.04405820369720459,
      "rewards/rejected": 0.33616065979003906,
      "step": 1163
    },
    {
      "epoch": 0.15231863907745155,
      "grad_norm": 4.783008098602295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47268667817115784,
      "logits/rejected": 0.42709851264953613,
      "logps/chosen": -387.7359619140625,
      "logps/rejected": -329.3575134277344,
      "loss": 0.3904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24671286344528198,
      "rewards/margins": 0.9600958824157715,
      "rewards/rejected": -1.2068086862564087,
      "step": 1164
    },
    {
      "epoch": 0.15244949701480331,
      "grad_norm": 5.4452009201049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018285736441612244,
      "logits/rejected": 0.058549754321575165,
      "logps/chosen": -282.9578857421875,
      "logps/rejected": -301.4190673828125,
      "loss": 0.5365,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.015684321522712708,
      "rewards/margins": 0.504075825214386,
      "rewards/rejected": -0.5197601318359375,
      "step": 1165
    },
    {
      "epoch": 0.15258035495215508,
      "grad_norm": 6.683312892913818,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3087265193462372,
      "logits/rejected": 0.39930760860443115,
      "logps/chosen": -678.396728515625,
      "logps/rejected": -399.7440185546875,
      "loss": 0.4281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.482552707195282,
      "rewards/margins": 0.861321210861206,
      "rewards/rejected": -0.37876853346824646,
      "step": 1166
    },
    {
      "epoch": 0.15271121288950684,
      "grad_norm": 7.162235260009766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2529808282852173,
      "logits/rejected": 0.12850627303123474,
      "logps/chosen": -436.43743896484375,
      "logps/rejected": -573.2444458007812,
      "loss": 0.5893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6315790414810181,
      "rewards/margins": 0.3117770850658417,
      "rewards/rejected": -0.9433561563491821,
      "step": 1167
    },
    {
      "epoch": 0.15284207082685858,
      "grad_norm": 10.591219902038574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18871794641017914,
      "logits/rejected": 0.3164973556995392,
      "logps/chosen": -668.92724609375,
      "logps/rejected": -571.4490356445312,
      "loss": 0.9255,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.3354555368423462,
      "rewards/margins": -0.041993916034698486,
      "rewards/rejected": 0.3774494230747223,
      "step": 1168
    },
    {
      "epoch": 0.15297292876421034,
      "grad_norm": 6.106741428375244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17626544833183289,
      "logits/rejected": 0.3928837776184082,
      "logps/chosen": -403.7666015625,
      "logps/rejected": -271.57940673828125,
      "loss": 0.5849,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2708732485771179,
      "rewards/margins": 0.470397412776947,
      "rewards/rejected": -0.7412706613540649,
      "step": 1169
    },
    {
      "epoch": 0.1531037867015621,
      "grad_norm": 6.413876533508301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1991472840309143,
      "logits/rejected": 0.3578842282295227,
      "logps/chosen": -521.7709350585938,
      "logps/rejected": -550.5194091796875,
      "loss": 0.4006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5193445086479187,
      "rewards/margins": 1.0381197929382324,
      "rewards/rejected": -0.518775224685669,
      "step": 1170
    },
    {
      "epoch": 0.15323464463891387,
      "grad_norm": 6.878713130950928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2429049015045166,
      "logits/rejected": 0.39446792006492615,
      "logps/chosen": -506.96783447265625,
      "logps/rejected": -359.582275390625,
      "loss": 0.6445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02142982929944992,
      "rewards/margins": 0.28462064266204834,
      "rewards/rejected": -0.30605047941207886,
      "step": 1171
    },
    {
      "epoch": 0.15336550257626563,
      "grad_norm": 3.649951696395874,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6219438910484314,
      "logits/rejected": 0.6053329706192017,
      "logps/chosen": -369.3594665527344,
      "logps/rejected": -386.07574462890625,
      "loss": 0.3587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09680099785327911,
      "rewards/margins": 1.0308308601379395,
      "rewards/rejected": -0.9340299963951111,
      "step": 1172
    },
    {
      "epoch": 0.1534963605136174,
      "grad_norm": 4.711677074432373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2070261389017105,
      "logits/rejected": 0.31364867091178894,
      "logps/chosen": -432.5718078613281,
      "logps/rejected": -422.087646484375,
      "loss": 0.5099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09979000687599182,
      "rewards/margins": 0.5607149004936218,
      "rewards/rejected": -0.660504937171936,
      "step": 1173
    },
    {
      "epoch": 0.15362721845096916,
      "grad_norm": 5.870359420776367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16563886404037476,
      "logits/rejected": 0.06014445051550865,
      "logps/chosen": -403.731201171875,
      "logps/rejected": -393.939697265625,
      "loss": 0.5837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8241106271743774,
      "rewards/margins": 0.47680437564849854,
      "rewards/rejected": -1.3009148836135864,
      "step": 1174
    },
    {
      "epoch": 0.15375807638832092,
      "grad_norm": 4.160248279571533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3163166344165802,
      "logits/rejected": 0.2037210762500763,
      "logps/chosen": -314.0182800292969,
      "logps/rejected": -323.8865051269531,
      "loss": 0.3885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3660200238227844,
      "rewards/margins": 1.3957831859588623,
      "rewards/rejected": -1.0297629833221436,
      "step": 1175
    },
    {
      "epoch": 0.1538889343256727,
      "grad_norm": 5.680543899536133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5258384943008423,
      "logits/rejected": 0.46242812275886536,
      "logps/chosen": -646.852783203125,
      "logps/rejected": -493.40191650390625,
      "loss": 0.4905,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3726745843887329,
      "rewards/margins": 0.6152200698852539,
      "rewards/rejected": -0.2425455003976822,
      "step": 1176
    },
    {
      "epoch": 0.15401979226302445,
      "grad_norm": 7.241481781005859,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17073902487754822,
      "logits/rejected": 0.37058353424072266,
      "logps/chosen": -256.32293701171875,
      "logps/rejected": -431.7208557128906,
      "loss": 0.5554,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2051895260810852,
      "rewards/margins": 0.6546001434326172,
      "rewards/rejected": -0.8597896695137024,
      "step": 1177
    },
    {
      "epoch": 0.15415065020037622,
      "grad_norm": 8.768957138061523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6121928691864014,
      "logits/rejected": 0.5683158040046692,
      "logps/chosen": -506.5939636230469,
      "logps/rejected": -459.52978515625,
      "loss": 0.8854,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.21979787945747375,
      "rewards/margins": -0.3014245629310608,
      "rewards/rejected": 0.08162671327590942,
      "step": 1178
    },
    {
      "epoch": 0.15428150813772798,
      "grad_norm": 4.454387187957764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38963666558265686,
      "logits/rejected": 0.3860298991203308,
      "logps/chosen": -435.6300048828125,
      "logps/rejected": -351.56390380859375,
      "loss": 0.4288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09861326217651367,
      "rewards/margins": 0.8864879608154297,
      "rewards/rejected": -0.9851011633872986,
      "step": 1179
    },
    {
      "epoch": 0.15441236607507974,
      "grad_norm": 5.622021675109863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37950465083122253,
      "logits/rejected": 0.30877113342285156,
      "logps/chosen": -385.3066101074219,
      "logps/rejected": -340.4613037109375,
      "loss": 0.6303,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10618020594120026,
      "rewards/margins": 0.3239252269268036,
      "rewards/rejected": -0.21774503588676453,
      "step": 1180
    },
    {
      "epoch": 0.1545432240124315,
      "grad_norm": 6.1017985343933105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2599896192550659,
      "logits/rejected": 0.2180522382259369,
      "logps/chosen": -361.941162109375,
      "logps/rejected": -364.67645263671875,
      "loss": 0.6645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.015445340424776077,
      "rewards/margins": 0.5250778198242188,
      "rewards/rejected": -0.5096325278282166,
      "step": 1181
    },
    {
      "epoch": 0.15467408194978327,
      "grad_norm": 6.65336799621582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08330047130584717,
      "logits/rejected": 0.10804182291030884,
      "logps/chosen": -544.877197265625,
      "logps/rejected": -463.9837646484375,
      "loss": 0.5659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10516330599784851,
      "rewards/margins": 0.4581654667854309,
      "rewards/rejected": -0.3530021905899048,
      "step": 1182
    },
    {
      "epoch": 0.15480493988713503,
      "grad_norm": 6.697859764099121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0030712205916643143,
      "logits/rejected": -0.007612135261297226,
      "logps/chosen": -395.5787353515625,
      "logps/rejected": -417.7703552246094,
      "loss": 0.6618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19878235459327698,
      "rewards/margins": 0.2840862274169922,
      "rewards/rejected": -0.4828685522079468,
      "step": 1183
    },
    {
      "epoch": 0.1549357978244868,
      "grad_norm": 6.845030784606934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3166819214820862,
      "logits/rejected": 0.31379905343055725,
      "logps/chosen": -333.0691223144531,
      "logps/rejected": -366.073486328125,
      "loss": 0.6402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5861080288887024,
      "rewards/margins": 0.26216280460357666,
      "rewards/rejected": -0.8482708930969238,
      "step": 1184
    },
    {
      "epoch": 0.15506665576183856,
      "grad_norm": 7.4208149909973145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.437859445810318,
      "logits/rejected": -0.004391789436340332,
      "logps/chosen": -356.6042175292969,
      "logps/rejected": -617.0715942382812,
      "loss": 0.6729,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.518817126750946,
      "rewards/margins": 0.6498317718505859,
      "rewards/rejected": -1.1686488389968872,
      "step": 1185
    },
    {
      "epoch": 0.15519751369919033,
      "grad_norm": 5.010304927825928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37768426537513733,
      "logits/rejected": 0.4580221474170685,
      "logps/chosen": -341.850830078125,
      "logps/rejected": -365.5005798339844,
      "loss": 0.7115,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.42103251814842224,
      "rewards/margins": 0.33854448795318604,
      "rewards/rejected": 0.08248808979988098,
      "step": 1186
    },
    {
      "epoch": 0.1553283716365421,
      "grad_norm": 4.71647310256958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26002171635627747,
      "logits/rejected": 0.36968013644218445,
      "logps/chosen": -437.47955322265625,
      "logps/rejected": -472.6266174316406,
      "loss": 0.4701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09400921314954758,
      "rewards/margins": 0.7993789911270142,
      "rewards/rejected": -0.7053697109222412,
      "step": 1187
    },
    {
      "epoch": 0.15545922957389385,
      "grad_norm": 5.644755840301514,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3911507725715637,
      "logits/rejected": 0.428972989320755,
      "logps/chosen": -446.5688171386719,
      "logps/rejected": -448.32012939453125,
      "loss": 0.4978,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0557745099067688,
      "rewards/margins": 0.5573453903198242,
      "rewards/rejected": -0.5015708804130554,
      "step": 1188
    },
    {
      "epoch": 0.15559008751124562,
      "grad_norm": 5.991391658782959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24389906227588654,
      "logits/rejected": 0.2734273672103882,
      "logps/chosen": -501.42633056640625,
      "logps/rejected": -357.1161193847656,
      "loss": 0.5793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022590458393096924,
      "rewards/margins": 0.6539072394371033,
      "rewards/rejected": -0.6313167810440063,
      "step": 1189
    },
    {
      "epoch": 0.15572094544859735,
      "grad_norm": 12.667750358581543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3807685375213623,
      "logits/rejected": 0.19168153405189514,
      "logps/chosen": -514.5780639648438,
      "logps/rejected": -381.4866943359375,
      "loss": 0.9367,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.857660174369812,
      "rewards/margins": -0.2963779866695404,
      "rewards/rejected": -0.5612821578979492,
      "step": 1190
    },
    {
      "epoch": 0.15585180338594912,
      "grad_norm": 8.411389350891113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12324284762144089,
      "logits/rejected": 0.046799737960100174,
      "logps/chosen": -560.6409912109375,
      "logps/rejected": -481.5481262207031,
      "loss": 0.6719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3052934408187866,
      "rewards/margins": 0.29979729652404785,
      "rewards/rejected": 0.005496203899383545,
      "step": 1191
    },
    {
      "epoch": 0.15598266132330088,
      "grad_norm": 7.155730247497559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3826492428779602,
      "logits/rejected": 0.17289239168167114,
      "logps/chosen": -556.9550170898438,
      "logps/rejected": -516.74267578125,
      "loss": 0.6709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20082971453666687,
      "rewards/margins": 0.2883649468421936,
      "rewards/rejected": -0.48919469118118286,
      "step": 1192
    },
    {
      "epoch": 0.15611351926065264,
      "grad_norm": 5.274278163909912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027774643152952194,
      "logits/rejected": 0.2659074366092682,
      "logps/chosen": -485.0701904296875,
      "logps/rejected": -476.8106994628906,
      "loss": 0.4491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16895200312137604,
      "rewards/margins": 0.7550458312034607,
      "rewards/rejected": -0.5860939621925354,
      "step": 1193
    },
    {
      "epoch": 0.1562443771980044,
      "grad_norm": 7.859689235687256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29180124402046204,
      "logits/rejected": 0.43150824308395386,
      "logps/chosen": -421.01629638671875,
      "logps/rejected": -477.2947998046875,
      "loss": 0.7141,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.61773681640625,
      "rewards/margins": 0.09566191583871841,
      "rewards/rejected": -0.713398814201355,
      "step": 1194
    },
    {
      "epoch": 0.15637523513535617,
      "grad_norm": 10.32332992553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33923742175102234,
      "logits/rejected": 0.17392055690288544,
      "logps/chosen": -574.902587890625,
      "logps/rejected": -550.6290283203125,
      "loss": 0.9764,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14144057035446167,
      "rewards/margins": -0.0054469965398311615,
      "rewards/rejected": -0.13599355518817902,
      "step": 1195
    },
    {
      "epoch": 0.15650609307270794,
      "grad_norm": 6.97359561920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5972500443458557,
      "logits/rejected": 0.36861535906791687,
      "logps/chosen": -599.917236328125,
      "logps/rejected": -535.018798828125,
      "loss": 0.4897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5783497095108032,
      "rewards/margins": 0.6103651523590088,
      "rewards/rejected": -1.1887147426605225,
      "step": 1196
    },
    {
      "epoch": 0.1566369510100597,
      "grad_norm": 7.7675557136535645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23223350942134857,
      "logits/rejected": 0.3045373558998108,
      "logps/chosen": -306.02008056640625,
      "logps/rejected": -351.96856689453125,
      "loss": 0.3832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03609009087085724,
      "rewards/margins": 1.106570839881897,
      "rewards/rejected": -1.1426608562469482,
      "step": 1197
    },
    {
      "epoch": 0.15676780894741146,
      "grad_norm": 5.961371898651123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28087303042411804,
      "logits/rejected": 0.35234782099723816,
      "logps/chosen": -421.69134521484375,
      "logps/rejected": -390.921630859375,
      "loss": 0.6251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09717482328414917,
      "rewards/margins": 0.4483291506767273,
      "rewards/rejected": -0.3511543273925781,
      "step": 1198
    },
    {
      "epoch": 0.15689866688476323,
      "grad_norm": 4.5592942237854,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24511846899986267,
      "logits/rejected": 0.2794269919395447,
      "logps/chosen": -500.77349853515625,
      "logps/rejected": -438.04931640625,
      "loss": 0.3668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12851183116436005,
      "rewards/margins": 0.9296776652336121,
      "rewards/rejected": -0.8011658787727356,
      "step": 1199
    },
    {
      "epoch": 0.157029524822115,
      "grad_norm": 5.192087173461914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3413243293762207,
      "logits/rejected": 0.15329095721244812,
      "logps/chosen": -414.4118347167969,
      "logps/rejected": -519.2476806640625,
      "loss": 0.3999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1975761353969574,
      "rewards/margins": 1.1902707815170288,
      "rewards/rejected": -1.3878467082977295,
      "step": 1200
    },
    {
      "epoch": 0.15716038275946675,
      "grad_norm": 7.184663772583008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14915478229522705,
      "logits/rejected": 0.035644881427288055,
      "logps/chosen": -512.9595336914062,
      "logps/rejected": -522.670654296875,
      "loss": 0.5007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8492592573165894,
      "rewards/margins": 0.7248951196670532,
      "rewards/rejected": -1.5741544961929321,
      "step": 1201
    },
    {
      "epoch": 0.15729124069681852,
      "grad_norm": 7.021354675292969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38598164916038513,
      "logits/rejected": 0.2377316951751709,
      "logps/chosen": -355.5419616699219,
      "logps/rejected": -414.4962463378906,
      "loss": 0.9065,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6486813426017761,
      "rewards/margins": -0.2591409683227539,
      "rewards/rejected": -0.38954028487205505,
      "step": 1202
    },
    {
      "epoch": 0.15742209863417028,
      "grad_norm": 7.115454196929932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38253870606422424,
      "logits/rejected": 0.3911585807800293,
      "logps/chosen": -416.80511474609375,
      "logps/rejected": -470.2713623046875,
      "loss": 0.6021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44681209325790405,
      "rewards/margins": 0.6268815994262695,
      "rewards/rejected": -1.0736936330795288,
      "step": 1203
    },
    {
      "epoch": 0.15755295657152205,
      "grad_norm": 4.187056064605713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3748106062412262,
      "logits/rejected": 0.36445873975753784,
      "logps/chosen": -450.0724792480469,
      "logps/rejected": -277.1742858886719,
      "loss": 0.4374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6047697067260742,
      "rewards/margins": 0.6849384903907776,
      "rewards/rejected": -1.2897082567214966,
      "step": 1204
    },
    {
      "epoch": 0.1576838145088738,
      "grad_norm": 9.257389068603516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04338919371366501,
      "logits/rejected": 0.05898774042725563,
      "logps/chosen": -415.34674072265625,
      "logps/rejected": -420.48345947265625,
      "loss": 0.7095,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3314037024974823,
      "rewards/margins": 0.2026604413986206,
      "rewards/rejected": -0.5340641140937805,
      "step": 1205
    },
    {
      "epoch": 0.15781467244622557,
      "grad_norm": 15.398764610290527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28252971172332764,
      "logits/rejected": 0.28437724709510803,
      "logps/chosen": -514.6822509765625,
      "logps/rejected": -431.4690246582031,
      "loss": 1.013,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47652381658554077,
      "rewards/margins": -0.2657986283302307,
      "rewards/rejected": -0.21072521805763245,
      "step": 1206
    },
    {
      "epoch": 0.15794553038357734,
      "grad_norm": 5.723148345947266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24604710936546326,
      "logits/rejected": 0.3303879499435425,
      "logps/chosen": -341.8118591308594,
      "logps/rejected": -335.1640625,
      "loss": 0.5722,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1941431164741516,
      "rewards/margins": 0.643684983253479,
      "rewards/rejected": -0.4495418667793274,
      "step": 1207
    },
    {
      "epoch": 0.1580763883209291,
      "grad_norm": 5.934060096740723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3483719825744629,
      "logits/rejected": 0.2701377868652344,
      "logps/chosen": -572.0316772460938,
      "logps/rejected": -462.3459167480469,
      "loss": 0.4678,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32712897658348083,
      "rewards/margins": 0.6264337301254272,
      "rewards/rejected": -0.9535627365112305,
      "step": 1208
    },
    {
      "epoch": 0.15820724625828086,
      "grad_norm": 6.968308448791504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15742893517017365,
      "logits/rejected": 0.04714946076273918,
      "logps/chosen": -412.1483459472656,
      "logps/rejected": -394.5047912597656,
      "loss": 0.5894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4203571677207947,
      "rewards/margins": 0.358847051858902,
      "rewards/rejected": -0.7792041301727295,
      "step": 1209
    },
    {
      "epoch": 0.15833810419563263,
      "grad_norm": 5.346187114715576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2226504683494568,
      "logits/rejected": 0.26884549856185913,
      "logps/chosen": -238.0161590576172,
      "logps/rejected": -309.0597839355469,
      "loss": 0.4754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36801132559776306,
      "rewards/margins": 0.7047901153564453,
      "rewards/rejected": -1.0728013515472412,
      "step": 1210
    },
    {
      "epoch": 0.1584689621329844,
      "grad_norm": 5.37034273147583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2511410415172577,
      "logits/rejected": 0.23549041152000427,
      "logps/chosen": -482.38018798828125,
      "logps/rejected": -447.7897033691406,
      "loss": 0.4455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10038566589355469,
      "rewards/margins": 0.7246875762939453,
      "rewards/rejected": -0.8250733017921448,
      "step": 1211
    },
    {
      "epoch": 0.15859982007033613,
      "grad_norm": 5.233062267303467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30852019786834717,
      "logits/rejected": 0.18140970170497894,
      "logps/chosen": -531.585205078125,
      "logps/rejected": -516.9779052734375,
      "loss": 0.431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.032826781272888184,
      "rewards/margins": 0.9498788118362427,
      "rewards/rejected": -0.9170520305633545,
      "step": 1212
    },
    {
      "epoch": 0.1587306780076879,
      "grad_norm": 4.7763142585754395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1509646624326706,
      "logits/rejected": 0.23410075902938843,
      "logps/chosen": -590.7910766601562,
      "logps/rejected": -483.5418395996094,
      "loss": 0.3743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13082335889339447,
      "rewards/margins": 1.0408177375793457,
      "rewards/rejected": -1.1716411113739014,
      "step": 1213
    },
    {
      "epoch": 0.15886153594503966,
      "grad_norm": 8.539769172668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15662193298339844,
      "logits/rejected": 0.30162620544433594,
      "logps/chosen": -537.9569702148438,
      "logps/rejected": -499.3026123046875,
      "loss": 0.5576,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6706279516220093,
      "rewards/margins": 0.8123350143432617,
      "rewards/rejected": -1.4829630851745605,
      "step": 1214
    },
    {
      "epoch": 0.15899239388239142,
      "grad_norm": 5.561347961425781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21495769917964935,
      "logits/rejected": 0.37835565209388733,
      "logps/chosen": -556.5303955078125,
      "logps/rejected": -558.4956665039062,
      "loss": 0.5336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10604172945022583,
      "rewards/margins": 0.4091501533985138,
      "rewards/rejected": -0.5151918530464172,
      "step": 1215
    },
    {
      "epoch": 0.15912325181974318,
      "grad_norm": 4.50929069519043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01698516681790352,
      "logits/rejected": -0.047701720148324966,
      "logps/chosen": -301.7804260253906,
      "logps/rejected": -463.7042541503906,
      "loss": 0.4137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24745100736618042,
      "rewards/margins": 0.7775705456733704,
      "rewards/rejected": -1.0250215530395508,
      "step": 1216
    },
    {
      "epoch": 0.15925410975709495,
      "grad_norm": 8.28783893585205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39483150839805603,
      "logits/rejected": 0.2648018002510071,
      "logps/chosen": -474.3662414550781,
      "logps/rejected": -358.9247741699219,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17688465118408203,
      "rewards/margins": 0.2275266796350479,
      "rewards/rejected": -0.40441128611564636,
      "step": 1217
    },
    {
      "epoch": 0.1593849676944467,
      "grad_norm": 7.901874542236328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5707244873046875,
      "logits/rejected": 0.09506908059120178,
      "logps/chosen": -624.6050415039062,
      "logps/rejected": -690.6087036132812,
      "loss": 0.6443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0015716254711151123,
      "rewards/margins": 0.6173273324966431,
      "rewards/rejected": -0.6188989877700806,
      "step": 1218
    },
    {
      "epoch": 0.15951582563179847,
      "grad_norm": 5.837120056152344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3075914978981018,
      "logits/rejected": 0.10010271519422531,
      "logps/chosen": -456.9879150390625,
      "logps/rejected": -428.5779113769531,
      "loss": 0.5383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6349664926528931,
      "rewards/margins": 0.4318149983882904,
      "rewards/rejected": -1.0667814016342163,
      "step": 1219
    },
    {
      "epoch": 0.15964668356915024,
      "grad_norm": 4.632948875427246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3493986427783966,
      "logits/rejected": 0.48865947127342224,
      "logps/chosen": -462.58489990234375,
      "logps/rejected": -392.6474304199219,
      "loss": 0.4295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02454642951488495,
      "rewards/margins": 0.8793224096298218,
      "rewards/rejected": -0.9038689136505127,
      "step": 1220
    },
    {
      "epoch": 0.159777541506502,
      "grad_norm": 5.410385608673096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35739269852638245,
      "logits/rejected": 0.1815701574087143,
      "logps/chosen": -392.64129638671875,
      "logps/rejected": -387.3461608886719,
      "loss": 0.4478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07698507606983185,
      "rewards/margins": 1.0123857259750366,
      "rewards/rejected": -0.935400664806366,
      "step": 1221
    },
    {
      "epoch": 0.15990839944385377,
      "grad_norm": 8.691167831420898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.469999223947525,
      "logits/rejected": 0.6017217636108398,
      "logps/chosen": -423.636474609375,
      "logps/rejected": -449.1759033203125,
      "loss": 0.7366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7195614576339722,
      "rewards/margins": 0.38254278898239136,
      "rewards/rejected": -1.1021041870117188,
      "step": 1222
    },
    {
      "epoch": 0.16003925738120553,
      "grad_norm": 9.770511627197266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5003527402877808,
      "logits/rejected": 0.2787220776081085,
      "logps/chosen": -563.7507934570312,
      "logps/rejected": -565.1990966796875,
      "loss": 0.8283,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6427286267280579,
      "rewards/margins": -0.020077478140592575,
      "rewards/rejected": -0.6226511001586914,
      "step": 1223
    },
    {
      "epoch": 0.1601701153185573,
      "grad_norm": 5.657323837280273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38414984941482544,
      "logits/rejected": 0.6443625092506409,
      "logps/chosen": -401.81427001953125,
      "logps/rejected": -543.676513671875,
      "loss": 0.4478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14820481836795807,
      "rewards/margins": 0.9855901598930359,
      "rewards/rejected": -1.1337950229644775,
      "step": 1224
    },
    {
      "epoch": 0.16030097325590906,
      "grad_norm": 4.1863322257995605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2553570866584778,
      "logits/rejected": 0.4428299069404602,
      "logps/chosen": -577.3302001953125,
      "logps/rejected": -407.30572509765625,
      "loss": 0.3087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18327578902244568,
      "rewards/margins": 1.1771841049194336,
      "rewards/rejected": -1.3604599237442017,
      "step": 1225
    },
    {
      "epoch": 0.16043183119326082,
      "grad_norm": 4.805008888244629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4987682104110718,
      "logits/rejected": 0.456169068813324,
      "logps/chosen": -434.4134216308594,
      "logps/rejected": -369.794677734375,
      "loss": 0.4459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21459409594535828,
      "rewards/margins": 0.6492232084274292,
      "rewards/rejected": -0.4346291124820709,
      "step": 1226
    },
    {
      "epoch": 0.16056268913061258,
      "grad_norm": 5.039410591125488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41114938259124756,
      "logits/rejected": 0.5232440829277039,
      "logps/chosen": -481.0313720703125,
      "logps/rejected": -355.87225341796875,
      "loss": 0.5199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41170620918273926,
      "rewards/margins": 0.780204176902771,
      "rewards/rejected": -1.1919102668762207,
      "step": 1227
    },
    {
      "epoch": 0.16069354706796435,
      "grad_norm": 7.467446804046631,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42022475600242615,
      "logits/rejected": 0.22391366958618164,
      "logps/chosen": -466.144287109375,
      "logps/rejected": -387.1808166503906,
      "loss": 0.6625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32047978043556213,
      "rewards/margins": 0.35503214597702026,
      "rewards/rejected": -0.67551189661026,
      "step": 1228
    },
    {
      "epoch": 0.1608244050053161,
      "grad_norm": 8.576502799987793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1067027598619461,
      "logits/rejected": 0.2600889503955841,
      "logps/chosen": -408.7159118652344,
      "logps/rejected": -513.3369750976562,
      "loss": 0.7075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8212307095527649,
      "rewards/margins": 0.28788721561431885,
      "rewards/rejected": -1.1091179847717285,
      "step": 1229
    },
    {
      "epoch": 0.16095526294266788,
      "grad_norm": 5.548731803894043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39703047275543213,
      "logits/rejected": 0.12949007749557495,
      "logps/chosen": -839.008544921875,
      "logps/rejected": -544.6514892578125,
      "loss": 0.4621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4296785593032837,
      "rewards/margins": 1.0538519620895386,
      "rewards/rejected": -0.6241734027862549,
      "step": 1230
    },
    {
      "epoch": 0.16108612088001964,
      "grad_norm": 5.792769432067871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33757930994033813,
      "logits/rejected": 0.3610643148422241,
      "logps/chosen": -521.2608642578125,
      "logps/rejected": -464.04156494140625,
      "loss": 0.5581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20104825496673584,
      "rewards/margins": 0.6057338714599609,
      "rewards/rejected": -0.8067821860313416,
      "step": 1231
    },
    {
      "epoch": 0.1612169788173714,
      "grad_norm": 6.148103713989258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21915896236896515,
      "logits/rejected": 0.18043449521064758,
      "logps/chosen": -364.2510070800781,
      "logps/rejected": -452.3653259277344,
      "loss": 0.6013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6478193402290344,
      "rewards/margins": 0.2790992558002472,
      "rewards/rejected": -0.9269186854362488,
      "step": 1232
    },
    {
      "epoch": 0.16134783675472317,
      "grad_norm": 5.385531425476074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19696614146232605,
      "logits/rejected": 0.45984700322151184,
      "logps/chosen": -393.2675476074219,
      "logps/rejected": -375.33203125,
      "loss": 0.6252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4339004456996918,
      "rewards/margins": 0.3388238847255707,
      "rewards/rejected": -0.7727243900299072,
      "step": 1233
    },
    {
      "epoch": 0.1614786946920749,
      "grad_norm": 6.74316930770874,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3800657093524933,
      "logits/rejected": 0.4615252614021301,
      "logps/chosen": -555.79736328125,
      "logps/rejected": -557.0860595703125,
      "loss": 0.5424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015456181019544601,
      "rewards/margins": 0.6333050727844238,
      "rewards/rejected": -0.6178487539291382,
      "step": 1234
    },
    {
      "epoch": 0.16160955262942667,
      "grad_norm": 6.675403118133545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18480825424194336,
      "logits/rejected": 0.37588298320770264,
      "logps/chosen": -492.8954162597656,
      "logps/rejected": -434.51129150390625,
      "loss": 0.5616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.502676784992218,
      "rewards/margins": 0.3418954610824585,
      "rewards/rejected": -0.8445723056793213,
      "step": 1235
    },
    {
      "epoch": 0.16174041056677843,
      "grad_norm": 5.565422058105469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0623849593102932,
      "logits/rejected": 0.06390770524740219,
      "logps/chosen": -327.103515625,
      "logps/rejected": -412.1851806640625,
      "loss": 0.4566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04443950951099396,
      "rewards/margins": 0.7579145431518555,
      "rewards/rejected": -0.802354097366333,
      "step": 1236
    },
    {
      "epoch": 0.1618712685041302,
      "grad_norm": 7.250511646270752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01691335439682007,
      "logits/rejected": 0.35941702127456665,
      "logps/chosen": -456.2374572753906,
      "logps/rejected": -480.96966552734375,
      "loss": 0.5231,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46032392978668213,
      "rewards/margins": 0.6612873077392578,
      "rewards/rejected": -1.12161123752594,
      "step": 1237
    },
    {
      "epoch": 0.16200212644148196,
      "grad_norm": 5.4495673179626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3717959225177765,
      "logits/rejected": 0.3021400570869446,
      "logps/chosen": -480.15069580078125,
      "logps/rejected": -399.2034912109375,
      "loss": 0.4329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6235793828964233,
      "rewards/margins": 0.7559699416160583,
      "rewards/rejected": -1.379549503326416,
      "step": 1238
    },
    {
      "epoch": 0.16213298437883372,
      "grad_norm": 6.527831077575684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31799596548080444,
      "logits/rejected": -0.054310984909534454,
      "logps/chosen": -470.0558776855469,
      "logps/rejected": -426.9502258300781,
      "loss": 0.6876,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9302173852920532,
      "rewards/margins": 0.11095292866230011,
      "rewards/rejected": -1.041170358657837,
      "step": 1239
    },
    {
      "epoch": 0.16226384231618549,
      "grad_norm": 4.591166973114014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35940438508987427,
      "logits/rejected": 0.4201919734477997,
      "logps/chosen": -480.26007080078125,
      "logps/rejected": -334.5738525390625,
      "loss": 0.4299,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.41122496128082275,
      "rewards/margins": 1.1661498546600342,
      "rewards/rejected": -0.7549249529838562,
      "step": 1240
    },
    {
      "epoch": 0.16239470025353725,
      "grad_norm": 7.214548587799072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21033687889575958,
      "logits/rejected": 0.372833788394928,
      "logps/chosen": -617.2793579101562,
      "logps/rejected": -446.3951416015625,
      "loss": 0.7937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7642083764076233,
      "rewards/margins": 0.28191831707954407,
      "rewards/rejected": -1.0461268424987793,
      "step": 1241
    },
    {
      "epoch": 0.162525558190889,
      "grad_norm": 6.024286270141602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07203260064125061,
      "logits/rejected": -0.047942377626895905,
      "logps/chosen": -763.5040283203125,
      "logps/rejected": -545.8392944335938,
      "loss": 0.5305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3235929608345032,
      "rewards/margins": 0.4415174722671509,
      "rewards/rejected": -0.11792450398206711,
      "step": 1242
    },
    {
      "epoch": 0.16265641612824078,
      "grad_norm": 5.762986660003662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43203628063201904,
      "logits/rejected": 0.37446096539497375,
      "logps/chosen": -662.1011352539062,
      "logps/rejected": -462.970947265625,
      "loss": 0.4317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.41211411356925964,
      "rewards/margins": 0.8209438920021057,
      "rewards/rejected": -0.40882983803749084,
      "step": 1243
    },
    {
      "epoch": 0.16278727406559254,
      "grad_norm": 7.20638370513916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3662899434566498,
      "logits/rejected": -0.031887833029031754,
      "logps/chosen": -519.3592529296875,
      "logps/rejected": -472.40777587890625,
      "loss": 0.6426,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44029921293258667,
      "rewards/margins": 0.5245487093925476,
      "rewards/rejected": -0.964847981929779,
      "step": 1244
    },
    {
      "epoch": 0.1629181320029443,
      "grad_norm": 7.805674076080322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6704086065292358,
      "logits/rejected": 0.5696216225624084,
      "logps/chosen": -399.886474609375,
      "logps/rejected": -367.615478515625,
      "loss": 0.7442,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0533058643341064,
      "rewards/margins": 0.19937562942504883,
      "rewards/rejected": -1.2526816129684448,
      "step": 1245
    },
    {
      "epoch": 0.16304898994029607,
      "grad_norm": 5.80177116394043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48918616771698,
      "logits/rejected": 0.5287668108940125,
      "logps/chosen": -421.3914489746094,
      "logps/rejected": -460.3016662597656,
      "loss": 0.4654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21298560500144958,
      "rewards/margins": 0.6410501599311829,
      "rewards/rejected": -0.8540357947349548,
      "step": 1246
    },
    {
      "epoch": 0.16317984787764783,
      "grad_norm": 6.327793121337891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3570076525211334,
      "logits/rejected": 0.4862220287322998,
      "logps/chosen": -427.7510986328125,
      "logps/rejected": -437.9864807128906,
      "loss": 0.6141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.057721905410289764,
      "rewards/margins": 0.36784934997558594,
      "rewards/rejected": -0.4255712330341339,
      "step": 1247
    },
    {
      "epoch": 0.1633107058149996,
      "grad_norm": 5.424379825592041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47574207186698914,
      "logits/rejected": 0.4827355444431305,
      "logps/chosen": -589.892578125,
      "logps/rejected": -513.4240112304688,
      "loss": 0.4119,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5151956677436829,
      "rewards/margins": 1.406084656715393,
      "rewards/rejected": -0.8908889889717102,
      "step": 1248
    },
    {
      "epoch": 0.16344156375235136,
      "grad_norm": 6.156889915466309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2515101432800293,
      "logits/rejected": 0.2716311812400818,
      "logps/chosen": -479.01959228515625,
      "logps/rejected": -410.9371337890625,
      "loss": 0.4253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1911270171403885,
      "rewards/margins": 1.0550737380981445,
      "rewards/rejected": -1.2462009191513062,
      "step": 1249
    },
    {
      "epoch": 0.16357242168970312,
      "grad_norm": 4.190683364868164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3912869989871979,
      "logits/rejected": 0.3954579830169678,
      "logps/chosen": -503.7442321777344,
      "logps/rejected": -451.4964904785156,
      "loss": 0.3686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.701306939125061,
      "rewards/margins": 1.0968029499053955,
      "rewards/rejected": -0.3954959809780121,
      "step": 1250
    },
    {
      "epoch": 0.1637032796270549,
      "grad_norm": 5.896333694458008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.360594779253006,
      "logits/rejected": 0.35810354351997375,
      "logps/chosen": -439.209228515625,
      "logps/rejected": -475.88018798828125,
      "loss": 0.4869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3076186776161194,
      "rewards/margins": 0.7712187170982361,
      "rewards/rejected": -1.0788373947143555,
      "step": 1251
    },
    {
      "epoch": 0.16383413756440665,
      "grad_norm": 5.678394317626953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5402612686157227,
      "logits/rejected": 0.6404037475585938,
      "logps/chosen": -526.634521484375,
      "logps/rejected": -391.85302734375,
      "loss": 0.5184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10011003911495209,
      "rewards/margins": 1.1505308151245117,
      "rewards/rejected": -1.0504207611083984,
      "step": 1252
    },
    {
      "epoch": 0.16396499550175841,
      "grad_norm": 6.3424859046936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3140225112438202,
      "logits/rejected": 0.5853803157806396,
      "logps/chosen": -546.0278930664062,
      "logps/rejected": -993.9812622070312,
      "loss": 0.4114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09079322963953018,
      "rewards/margins": 0.8200645446777344,
      "rewards/rejected": -0.9108577370643616,
      "step": 1253
    },
    {
      "epoch": 0.16409585343911018,
      "grad_norm": 5.501661777496338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4759702682495117,
      "logits/rejected": 0.38014495372772217,
      "logps/chosen": -392.01611328125,
      "logps/rejected": -385.6724853515625,
      "loss": 0.5025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.054444119334220886,
      "rewards/margins": 0.5729833841323853,
      "rewards/rejected": -0.6274274587631226,
      "step": 1254
    },
    {
      "epoch": 0.16422671137646194,
      "grad_norm": 7.9453558921813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08150963485240936,
      "logits/rejected": -0.062130410224199295,
      "logps/chosen": -668.0546875,
      "logps/rejected": -600.744140625,
      "loss": 0.4946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.050284773111343384,
      "rewards/margins": 0.9835150241851807,
      "rewards/rejected": -0.9332301616668701,
      "step": 1255
    },
    {
      "epoch": 0.16435756931381368,
      "grad_norm": 3.679263114929199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37232843041419983,
      "logits/rejected": 0.2986338436603546,
      "logps/chosen": -413.673828125,
      "logps/rejected": -432.56658935546875,
      "loss": 0.4295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14288979768753052,
      "rewards/margins": 0.8079996109008789,
      "rewards/rejected": -0.6651098728179932,
      "step": 1256
    },
    {
      "epoch": 0.16448842725116544,
      "grad_norm": 7.633252143859863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5232369303703308,
      "logits/rejected": 0.6480036973953247,
      "logps/chosen": -555.7692260742188,
      "logps/rejected": -543.6369018554688,
      "loss": 0.6846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18749657273292542,
      "rewards/margins": 0.5176117420196533,
      "rewards/rejected": -0.3301151990890503,
      "step": 1257
    },
    {
      "epoch": 0.1646192851885172,
      "grad_norm": 5.908275604248047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46795177459716797,
      "logits/rejected": 0.49231618642807007,
      "logps/chosen": -351.82940673828125,
      "logps/rejected": -365.0764465332031,
      "loss": 0.4562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.023698031902313232,
      "rewards/margins": 0.8085651397705078,
      "rewards/rejected": -0.7848671078681946,
      "step": 1258
    },
    {
      "epoch": 0.16475014312586897,
      "grad_norm": 5.4663166999816895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4189307689666748,
      "logits/rejected": 0.38940566778182983,
      "logps/chosen": -599.9552001953125,
      "logps/rejected": -498.1238098144531,
      "loss": 0.5352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43846508860588074,
      "rewards/margins": 0.8085060119628906,
      "rewards/rejected": -0.3700409233570099,
      "step": 1259
    },
    {
      "epoch": 0.16488100106322073,
      "grad_norm": 7.8345417976379395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.459552139043808,
      "logits/rejected": 0.3830718994140625,
      "logps/chosen": -473.711181640625,
      "logps/rejected": -535.04736328125,
      "loss": 0.5954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08239155262708664,
      "rewards/margins": 0.3539237976074219,
      "rewards/rejected": -0.27153223752975464,
      "step": 1260
    },
    {
      "epoch": 0.1650118590005725,
      "grad_norm": 8.635229110717773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3366239666938782,
      "logits/rejected": 0.22246485948562622,
      "logps/chosen": -494.6300964355469,
      "logps/rejected": -513.0736694335938,
      "loss": 0.8415,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.3275318443775177,
      "rewards/margins": -0.01579001545906067,
      "rewards/rejected": -0.3117418587207794,
      "step": 1261
    },
    {
      "epoch": 0.16514271693792426,
      "grad_norm": 8.259124755859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12615329027175903,
      "logits/rejected": 0.04858168587088585,
      "logps/chosen": -429.7297058105469,
      "logps/rejected": -428.5669250488281,
      "loss": 0.8091,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5758212208747864,
      "rewards/margins": -0.034856006503105164,
      "rewards/rejected": -0.5409650802612305,
      "step": 1262
    },
    {
      "epoch": 0.16527357487527602,
      "grad_norm": 7.198568344116211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3324487805366516,
      "logits/rejected": 0.4026057720184326,
      "logps/chosen": -497.1443786621094,
      "logps/rejected": -511.33807373046875,
      "loss": 0.6761,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04589194059371948,
      "rewards/margins": 0.5105167627334595,
      "rewards/rejected": -0.5564086437225342,
      "step": 1263
    },
    {
      "epoch": 0.1654044328126278,
      "grad_norm": 7.353696346282959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3796588182449341,
      "logits/rejected": 0.444733589887619,
      "logps/chosen": -398.4316711425781,
      "logps/rejected": -333.79534912109375,
      "loss": 0.5978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4605415463447571,
      "rewards/margins": 0.35703104734420776,
      "rewards/rejected": -0.8175725936889648,
      "step": 1264
    },
    {
      "epoch": 0.16553529074997955,
      "grad_norm": 6.946878910064697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12063665688037872,
      "logits/rejected": -0.043659526854753494,
      "logps/chosen": -494.91107177734375,
      "logps/rejected": -489.6282043457031,
      "loss": 0.7077,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2225072681903839,
      "rewards/margins": 0.39983826875686646,
      "rewards/rejected": -0.17733100056648254,
      "step": 1265
    },
    {
      "epoch": 0.16566614868733132,
      "grad_norm": 4.25194787979126,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05341134965419769,
      "logits/rejected": -0.0025934111326932907,
      "logps/chosen": -250.08279418945312,
      "logps/rejected": -290.8846740722656,
      "loss": 0.5134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13055384159088135,
      "rewards/margins": 0.5385007858276367,
      "rewards/rejected": -0.6690545678138733,
      "step": 1266
    },
    {
      "epoch": 0.16579700662468308,
      "grad_norm": 6.850473403930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00025153905153274536,
      "logits/rejected": 0.24976080656051636,
      "logps/chosen": -398.2414245605469,
      "logps/rejected": -473.82525634765625,
      "loss": 0.5696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5233400464057922,
      "rewards/margins": 0.3877995014190674,
      "rewards/rejected": -0.9111395478248596,
      "step": 1267
    },
    {
      "epoch": 0.16592786456203484,
      "grad_norm": 4.081303596496582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2699592411518097,
      "logits/rejected": 0.3617461323738098,
      "logps/chosen": -519.9833984375,
      "logps/rejected": -532.3284301757812,
      "loss": 0.3471,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21767579019069672,
      "rewards/margins": 1.1035982370376587,
      "rewards/rejected": -0.885922372341156,
      "step": 1268
    },
    {
      "epoch": 0.1660587224993866,
      "grad_norm": 4.052979469299316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22717466950416565,
      "logits/rejected": 0.4885559678077698,
      "logps/chosen": -399.5633239746094,
      "logps/rejected": -387.88006591796875,
      "loss": 0.3241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21372050046920776,
      "rewards/margins": 1.268439531326294,
      "rewards/rejected": -1.0547189712524414,
      "step": 1269
    },
    {
      "epoch": 0.16618958043673837,
      "grad_norm": 6.900609016418457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45291879773139954,
      "logits/rejected": 0.46232956647872925,
      "logps/chosen": -552.222412109375,
      "logps/rejected": -501.051025390625,
      "loss": 0.5306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4853155016899109,
      "rewards/margins": 0.802787184715271,
      "rewards/rejected": -1.2881027460098267,
      "step": 1270
    },
    {
      "epoch": 0.16632043837409013,
      "grad_norm": 4.24949836730957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4090992510318756,
      "logits/rejected": 0.2845360040664673,
      "logps/chosen": -477.34661865234375,
      "logps/rejected": -380.4486389160156,
      "loss": 0.4895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24667534232139587,
      "rewards/margins": 0.6866294741630554,
      "rewards/rejected": -0.4399542510509491,
      "step": 1271
    },
    {
      "epoch": 0.1664512963114419,
      "grad_norm": 5.43436336517334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2571052610874176,
      "logits/rejected": -0.00017379224300384521,
      "logps/chosen": -440.7887268066406,
      "logps/rejected": -361.48785400390625,
      "loss": 0.5044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6044660210609436,
      "rewards/margins": 0.5988632440567017,
      "rewards/rejected": -1.20332932472229,
      "step": 1272
    },
    {
      "epoch": 0.16658215424879366,
      "grad_norm": 6.651472568511963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33769115805625916,
      "logits/rejected": 0.456924170255661,
      "logps/chosen": -404.516357421875,
      "logps/rejected": -480.59185791015625,
      "loss": 0.8028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5670757293701172,
      "rewards/margins": 0.05970136076211929,
      "rewards/rejected": -0.6267771124839783,
      "step": 1273
    },
    {
      "epoch": 0.16671301218614543,
      "grad_norm": 10.150586128234863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1447891741991043,
      "logits/rejected": -0.05189505219459534,
      "logps/chosen": -447.7135009765625,
      "logps/rejected": -366.0702209472656,
      "loss": 0.8291,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7493002414703369,
      "rewards/margins": 0.046666160225868225,
      "rewards/rejected": -0.795966386795044,
      "step": 1274
    },
    {
      "epoch": 0.1668438701234972,
      "grad_norm": 6.717390060424805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12222795188426971,
      "logits/rejected": 0.3300964832305908,
      "logps/chosen": -437.54205322265625,
      "logps/rejected": -462.1613464355469,
      "loss": 0.7269,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17513523995876312,
      "rewards/margins": 0.15867233276367188,
      "rewards/rejected": -0.3338075280189514,
      "step": 1275
    },
    {
      "epoch": 0.16697472806084895,
      "grad_norm": 5.362870693206787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10036749392747879,
      "logits/rejected": 0.0451749712228775,
      "logps/chosen": -515.3988647460938,
      "logps/rejected": -347.6269836425781,
      "loss": 0.4967,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29004234075546265,
      "rewards/margins": 0.6461746096611023,
      "rewards/rejected": -0.9362169504165649,
      "step": 1276
    },
    {
      "epoch": 0.1671055859982007,
      "grad_norm": 8.22146224975586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4233207404613495,
      "logits/rejected": 0.37084826827049255,
      "logps/chosen": -475.6465759277344,
      "logps/rejected": -454.97918701171875,
      "loss": 0.5531,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6077255010604858,
      "rewards/margins": 0.7208973169326782,
      "rewards/rejected": -1.328622817993164,
      "step": 1277
    },
    {
      "epoch": 0.16723644393555245,
      "grad_norm": 5.678837299346924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2542341947555542,
      "logits/rejected": 0.11733356863260269,
      "logps/chosen": -427.2438049316406,
      "logps/rejected": -426.8126525878906,
      "loss": 0.5106,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22587059438228607,
      "rewards/margins": 0.6149803996086121,
      "rewards/rejected": -0.3891097903251648,
      "step": 1278
    },
    {
      "epoch": 0.16736730187290422,
      "grad_norm": 5.354800224304199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5221077799797058,
      "logits/rejected": 0.4142569601535797,
      "logps/chosen": -416.8439636230469,
      "logps/rejected": -367.1920166015625,
      "loss": 0.3994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.262564480304718,
      "rewards/margins": 0.9459501504898071,
      "rewards/rejected": -1.2085145711898804,
      "step": 1279
    },
    {
      "epoch": 0.16749815981025598,
      "grad_norm": 6.6083221435546875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28748971223831177,
      "logits/rejected": 0.3071616291999817,
      "logps/chosen": -469.21160888671875,
      "logps/rejected": -449.7794494628906,
      "loss": 0.4874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3968757390975952,
      "rewards/margins": 0.6039159893989563,
      "rewards/rejected": -1.0007916688919067,
      "step": 1280
    },
    {
      "epoch": 0.16762901774760774,
      "grad_norm": 5.482837200164795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4189966917037964,
      "logits/rejected": 0.34316158294677734,
      "logps/chosen": -445.7204284667969,
      "logps/rejected": -393.3356628417969,
      "loss": 0.5455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6530902981758118,
      "rewards/margins": 0.5617042183876038,
      "rewards/rejected": -1.214794635772705,
      "step": 1281
    },
    {
      "epoch": 0.1677598756849595,
      "grad_norm": 4.756669044494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030655378475785255,
      "logits/rejected": 0.09394174069166183,
      "logps/chosen": -313.88934326171875,
      "logps/rejected": -453.24578857421875,
      "loss": 0.4612,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4064714312553406,
      "rewards/margins": 0.7957124710083008,
      "rewards/rejected": -1.2021839618682861,
      "step": 1282
    },
    {
      "epoch": 0.16789073362231127,
      "grad_norm": 7.07202672958374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4944242537021637,
      "logits/rejected": 0.40256741642951965,
      "logps/chosen": -379.1819763183594,
      "logps/rejected": -289.2228698730469,
      "loss": 0.5537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6600240468978882,
      "rewards/margins": 0.5853981971740723,
      "rewards/rejected": -1.245422124862671,
      "step": 1283
    },
    {
      "epoch": 0.16802159155966304,
      "grad_norm": 5.761295795440674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20060992240905762,
      "logits/rejected": 0.09480226039886475,
      "logps/chosen": -318.4014892578125,
      "logps/rejected": -434.4974060058594,
      "loss": 0.5402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2127992808818817,
      "rewards/margins": 0.6038376092910767,
      "rewards/rejected": -0.816636860370636,
      "step": 1284
    },
    {
      "epoch": 0.1681524494970148,
      "grad_norm": 6.6749067306518555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38044270873069763,
      "logits/rejected": 0.25940167903900146,
      "logps/chosen": -430.03094482421875,
      "logps/rejected": -422.37457275390625,
      "loss": 0.6384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40124648809432983,
      "rewards/margins": 0.41058745980262756,
      "rewards/rejected": -0.8118339776992798,
      "step": 1285
    },
    {
      "epoch": 0.16828330743436656,
      "grad_norm": 7.212172985076904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5545464754104614,
      "logits/rejected": 0.33506399393081665,
      "logps/chosen": -334.6357727050781,
      "logps/rejected": -324.0272521972656,
      "loss": 0.6789,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6902538537979126,
      "rewards/margins": 0.21486720442771912,
      "rewards/rejected": -0.9051210880279541,
      "step": 1286
    },
    {
      "epoch": 0.16841416537171833,
      "grad_norm": 5.1012492179870605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44462305307388306,
      "logits/rejected": 0.27932029962539673,
      "logps/chosen": -437.9771423339844,
      "logps/rejected": -396.8984375,
      "loss": 0.4487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16613729298114777,
      "rewards/margins": 0.7789984345436096,
      "rewards/rejected": -0.6128610968589783,
      "step": 1287
    },
    {
      "epoch": 0.1685450233090701,
      "grad_norm": 5.067662715911865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5765621066093445,
      "logits/rejected": 0.5204575657844543,
      "logps/chosen": -381.40960693359375,
      "logps/rejected": -290.3114929199219,
      "loss": 0.4629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3543931841850281,
      "rewards/margins": 0.6515283584594727,
      "rewards/rejected": -1.0059216022491455,
      "step": 1288
    },
    {
      "epoch": 0.16867588124642185,
      "grad_norm": 6.363711833953857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4958133399486542,
      "logits/rejected": 0.4575510025024414,
      "logps/chosen": -755.8220825195312,
      "logps/rejected": -618.8872680664062,
      "loss": 0.5522,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18056946992874146,
      "rewards/margins": 0.4900533854961395,
      "rewards/rejected": -0.3094838857650757,
      "step": 1289
    },
    {
      "epoch": 0.16880673918377362,
      "grad_norm": 7.241693019866943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003403458744287491,
      "logits/rejected": 0.06875424087047577,
      "logps/chosen": -311.53253173828125,
      "logps/rejected": -440.5450134277344,
      "loss": 0.6756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22805233299732208,
      "rewards/margins": 0.38545843958854675,
      "rewards/rejected": -0.61351078748703,
      "step": 1290
    },
    {
      "epoch": 0.16893759712112538,
      "grad_norm": 6.235871315002441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3812274634838104,
      "logits/rejected": 0.6010349988937378,
      "logps/chosen": -315.95556640625,
      "logps/rejected": -289.1291809082031,
      "loss": 0.6688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27720412611961365,
      "rewards/margins": 0.347708523273468,
      "rewards/rejected": -0.624912679195404,
      "step": 1291
    },
    {
      "epoch": 0.16906845505847715,
      "grad_norm": 5.007917404174805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2893040180206299,
      "logits/rejected": 0.2008601725101471,
      "logps/chosen": -639.9434204101562,
      "logps/rejected": -400.7460632324219,
      "loss": 0.3921,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1494196057319641,
      "rewards/margins": 1.1391559839248657,
      "rewards/rejected": -0.9897363781929016,
      "step": 1292
    },
    {
      "epoch": 0.1691993129958289,
      "grad_norm": 8.235940933227539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1255284994840622,
      "logits/rejected": -0.23105967044830322,
      "logps/chosen": -508.78839111328125,
      "logps/rejected": -498.90252685546875,
      "loss": 0.6137,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02149031311273575,
      "rewards/margins": 0.8352306485176086,
      "rewards/rejected": -0.8567209243774414,
      "step": 1293
    },
    {
      "epoch": 0.16933017093318067,
      "grad_norm": 7.627959728240967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4314453601837158,
      "logits/rejected": 0.41695570945739746,
      "logps/chosen": -448.4929504394531,
      "logps/rejected": -511.59185791015625,
      "loss": 0.7686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13960133492946625,
      "rewards/margins": 0.20757389068603516,
      "rewards/rejected": -0.0679725855588913,
      "step": 1294
    },
    {
      "epoch": 0.16946102887053244,
      "grad_norm": 6.972244739532471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4490235149860382,
      "logits/rejected": 0.3247337341308594,
      "logps/chosen": -321.8530578613281,
      "logps/rejected": -580.9546508789062,
      "loss": 0.617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6460535526275635,
      "rewards/margins": 0.18731994926929474,
      "rewards/rejected": -0.833373486995697,
      "step": 1295
    },
    {
      "epoch": 0.1695918868078842,
      "grad_norm": 8.518906593322754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21121568977832794,
      "logits/rejected": 0.05359305813908577,
      "logps/chosen": -321.8908996582031,
      "logps/rejected": -316.5007019042969,
      "loss": 0.6645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4921284019947052,
      "rewards/margins": 0.1789611577987671,
      "rewards/rejected": -0.6710895895957947,
      "step": 1296
    },
    {
      "epoch": 0.16972274474523596,
      "grad_norm": 4.170012950897217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4943666160106659,
      "logits/rejected": 0.4176597595214844,
      "logps/chosen": -439.9541015625,
      "logps/rejected": -394.9809875488281,
      "loss": 0.3669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0138181671500206,
      "rewards/margins": 0.8806514143943787,
      "rewards/rejected": -0.8668332099914551,
      "step": 1297
    },
    {
      "epoch": 0.16985360268258773,
      "grad_norm": 7.176602363586426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5260903835296631,
      "logits/rejected": 0.5647156238555908,
      "logps/chosen": -395.117431640625,
      "logps/rejected": -435.52203369140625,
      "loss": 0.6456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35592326521873474,
      "rewards/margins": 0.1466074287891388,
      "rewards/rejected": -0.5025306940078735,
      "step": 1298
    },
    {
      "epoch": 0.16998446061993946,
      "grad_norm": 4.85520601272583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17975719273090363,
      "logits/rejected": 0.23366950452327728,
      "logps/chosen": -417.7501525878906,
      "logps/rejected": -436.69085693359375,
      "loss": 0.4294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18647156655788422,
      "rewards/margins": 0.6904119253158569,
      "rewards/rejected": -0.8768835663795471,
      "step": 1299
    },
    {
      "epoch": 0.17011531855729123,
      "grad_norm": 8.70553970336914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3498576879501343,
      "logits/rejected": 0.28741541504859924,
      "logps/chosen": -427.2903137207031,
      "logps/rejected": -395.3782043457031,
      "loss": 0.7158,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25223445892333984,
      "rewards/margins": 0.17163747549057007,
      "rewards/rejected": -0.4238719940185547,
      "step": 1300
    },
    {
      "epoch": 0.170246176494643,
      "grad_norm": 6.813905239105225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3171365261077881,
      "logits/rejected": 0.2888454496860504,
      "logps/chosen": -400.79522705078125,
      "logps/rejected": -397.80859375,
      "loss": 0.5265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37783241271972656,
      "rewards/margins": 0.4321041405200958,
      "rewards/rejected": -0.8099365830421448,
      "step": 1301
    },
    {
      "epoch": 0.17037703443199476,
      "grad_norm": 5.745041847229004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1255837082862854,
      "logits/rejected": 0.10190160572528839,
      "logps/chosen": -564.5220336914062,
      "logps/rejected": -570.080078125,
      "loss": 0.4979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.016025740653276443,
      "rewards/margins": 0.7904742956161499,
      "rewards/rejected": -0.8065001368522644,
      "step": 1302
    },
    {
      "epoch": 0.17050789236934652,
      "grad_norm": 6.7168288230896,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1136307641863823,
      "logits/rejected": 0.0790061429142952,
      "logps/chosen": -460.2396240234375,
      "logps/rejected": -407.8731689453125,
      "loss": 0.5655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6439800262451172,
      "rewards/margins": 0.7907909154891968,
      "rewards/rejected": -1.434770941734314,
      "step": 1303
    },
    {
      "epoch": 0.17063875030669828,
      "grad_norm": 4.448450565338135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43636083602905273,
      "logits/rejected": 0.18782436847686768,
      "logps/chosen": -410.17425537109375,
      "logps/rejected": -614.3111572265625,
      "loss": 0.3612,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27283668518066406,
      "rewards/margins": 0.990402102470398,
      "rewards/rejected": -1.263238787651062,
      "step": 1304
    },
    {
      "epoch": 0.17076960824405005,
      "grad_norm": 5.926846027374268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3668391704559326,
      "logits/rejected": 0.2721020579338074,
      "logps/chosen": -407.29205322265625,
      "logps/rejected": -462.3133544921875,
      "loss": 0.4702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3644908666610718,
      "rewards/margins": 0.6389749646186829,
      "rewards/rejected": -1.0034658908843994,
      "step": 1305
    },
    {
      "epoch": 0.1709004661814018,
      "grad_norm": 3.7225282192230225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4023906886577606,
      "logits/rejected": 0.39431777596473694,
      "logps/chosen": -328.64495849609375,
      "logps/rejected": -350.90325927734375,
      "loss": 0.3641,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2729167938232422,
      "rewards/margins": 0.9593874216079712,
      "rewards/rejected": -0.686470627784729,
      "step": 1306
    },
    {
      "epoch": 0.17103132411875357,
      "grad_norm": 5.9703474044799805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41805338859558105,
      "logits/rejected": 0.25861072540283203,
      "logps/chosen": -444.80645751953125,
      "logps/rejected": -377.13348388671875,
      "loss": 0.4976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17289313673973083,
      "rewards/margins": 0.7950546741485596,
      "rewards/rejected": -0.6221615076065063,
      "step": 1307
    },
    {
      "epoch": 0.17116218205610534,
      "grad_norm": 5.212705612182617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5582775473594666,
      "logits/rejected": 0.4527624845504761,
      "logps/chosen": -472.63525390625,
      "logps/rejected": -473.4617004394531,
      "loss": 0.5678,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1523195207118988,
      "rewards/margins": 0.561794638633728,
      "rewards/rejected": -0.7141141891479492,
      "step": 1308
    },
    {
      "epoch": 0.1712930399934571,
      "grad_norm": 6.195852279663086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040740225464105606,
      "logits/rejected": -0.2938326597213745,
      "logps/chosen": -464.765625,
      "logps/rejected": -596.0286254882812,
      "loss": 0.5684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6443252563476562,
      "rewards/margins": 0.3267875611782074,
      "rewards/rejected": -0.971112847328186,
      "step": 1309
    },
    {
      "epoch": 0.17142389793080887,
      "grad_norm": 5.678450107574463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39832761883735657,
      "logits/rejected": 0.349387526512146,
      "logps/chosen": -357.52850341796875,
      "logps/rejected": -345.11962890625,
      "loss": 0.5021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.252842515707016,
      "rewards/margins": 0.5605037808418274,
      "rewards/rejected": -0.813346266746521,
      "step": 1310
    },
    {
      "epoch": 0.17155475586816063,
      "grad_norm": 7.438867568969727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5226945281028748,
      "logits/rejected": 0.43613341450691223,
      "logps/chosen": -521.5330810546875,
      "logps/rejected": -479.6471862792969,
      "loss": 0.7698,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7677463293075562,
      "rewards/margins": 0.2739133834838867,
      "rewards/rejected": -1.0416597127914429,
      "step": 1311
    },
    {
      "epoch": 0.1716856138055124,
      "grad_norm": 5.9957733154296875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2449909746646881,
      "logits/rejected": 0.3178691864013672,
      "logps/chosen": -607.9176025390625,
      "logps/rejected": -627.2001953125,
      "loss": 0.4051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8740673661231995,
      "rewards/margins": 0.9077298045158386,
      "rewards/rejected": -0.03366243839263916,
      "step": 1312
    },
    {
      "epoch": 0.17181647174286416,
      "grad_norm": 9.990887641906738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31573763489723206,
      "logits/rejected": 0.3426743745803833,
      "logps/chosen": -563.4429931640625,
      "logps/rejected": -558.1900024414062,
      "loss": 0.9674,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21895752847194672,
      "rewards/margins": -0.033512137830257416,
      "rewards/rejected": -0.1854453980922699,
      "step": 1313
    },
    {
      "epoch": 0.17194732968021592,
      "grad_norm": 6.785089015960693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35720014572143555,
      "logits/rejected": 0.327909380197525,
      "logps/chosen": -390.884033203125,
      "logps/rejected": -374.92083740234375,
      "loss": 0.6619,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5519616603851318,
      "rewards/margins": 0.48448216915130615,
      "rewards/rejected": -1.0364439487457275,
      "step": 1314
    },
    {
      "epoch": 0.17207818761756768,
      "grad_norm": 6.77738618850708,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5757855176925659,
      "logits/rejected": 0.11937965452671051,
      "logps/chosen": -404.7109375,
      "logps/rejected": -459.8577575683594,
      "loss": 0.5801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6127304434776306,
      "rewards/margins": 0.42713087797164917,
      "rewards/rejected": -1.0398613214492798,
      "step": 1315
    },
    {
      "epoch": 0.17220904555491945,
      "grad_norm": 7.852245807647705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03616039454936981,
      "logits/rejected": 0.1569201648235321,
      "logps/chosen": -679.6153564453125,
      "logps/rejected": -493.1514892578125,
      "loss": 0.6185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6345133781433105,
      "rewards/margins": 0.8715533018112183,
      "rewards/rejected": -0.2370399683713913,
      "step": 1316
    },
    {
      "epoch": 0.1723399034922712,
      "grad_norm": 5.166804313659668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5274815559387207,
      "logits/rejected": 0.31622183322906494,
      "logps/chosen": -653.2622680664062,
      "logps/rejected": -511.927001953125,
      "loss": 0.3295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4298335909843445,
      "rewards/margins": 1.3131821155548096,
      "rewards/rejected": -0.8833485245704651,
      "step": 1317
    },
    {
      "epoch": 0.17247076142962298,
      "grad_norm": 7.826681137084961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3716103136539459,
      "logits/rejected": 0.3511396646499634,
      "logps/chosen": -408.49627685546875,
      "logps/rejected": -459.7693176269531,
      "loss": 0.7112,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2432708740234375,
      "rewards/margins": 0.25048428773880005,
      "rewards/rejected": -0.49375516176223755,
      "step": 1318
    },
    {
      "epoch": 0.17260161936697474,
      "grad_norm": 6.150960922241211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2318187803030014,
      "logits/rejected": 0.26844215393066406,
      "logps/chosen": -475.70111083984375,
      "logps/rejected": -324.17681884765625,
      "loss": 0.5553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1061948761343956,
      "rewards/margins": 0.38508546352386475,
      "rewards/rejected": -0.49128034710884094,
      "step": 1319
    },
    {
      "epoch": 0.1727324773043265,
      "grad_norm": 6.468433856964111,
      "learning_rate": 5e-06,
      "logits/chosen": 0.099993035197258,
      "logits/rejected": 0.24194955825805664,
      "logps/chosen": -402.38055419921875,
      "logps/rejected": -482.08880615234375,
      "loss": 0.5415,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07095125317573547,
      "rewards/margins": 0.7263363599777222,
      "rewards/rejected": -0.6553849577903748,
      "step": 1320
    },
    {
      "epoch": 0.17286333524167824,
      "grad_norm": 7.6241326332092285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31866320967674255,
      "logits/rejected": 0.14614123106002808,
      "logps/chosen": -435.38330078125,
      "logps/rejected": -431.5013732910156,
      "loss": 0.6041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6001842021942139,
      "rewards/margins": 0.2798164486885071,
      "rewards/rejected": -0.880000650882721,
      "step": 1321
    },
    {
      "epoch": 0.17299419317903,
      "grad_norm": 9.130906105041504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1612560898065567,
      "logits/rejected": 0.13094474375247955,
      "logps/chosen": -416.976806640625,
      "logps/rejected": -498.4444274902344,
      "loss": 0.9447,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0834497213363647,
      "rewards/margins": -0.24204124510288239,
      "rewards/rejected": -0.8414084911346436,
      "step": 1322
    },
    {
      "epoch": 0.17312505111638177,
      "grad_norm": 4.429389476776123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4518638253211975,
      "logits/rejected": 0.5775808691978455,
      "logps/chosen": -361.7037658691406,
      "logps/rejected": -376.69244384765625,
      "loss": 0.3777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04437446594238281,
      "rewards/margins": 1.0024734735488892,
      "rewards/rejected": -1.046847939491272,
      "step": 1323
    },
    {
      "epoch": 0.17325590905373353,
      "grad_norm": 8.42518424987793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48110538721084595,
      "logits/rejected": 0.3627191185951233,
      "logps/chosen": -471.6160888671875,
      "logps/rejected": -496.73138427734375,
      "loss": 0.6396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6632137298583984,
      "rewards/margins": 0.4395408630371094,
      "rewards/rejected": -1.1027545928955078,
      "step": 1324
    },
    {
      "epoch": 0.1733867669910853,
      "grad_norm": 6.355416297912598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19810663163661957,
      "logits/rejected": 0.14937296509742737,
      "logps/chosen": -355.91656494140625,
      "logps/rejected": -338.96868896484375,
      "loss": 0.6637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24542219936847687,
      "rewards/margins": 0.2800138592720032,
      "rewards/rejected": -0.5254360437393188,
      "step": 1325
    },
    {
      "epoch": 0.17351762492843706,
      "grad_norm": 5.985579490661621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43600916862487793,
      "logits/rejected": 0.18554922938346863,
      "logps/chosen": -323.0829772949219,
      "logps/rejected": -322.1459045410156,
      "loss": 0.632,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.579537034034729,
      "rewards/margins": 0.26620715856552124,
      "rewards/rejected": -0.8457441926002502,
      "step": 1326
    },
    {
      "epoch": 0.17364848286578882,
      "grad_norm": 6.359167098999023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43817007541656494,
      "logits/rejected": 0.40259578824043274,
      "logps/chosen": -447.3368835449219,
      "logps/rejected": -505.8390808105469,
      "loss": 0.5002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0995105728507042,
      "rewards/margins": 0.5245550274848938,
      "rewards/rejected": -0.624065637588501,
      "step": 1327
    },
    {
      "epoch": 0.1737793408031406,
      "grad_norm": 7.441427230834961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45170414447784424,
      "logits/rejected": 0.2471402883529663,
      "logps/chosen": -665.8135986328125,
      "logps/rejected": -523.5269165039062,
      "loss": 0.4362,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36961251497268677,
      "rewards/margins": 0.9157852530479431,
      "rewards/rejected": -1.2853977680206299,
      "step": 1328
    },
    {
      "epoch": 0.17391019874049235,
      "grad_norm": 6.88876485824585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2170914113521576,
      "logits/rejected": 0.06643014401197433,
      "logps/chosen": -446.9158935546875,
      "logps/rejected": -339.1568908691406,
      "loss": 0.7339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.556684672832489,
      "rewards/margins": 0.11717493832111359,
      "rewards/rejected": -0.6738596558570862,
      "step": 1329
    },
    {
      "epoch": 0.1740410566778441,
      "grad_norm": 6.278741359710693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026711782440543175,
      "logits/rejected": -0.0011976389214396477,
      "logps/chosen": -479.9688415527344,
      "logps/rejected": -480.3443908691406,
      "loss": 0.6275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9110934734344482,
      "rewards/margins": 0.8314551115036011,
      "rewards/rejected": -1.7425487041473389,
      "step": 1330
    },
    {
      "epoch": 0.17417191461519588,
      "grad_norm": 5.037548065185547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2758977711200714,
      "logits/rejected": 0.21479327976703644,
      "logps/chosen": -386.4352722167969,
      "logps/rejected": -414.4274597167969,
      "loss": 0.4406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.020093142986297607,
      "rewards/margins": 1.0883935689926147,
      "rewards/rejected": -1.0683003664016724,
      "step": 1331
    },
    {
      "epoch": 0.17430277255254764,
      "grad_norm": 7.157739162445068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02539600431919098,
      "logits/rejected": 0.16892442107200623,
      "logps/chosen": -562.2944946289062,
      "logps/rejected": -543.0454711914062,
      "loss": 0.5921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38144588470458984,
      "rewards/margins": 0.553026020526886,
      "rewards/rejected": -0.9344719648361206,
      "step": 1332
    },
    {
      "epoch": 0.1744336304898994,
      "grad_norm": 4.089100360870361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022201605141162872,
      "logits/rejected": 0.0027635395526885986,
      "logps/chosen": -451.4317321777344,
      "logps/rejected": -410.72509765625,
      "loss": 0.2555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06004561111330986,
      "rewards/margins": 1.5813688039779663,
      "rewards/rejected": -1.6414145231246948,
      "step": 1333
    },
    {
      "epoch": 0.17456448842725117,
      "grad_norm": 6.241490840911865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1548691838979721,
      "logits/rejected": 0.18027497828006744,
      "logps/chosen": -567.3573608398438,
      "logps/rejected": -547.5681762695312,
      "loss": 0.6284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3722679316997528,
      "rewards/margins": 0.6545979380607605,
      "rewards/rejected": -1.0268657207489014,
      "step": 1334
    },
    {
      "epoch": 0.17469534636460293,
      "grad_norm": 10.025079727172852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2553708255290985,
      "logits/rejected": 0.10676752775907516,
      "logps/chosen": -550.1797485351562,
      "logps/rejected": -500.66229248046875,
      "loss": 0.7561,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5131202936172485,
      "rewards/margins": 0.6481277942657471,
      "rewards/rejected": -1.161247968673706,
      "step": 1335
    },
    {
      "epoch": 0.1748262043019547,
      "grad_norm": 12.394033432006836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11104084551334381,
      "logits/rejected": 0.3384082019329071,
      "logps/chosen": -428.4512023925781,
      "logps/rejected": -341.91485595703125,
      "loss": 0.7129,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7167500853538513,
      "rewards/margins": 0.665608286857605,
      "rewards/rejected": -1.382358431816101,
      "step": 1336
    },
    {
      "epoch": 0.17495706223930646,
      "grad_norm": 7.805538654327393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03173479437828064,
      "logits/rejected": 0.2122519165277481,
      "logps/chosen": -382.5783386230469,
      "logps/rejected": -354.1275634765625,
      "loss": 0.6754,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19926223158836365,
      "rewards/margins": 0.37785154581069946,
      "rewards/rejected": -0.5771138072013855,
      "step": 1337
    },
    {
      "epoch": 0.17508792017665822,
      "grad_norm": 5.625957489013672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2533475160598755,
      "logits/rejected": -0.09345097094774246,
      "logps/chosen": -455.38818359375,
      "logps/rejected": -796.1805419921875,
      "loss": 0.3889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6059120297431946,
      "rewards/margins": 0.8742282390594482,
      "rewards/rejected": -1.4801404476165771,
      "step": 1338
    },
    {
      "epoch": 0.17521877811401,
      "grad_norm": 4.422956943511963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4850645065307617,
      "logits/rejected": 0.47210657596588135,
      "logps/chosen": -371.575439453125,
      "logps/rejected": -424.1919250488281,
      "loss": 0.3145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21531793475151062,
      "rewards/margins": 1.1637167930603027,
      "rewards/rejected": -1.3790348768234253,
      "step": 1339
    },
    {
      "epoch": 0.17534963605136175,
      "grad_norm": 4.860403060913086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29916319251060486,
      "logits/rejected": 0.3964506685733795,
      "logps/chosen": -509.10400390625,
      "logps/rejected": -598.1898193359375,
      "loss": 0.3968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5071678161621094,
      "rewards/margins": 0.9880344271659851,
      "rewards/rejected": -1.4952023029327393,
      "step": 1340
    },
    {
      "epoch": 0.17548049398871352,
      "grad_norm": 6.5524210929870605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2580907344818115,
      "logits/rejected": 0.19598165154457092,
      "logps/chosen": -362.12347412109375,
      "logps/rejected": -414.3842468261719,
      "loss": 0.6066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23872368037700653,
      "rewards/margins": 0.5764906406402588,
      "rewards/rejected": -0.8152143359184265,
      "step": 1341
    },
    {
      "epoch": 0.17561135192606528,
      "grad_norm": 7.7224907875061035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16356977820396423,
      "logits/rejected": 0.06417841464281082,
      "logps/chosen": -358.264892578125,
      "logps/rejected": -484.90875244140625,
      "loss": 0.6519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4374597668647766,
      "rewards/margins": 0.4151127338409424,
      "rewards/rejected": -0.852572500705719,
      "step": 1342
    },
    {
      "epoch": 0.17574220986341701,
      "grad_norm": 8.843308448791504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2106875628232956,
      "logits/rejected": 0.23491621017456055,
      "logps/chosen": -429.5339660644531,
      "logps/rejected": -433.20745849609375,
      "loss": 0.7618,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.40672001242637634,
      "rewards/margins": 0.12202932685613632,
      "rewards/rejected": -0.5287492871284485,
      "step": 1343
    },
    {
      "epoch": 0.17587306780076878,
      "grad_norm": 12.487836837768555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20197315514087677,
      "logits/rejected": 0.29579994082450867,
      "logps/chosen": -471.9087219238281,
      "logps/rejected": -304.15338134765625,
      "loss": 0.7833,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8680295944213867,
      "rewards/margins": -0.04760587215423584,
      "rewards/rejected": -0.8204237222671509,
      "step": 1344
    },
    {
      "epoch": 0.17600392573812054,
      "grad_norm": 6.907110214233398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.513344407081604,
      "logits/rejected": 0.28849297761917114,
      "logps/chosen": -369.94287109375,
      "logps/rejected": -507.1888427734375,
      "loss": 0.4713,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7942596077919006,
      "rewards/margins": 1.150153636932373,
      "rewards/rejected": -1.944413423538208,
      "step": 1345
    },
    {
      "epoch": 0.1761347836754723,
      "grad_norm": 5.947015285491943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24132423102855682,
      "logits/rejected": 0.28499072790145874,
      "logps/chosen": -524.5138549804688,
      "logps/rejected": -542.1643676757812,
      "loss": 0.4149,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1586365699768066,
      "rewards/margins": 1.0721795558929443,
      "rewards/rejected": -2.230816125869751,
      "step": 1346
    },
    {
      "epoch": 0.17626564161282407,
      "grad_norm": 5.352518081665039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0507081001996994,
      "logits/rejected": 0.09194482862949371,
      "logps/chosen": -443.741943359375,
      "logps/rejected": -602.0953979492188,
      "loss": 0.3823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0712711289525032,
      "rewards/margins": 1.4335020780563354,
      "rewards/rejected": -1.5047731399536133,
      "step": 1347
    },
    {
      "epoch": 0.17639649955017583,
      "grad_norm": 5.575090408325195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019703928381204605,
      "logits/rejected": 0.06683873385190964,
      "logps/chosen": -424.5859069824219,
      "logps/rejected": -530.326171875,
      "loss": 0.3688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.035824958235025406,
      "rewards/margins": 1.1628694534301758,
      "rewards/rejected": -1.1986944675445557,
      "step": 1348
    },
    {
      "epoch": 0.1765273574875276,
      "grad_norm": 7.488029956817627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.215216264128685,
      "logits/rejected": 0.13309937715530396,
      "logps/chosen": -446.28265380859375,
      "logps/rejected": -448.91314697265625,
      "loss": 0.6762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7222867608070374,
      "rewards/margins": 0.3228670060634613,
      "rewards/rejected": -1.0451538562774658,
      "step": 1349
    },
    {
      "epoch": 0.17665821542487936,
      "grad_norm": 9.381888389587402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31060436367988586,
      "logits/rejected": 0.08442898839712143,
      "logps/chosen": -513.0445556640625,
      "logps/rejected": -594.5259399414062,
      "loss": 0.9373,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3736423850059509,
      "rewards/margins": -0.07115378975868225,
      "rewards/rejected": -0.3024885654449463,
      "step": 1350
    },
    {
      "epoch": 0.17678907336223113,
      "grad_norm": 6.201504230499268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27187827229499817,
      "logits/rejected": 0.11336767673492432,
      "logps/chosen": -341.6315612792969,
      "logps/rejected": -360.8699645996094,
      "loss": 0.4513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4851827621459961,
      "rewards/margins": 0.8046016693115234,
      "rewards/rejected": -1.2897844314575195,
      "step": 1351
    },
    {
      "epoch": 0.1769199312995829,
      "grad_norm": 7.237081527709961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3541123867034912,
      "logits/rejected": 0.34572768211364746,
      "logps/chosen": -453.23681640625,
      "logps/rejected": -394.4842224121094,
      "loss": 0.7122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6243494153022766,
      "rewards/margins": 0.2577945590019226,
      "rewards/rejected": -0.8821439743041992,
      "step": 1352
    },
    {
      "epoch": 0.17705078923693465,
      "grad_norm": 6.378726959228516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4139823913574219,
      "logits/rejected": 0.3487480878829956,
      "logps/chosen": -450.57781982421875,
      "logps/rejected": -400.85260009765625,
      "loss": 0.5528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8012315630912781,
      "rewards/margins": 0.4957958459854126,
      "rewards/rejected": -1.2970274686813354,
      "step": 1353
    },
    {
      "epoch": 0.17718164717428642,
      "grad_norm": 5.783170223236084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2623614966869354,
      "logits/rejected": 0.20872193574905396,
      "logps/chosen": -600.4239501953125,
      "logps/rejected": -561.9281005859375,
      "loss": 0.4891,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6229183673858643,
      "rewards/margins": 0.8422256708145142,
      "rewards/rejected": -1.4651440382003784,
      "step": 1354
    },
    {
      "epoch": 0.17731250511163818,
      "grad_norm": 5.088136196136475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017689265310764313,
      "logits/rejected": -0.030086003243923187,
      "logps/chosen": -596.158447265625,
      "logps/rejected": -538.50537109375,
      "loss": 0.4587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3355989456176758,
      "rewards/margins": 1.0323381423950195,
      "rewards/rejected": -1.3679370880126953,
      "step": 1355
    },
    {
      "epoch": 0.17744336304898994,
      "grad_norm": 4.065127849578857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0683317556977272,
      "logits/rejected": -0.02085820958018303,
      "logps/chosen": -571.52099609375,
      "logps/rejected": -500.9062194824219,
      "loss": 0.3,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.014404483139514923,
      "rewards/margins": 1.5597292184829712,
      "rewards/rejected": -1.5741336345672607,
      "step": 1356
    },
    {
      "epoch": 0.1775742209863417,
      "grad_norm": 8.15420913696289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5091052055358887,
      "logits/rejected": 0.39558321237564087,
      "logps/chosen": -427.86175537109375,
      "logps/rejected": -384.08209228515625,
      "loss": 0.6749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.520222008228302,
      "rewards/margins": 0.4724571704864502,
      "rewards/rejected": -0.9926792979240417,
      "step": 1357
    },
    {
      "epoch": 0.17770507892369347,
      "grad_norm": 8.782232284545898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44943735003471375,
      "logits/rejected": 0.4275728464126587,
      "logps/chosen": -376.80828857421875,
      "logps/rejected": -288.0379333496094,
      "loss": 0.8288,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7252720594406128,
      "rewards/margins": 0.005368158221244812,
      "rewards/rejected": -0.7306401133537292,
      "step": 1358
    },
    {
      "epoch": 0.17783593686104524,
      "grad_norm": 5.608109951019287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2948087453842163,
      "logits/rejected": 0.14119534194469452,
      "logps/chosen": -395.0916442871094,
      "logps/rejected": -427.1742248535156,
      "loss": 0.5053,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6535494327545166,
      "rewards/margins": 0.6129688620567322,
      "rewards/rejected": -1.266518235206604,
      "step": 1359
    },
    {
      "epoch": 0.177966794798397,
      "grad_norm": 5.772524833679199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3402661085128784,
      "logits/rejected": 0.23749077320098877,
      "logps/chosen": -230.5049285888672,
      "logps/rejected": -335.61669921875,
      "loss": 0.4704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.759050726890564,
      "rewards/margins": 0.540810763835907,
      "rewards/rejected": -1.2998615503311157,
      "step": 1360
    },
    {
      "epoch": 0.17809765273574876,
      "grad_norm": 9.581230163574219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2804742753505707,
      "logits/rejected": 0.34976133704185486,
      "logps/chosen": -473.260009765625,
      "logps/rejected": -536.0267333984375,
      "loss": 1.0778,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6288387179374695,
      "rewards/margins": -0.5682968497276306,
      "rewards/rejected": -0.06054193153977394,
      "step": 1361
    },
    {
      "epoch": 0.17822851067310053,
      "grad_norm": 8.629536628723145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01187560148537159,
      "logits/rejected": -0.13026955723762512,
      "logps/chosen": -576.49951171875,
      "logps/rejected": -505.3110046386719,
      "loss": 0.6396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7994978427886963,
      "rewards/margins": 0.39489325881004333,
      "rewards/rejected": -1.194391131401062,
      "step": 1362
    },
    {
      "epoch": 0.1783593686104523,
      "grad_norm": 5.517137050628662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3608537018299103,
      "logits/rejected": -0.009164348244667053,
      "logps/chosen": -426.35760498046875,
      "logps/rejected": -382.84747314453125,
      "loss": 0.5519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9473012685775757,
      "rewards/margins": 0.46044617891311646,
      "rewards/rejected": -1.407747507095337,
      "step": 1363
    },
    {
      "epoch": 0.17849022654780405,
      "grad_norm": 6.877335071563721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24929632246494293,
      "logits/rejected": 0.34968265891075134,
      "logps/chosen": -454.0537109375,
      "logps/rejected": -456.3076477050781,
      "loss": 0.5552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49524635076522827,
      "rewards/margins": 0.9203988313674927,
      "rewards/rejected": -1.4156451225280762,
      "step": 1364
    },
    {
      "epoch": 0.1786210844851558,
      "grad_norm": 6.903815269470215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14567050337791443,
      "logits/rejected": 0.13131177425384521,
      "logps/chosen": -548.170654296875,
      "logps/rejected": -388.2901306152344,
      "loss": 0.5719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7751474380493164,
      "rewards/margins": 0.39387497305870056,
      "rewards/rejected": -1.1690223217010498,
      "step": 1365
    },
    {
      "epoch": 0.17875194242250755,
      "grad_norm": 7.7661662101745605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31948742270469666,
      "logits/rejected": 0.2502301037311554,
      "logps/chosen": -554.0570068359375,
      "logps/rejected": -483.8906555175781,
      "loss": 0.6994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6018261313438416,
      "rewards/margins": 0.3020383417606354,
      "rewards/rejected": -0.9038644433021545,
      "step": 1366
    },
    {
      "epoch": 0.17888280035985932,
      "grad_norm": 7.46135950088501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0024554431438446045,
      "logits/rejected": 0.16046038269996643,
      "logps/chosen": -364.01971435546875,
      "logps/rejected": -331.7918701171875,
      "loss": 0.5024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.785605788230896,
      "rewards/margins": 0.7752037048339844,
      "rewards/rejected": -1.5608094930648804,
      "step": 1367
    },
    {
      "epoch": 0.17901365829721108,
      "grad_norm": 7.292722702026367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1795150190591812,
      "logits/rejected": 0.22072526812553406,
      "logps/chosen": -520.885009765625,
      "logps/rejected": -457.49798583984375,
      "loss": 0.7378,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8727144598960876,
      "rewards/margins": 0.1153833270072937,
      "rewards/rejected": -0.9880977869033813,
      "step": 1368
    },
    {
      "epoch": 0.17914451623456285,
      "grad_norm": 6.593092441558838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47067514061927795,
      "logits/rejected": 0.2268255054950714,
      "logps/chosen": -396.3222351074219,
      "logps/rejected": -460.6983947753906,
      "loss": 0.54,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6029243469238281,
      "rewards/margins": 0.5361167788505554,
      "rewards/rejected": -1.1390411853790283,
      "step": 1369
    },
    {
      "epoch": 0.1792753741719146,
      "grad_norm": 4.63119649887085,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5459938645362854,
      "logits/rejected": 0.5092217326164246,
      "logps/chosen": -404.4896240234375,
      "logps/rejected": -491.6988525390625,
      "loss": 0.3855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2665821313858032,
      "rewards/margins": 1.0327534675598145,
      "rewards/rejected": -1.2993354797363281,
      "step": 1370
    },
    {
      "epoch": 0.17940623210926637,
      "grad_norm": 9.230603218078613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3676898181438446,
      "logits/rejected": -0.09053352475166321,
      "logps/chosen": -410.1913757324219,
      "logps/rejected": -336.8762512207031,
      "loss": 0.6754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7168241739273071,
      "rewards/margins": 0.3345130980014801,
      "rewards/rejected": -1.0513372421264648,
      "step": 1371
    },
    {
      "epoch": 0.17953709004661814,
      "grad_norm": 7.7888336181640625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3119497299194336,
      "logits/rejected": 0.4193052649497986,
      "logps/chosen": -532.8275146484375,
      "logps/rejected": -602.30224609375,
      "loss": 0.6697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.730612576007843,
      "rewards/margins": 0.3950205147266388,
      "rewards/rejected": -1.1256330013275146,
      "step": 1372
    },
    {
      "epoch": 0.1796679479839699,
      "grad_norm": 6.205885410308838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30937033891677856,
      "logits/rejected": 0.3206769824028015,
      "logps/chosen": -428.5567932128906,
      "logps/rejected": -315.3561096191406,
      "loss": 0.5144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48500770330429077,
      "rewards/margins": 0.5500598549842834,
      "rewards/rejected": -1.0350675582885742,
      "step": 1373
    },
    {
      "epoch": 0.17979880592132166,
      "grad_norm": 5.324701309204102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0642530545592308,
      "logits/rejected": 0.06901519000530243,
      "logps/chosen": -258.32568359375,
      "logps/rejected": -376.90606689453125,
      "loss": 0.5243,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.90725177526474,
      "rewards/margins": 0.6965570449829102,
      "rewards/rejected": -1.6038087606430054,
      "step": 1374
    },
    {
      "epoch": 0.17992966385867343,
      "grad_norm": 4.16204309463501,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24398192763328552,
      "logits/rejected": 0.05192127451300621,
      "logps/chosen": -265.1581115722656,
      "logps/rejected": -339.29644775390625,
      "loss": 0.4314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42592737078666687,
      "rewards/margins": 1.13191556930542,
      "rewards/rejected": -1.5578429698944092,
      "step": 1375
    },
    {
      "epoch": 0.1800605217960252,
      "grad_norm": 5.737246036529541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2080872803926468,
      "logits/rejected": 0.1084219291806221,
      "logps/chosen": -407.03582763671875,
      "logps/rejected": -388.3501281738281,
      "loss": 0.5069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7805200815200806,
      "rewards/margins": 0.7461634278297424,
      "rewards/rejected": -1.5266834497451782,
      "step": 1376
    },
    {
      "epoch": 0.18019137973337696,
      "grad_norm": 6.6492204666137695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31521308422088623,
      "logits/rejected": 0.35070279240608215,
      "logps/chosen": -448.6700439453125,
      "logps/rejected": -427.8312072753906,
      "loss": 0.6332,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7110630869865417,
      "rewards/margins": 0.365223228931427,
      "rewards/rejected": -1.0762863159179688,
      "step": 1377
    },
    {
      "epoch": 0.18032223767072872,
      "grad_norm": 6.626095771789551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.515644907951355,
      "logits/rejected": 0.28130975365638733,
      "logps/chosen": -302.4432678222656,
      "logps/rejected": -264.9061279296875,
      "loss": 0.635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9807680249214172,
      "rewards/margins": 0.31641289591789246,
      "rewards/rejected": -1.2971808910369873,
      "step": 1378
    },
    {
      "epoch": 0.18045309560808048,
      "grad_norm": 6.332429885864258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11089589446783066,
      "logits/rejected": 0.03927859291434288,
      "logps/chosen": -291.95721435546875,
      "logps/rejected": -386.25543212890625,
      "loss": 0.5624,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6480539441108704,
      "rewards/margins": 0.7592481970787048,
      "rewards/rejected": -1.4073021411895752,
      "step": 1379
    },
    {
      "epoch": 0.18058395354543225,
      "grad_norm": 8.640925407409668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3216323256492615,
      "logits/rejected": 0.39709946513175964,
      "logps/chosen": -487.32037353515625,
      "logps/rejected": -497.7972412109375,
      "loss": 0.813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027439884841442108,
      "rewards/margins": -0.02986346185207367,
      "rewards/rejected": 0.0024235695600509644,
      "step": 1380
    },
    {
      "epoch": 0.180714811482784,
      "grad_norm": 5.349815845489502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27969399094581604,
      "logits/rejected": 0.2810632288455963,
      "logps/chosen": -393.6175842285156,
      "logps/rejected": -377.02392578125,
      "loss": 0.3749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39009881019592285,
      "rewards/margins": 1.0104364156723022,
      "rewards/rejected": -1.400535225868225,
      "step": 1381
    },
    {
      "epoch": 0.18084566942013577,
      "grad_norm": 6.10128927230835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2504453659057617,
      "logits/rejected": -0.2276248037815094,
      "logps/chosen": -399.054931640625,
      "logps/rejected": -391.28179931640625,
      "loss": 0.5286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7011263370513916,
      "rewards/margins": 0.4182642102241516,
      "rewards/rejected": -1.1193904876708984,
      "step": 1382
    },
    {
      "epoch": 0.18097652735748754,
      "grad_norm": 7.327820301055908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32947254180908203,
      "logits/rejected": 0.459964394569397,
      "logps/chosen": -459.6220703125,
      "logps/rejected": -426.8697204589844,
      "loss": 0.5728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20123404264450073,
      "rewards/margins": 0.5384171009063721,
      "rewards/rejected": -0.7396512031555176,
      "step": 1383
    },
    {
      "epoch": 0.1811073852948393,
      "grad_norm": 6.192204475402832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1609697937965393,
      "logits/rejected": -0.14502233266830444,
      "logps/chosen": -674.603759765625,
      "logps/rejected": -488.3974609375,
      "loss": 0.4853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2978702783584595,
      "rewards/margins": 0.6061797738075256,
      "rewards/rejected": -0.9040501117706299,
      "step": 1384
    },
    {
      "epoch": 0.18123824323219107,
      "grad_norm": 9.810341835021973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014963088557124138,
      "logits/rejected": -0.014343376271426678,
      "logps/chosen": -568.0878295898438,
      "logps/rejected": -440.3675537109375,
      "loss": 0.7508,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3212534189224243,
      "rewards/margins": -0.013039544224739075,
      "rewards/rejected": -1.3082138299942017,
      "step": 1385
    },
    {
      "epoch": 0.1813691011695428,
      "grad_norm": 4.043344020843506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0511644221842289,
      "logits/rejected": -0.07906215637922287,
      "logps/chosen": -385.3708801269531,
      "logps/rejected": -370.6547546386719,
      "loss": 0.3627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.259063720703125,
      "rewards/margins": 0.9811282753944397,
      "rewards/rejected": -1.2401920557022095,
      "step": 1386
    },
    {
      "epoch": 0.18149995910689457,
      "grad_norm": 6.542601585388184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14793017506599426,
      "logits/rejected": 0.29638174176216125,
      "logps/chosen": -512.2091674804688,
      "logps/rejected": -445.99615478515625,
      "loss": 0.5979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7302746176719666,
      "rewards/margins": 0.5087230801582336,
      "rewards/rejected": -1.2389975786209106,
      "step": 1387
    },
    {
      "epoch": 0.18163081704424633,
      "grad_norm": 5.868104934692383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0825648084282875,
      "logits/rejected": 0.1888577938079834,
      "logps/chosen": -356.92901611328125,
      "logps/rejected": -482.227783203125,
      "loss": 0.5379,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1784956157207489,
      "rewards/margins": 0.7649012804031372,
      "rewards/rejected": -0.9433969259262085,
      "step": 1388
    },
    {
      "epoch": 0.1817616749815981,
      "grad_norm": 5.5642290115356445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017872080206871033,
      "logits/rejected": -0.14906111359596252,
      "logps/chosen": -450.4095458984375,
      "logps/rejected": -351.1260986328125,
      "loss": 0.4388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8113977909088135,
      "rewards/margins": 0.7225759029388428,
      "rewards/rejected": -1.5339736938476562,
      "step": 1389
    },
    {
      "epoch": 0.18189253291894986,
      "grad_norm": 8.576400756835938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21652740240097046,
      "logits/rejected": 0.17552588880062103,
      "logps/chosen": -714.7952880859375,
      "logps/rejected": -503.9393310546875,
      "loss": 0.6528,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9792129397392273,
      "rewards/margins": 0.30254918336868286,
      "rewards/rejected": -1.2817621231079102,
      "step": 1390
    },
    {
      "epoch": 0.18202339085630162,
      "grad_norm": 9.14046859741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039140019565820694,
      "logits/rejected": -0.25413793325424194,
      "logps/chosen": -401.4293212890625,
      "logps/rejected": -520.3629760742188,
      "loss": 0.5732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5501986145973206,
      "rewards/margins": 0.4216257333755493,
      "rewards/rejected": -0.9718242287635803,
      "step": 1391
    },
    {
      "epoch": 0.18215424879365338,
      "grad_norm": 7.179779052734375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37108540534973145,
      "logits/rejected": 0.4942311942577362,
      "logps/chosen": -500.4876708984375,
      "logps/rejected": -407.22076416015625,
      "loss": 0.7206,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7615790963172913,
      "rewards/margins": 0.05445767566561699,
      "rewards/rejected": -0.8160368204116821,
      "step": 1392
    },
    {
      "epoch": 0.18228510673100515,
      "grad_norm": 8.882396697998047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09522383660078049,
      "logits/rejected": 0.12591244280338287,
      "logps/chosen": -664.0827026367188,
      "logps/rejected": -709.9664916992188,
      "loss": 0.5791,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5179657340049744,
      "rewards/margins": 0.791305422782898,
      "rewards/rejected": -1.3092710971832275,
      "step": 1393
    },
    {
      "epoch": 0.1824159646683569,
      "grad_norm": 6.653695106506348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2671642303466797,
      "logits/rejected": 0.42588767409324646,
      "logps/chosen": -427.4843444824219,
      "logps/rejected": -425.2647705078125,
      "loss": 0.5147,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11903439462184906,
      "rewards/margins": 0.6036503314971924,
      "rewards/rejected": -0.7226846814155579,
      "step": 1394
    },
    {
      "epoch": 0.18254682260570868,
      "grad_norm": 5.598435401916504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5166267156600952,
      "logits/rejected": 0.29323747754096985,
      "logps/chosen": -594.33837890625,
      "logps/rejected": -475.3148193359375,
      "loss": 0.4772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05253295227885246,
      "rewards/margins": 0.7199869751930237,
      "rewards/rejected": -0.7725198864936829,
      "step": 1395
    },
    {
      "epoch": 0.18267768054306044,
      "grad_norm": 6.317687034606934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2806340456008911,
      "logits/rejected": 0.1882985234260559,
      "logps/chosen": -550.9981079101562,
      "logps/rejected": -499.5545349121094,
      "loss": 0.6244,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006809242069721222,
      "rewards/margins": 0.284310519695282,
      "rewards/rejected": -0.2911197543144226,
      "step": 1396
    },
    {
      "epoch": 0.1828085384804122,
      "grad_norm": 7.636439800262451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23255756497383118,
      "logits/rejected": 0.14364376664161682,
      "logps/chosen": -533.7996826171875,
      "logps/rejected": -404.8547058105469,
      "loss": 0.8421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5229294300079346,
      "rewards/margins": 0.03934039548039436,
      "rewards/rejected": -0.5622698068618774,
      "step": 1397
    },
    {
      "epoch": 0.18293939641776397,
      "grad_norm": 10.918179512023926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24802306294441223,
      "logits/rejected": -0.044565580785274506,
      "logps/chosen": -399.82672119140625,
      "logps/rejected": -364.07098388671875,
      "loss": 0.9764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7550050616264343,
      "rewards/margins": -0.09341453015804291,
      "rewards/rejected": -0.6615905165672302,
      "step": 1398
    },
    {
      "epoch": 0.18307025435511573,
      "grad_norm": 7.105988502502441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23539017140865326,
      "logits/rejected": 0.05312756076455116,
      "logps/chosen": -539.8104248046875,
      "logps/rejected": -349.95086669921875,
      "loss": 0.6313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5149645209312439,
      "rewards/margins": 0.4324052929878235,
      "rewards/rejected": -0.9473698139190674,
      "step": 1399
    },
    {
      "epoch": 0.1832011122924675,
      "grad_norm": 7.796037673950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3160056173801422,
      "logits/rejected": 0.24910730123519897,
      "logps/chosen": -374.89068603515625,
      "logps/rejected": -339.134521484375,
      "loss": 0.5631,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5921729803085327,
      "rewards/margins": 0.5022215247154236,
      "rewards/rejected": -1.0943944454193115,
      "step": 1400
    },
    {
      "epoch": 0.18333197022981926,
      "grad_norm": 6.231539726257324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4811137020587921,
      "logits/rejected": 0.452833354473114,
      "logps/chosen": -333.07928466796875,
      "logps/rejected": -323.6813659667969,
      "loss": 0.6956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5664852857589722,
      "rewards/margins": 0.09394609928131104,
      "rewards/rejected": -0.6604313254356384,
      "step": 1401
    },
    {
      "epoch": 0.18346282816717102,
      "grad_norm": 5.911405563354492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18080435693264008,
      "logits/rejected": 0.038786523044109344,
      "logps/chosen": -459.439453125,
      "logps/rejected": -480.38665771484375,
      "loss": 0.4586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0057283490896224976,
      "rewards/margins": 0.7589390277862549,
      "rewards/rejected": -0.7646673321723938,
      "step": 1402
    },
    {
      "epoch": 0.18359368610452279,
      "grad_norm": 4.405777454376221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4747205376625061,
      "logits/rejected": 0.3667846918106079,
      "logps/chosen": -445.984375,
      "logps/rejected": -339.6905517578125,
      "loss": 0.3254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.011486425995826721,
      "rewards/margins": 1.1373631954193115,
      "rewards/rejected": -1.1258766651153564,
      "step": 1403
    },
    {
      "epoch": 0.18372454404187455,
      "grad_norm": 9.493364334106445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07248596847057343,
      "logits/rejected": 0.15080295503139496,
      "logps/chosen": -326.25335693359375,
      "logps/rejected": -367.6064147949219,
      "loss": 0.8093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5498042702674866,
      "rewards/margins": -0.027603700757026672,
      "rewards/rejected": -0.5222005844116211,
      "step": 1404
    },
    {
      "epoch": 0.1838554019792263,
      "grad_norm": 6.455997467041016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2991635799407959,
      "logits/rejected": 0.2640884518623352,
      "logps/chosen": -442.1815185546875,
      "logps/rejected": -490.8353576660156,
      "loss": 0.5859,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39196452498435974,
      "rewards/margins": 0.7507489919662476,
      "rewards/rejected": -1.1427135467529297,
      "step": 1405
    },
    {
      "epoch": 0.18398625991657808,
      "grad_norm": 7.155182361602783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02166886255145073,
      "logits/rejected": 0.18211302161216736,
      "logps/chosen": -570.8563232421875,
      "logps/rejected": -522.0816040039062,
      "loss": 0.6099,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3176324963569641,
      "rewards/margins": 0.3317771852016449,
      "rewards/rejected": -0.6494096517562866,
      "step": 1406
    },
    {
      "epoch": 0.18411711785392984,
      "grad_norm": 9.535467147827148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17254061996936798,
      "logits/rejected": 0.0848444402217865,
      "logps/chosen": -546.2813720703125,
      "logps/rejected": -682.07666015625,
      "loss": 0.7882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8573306798934937,
      "rewards/margins": 0.16265563666820526,
      "rewards/rejected": -1.0199862718582153,
      "step": 1407
    },
    {
      "epoch": 0.18424797579128158,
      "grad_norm": 5.687534332275391,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20325937867164612,
      "logits/rejected": 0.24042004346847534,
      "logps/chosen": -524.7778930664062,
      "logps/rejected": -594.357421875,
      "loss": 0.484,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23088283836841583,
      "rewards/margins": 0.8977053165435791,
      "rewards/rejected": -1.128588080406189,
      "step": 1408
    },
    {
      "epoch": 0.18437883372863334,
      "grad_norm": 10.103507995605469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3825092315673828,
      "logits/rejected": 0.3257724642753601,
      "logps/chosen": -540.9332275390625,
      "logps/rejected": -511.49737548828125,
      "loss": 0.8242,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8879866003990173,
      "rewards/margins": 0.026189446449279785,
      "rewards/rejected": -0.9141759276390076,
      "step": 1409
    },
    {
      "epoch": 0.1845096916659851,
      "grad_norm": 6.808079242706299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13876795768737793,
      "logits/rejected": -0.0062233395874500275,
      "logps/chosen": -420.0090637207031,
      "logps/rejected": -380.0224609375,
      "loss": 0.454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5381177663803101,
      "rewards/margins": 0.7373363375663757,
      "rewards/rejected": -1.2754541635513306,
      "step": 1410
    },
    {
      "epoch": 0.18464054960333687,
      "grad_norm": 11.035309791564941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18839600682258606,
      "logits/rejected": 0.02339746057987213,
      "logps/chosen": -402.6906433105469,
      "logps/rejected": -397.4539489746094,
      "loss": 1.138,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.118568778038025,
      "rewards/margins": -0.47426342964172363,
      "rewards/rejected": -0.644305408000946,
      "step": 1411
    },
    {
      "epoch": 0.18477140754068863,
      "grad_norm": 6.0207839012146,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32956239581108093,
      "logits/rejected": 0.3590199947357178,
      "logps/chosen": -452.8714599609375,
      "logps/rejected": -427.18853759765625,
      "loss": 0.5046,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32599374651908875,
      "rewards/margins": 0.8314304351806641,
      "rewards/rejected": -1.1574242115020752,
      "step": 1412
    },
    {
      "epoch": 0.1849022654780404,
      "grad_norm": 9.333597183227539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005060799419879913,
      "logits/rejected": 0.13227909803390503,
      "logps/chosen": -435.07373046875,
      "logps/rejected": -363.30657958984375,
      "loss": 0.689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7494451403617859,
      "rewards/margins": 0.26613733172416687,
      "rewards/rejected": -1.01558256149292,
      "step": 1413
    },
    {
      "epoch": 0.18503312341539216,
      "grad_norm": 12.304281234741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1929883062839508,
      "logits/rejected": 0.018159370869398117,
      "logps/chosen": -669.1541748046875,
      "logps/rejected": -479.525390625,
      "loss": 0.7767,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5092760324478149,
      "rewards/margins": 0.33129191398620605,
      "rewards/rejected": -0.8405680060386658,
      "step": 1414
    },
    {
      "epoch": 0.18516398135274392,
      "grad_norm": 6.940375804901123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.46134817600250244,
      "logits/rejected": 0.4682505130767822,
      "logps/chosen": -579.57958984375,
      "logps/rejected": -369.39642333984375,
      "loss": 0.5356,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5258299112319946,
      "rewards/margins": 0.6946078538894653,
      "rewards/rejected": -1.2204376459121704,
      "step": 1415
    },
    {
      "epoch": 0.1852948392900957,
      "grad_norm": 7.419173717498779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1633118987083435,
      "logits/rejected": 0.17440448701381683,
      "logps/chosen": -538.420166015625,
      "logps/rejected": -473.5823974609375,
      "loss": 0.5855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5104967355728149,
      "rewards/margins": 0.40035608410835266,
      "rewards/rejected": -0.91085284948349,
      "step": 1416
    },
    {
      "epoch": 0.18542569722744745,
      "grad_norm": 7.708871364593506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08391456305980682,
      "logits/rejected": -0.11988625675439835,
      "logps/chosen": -708.774169921875,
      "logps/rejected": -621.1200561523438,
      "loss": 0.7655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7891706824302673,
      "rewards/margins": 0.011187829077243805,
      "rewards/rejected": -0.8003584742546082,
      "step": 1417
    },
    {
      "epoch": 0.18555655516479921,
      "grad_norm": 7.001254558563232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2508499026298523,
      "logits/rejected": 0.3299875557422638,
      "logps/chosen": -553.8438110351562,
      "logps/rejected": -545.8262939453125,
      "loss": 0.5845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03038787841796875,
      "rewards/margins": 0.4704604744911194,
      "rewards/rejected": -0.4400726556777954,
      "step": 1418
    },
    {
      "epoch": 0.18568741310215098,
      "grad_norm": 5.899073123931885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4982829689979553,
      "logits/rejected": 0.13393595814704895,
      "logps/chosen": -502.6246032714844,
      "logps/rejected": -529.274169921875,
      "loss": 0.4405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10308971256017685,
      "rewards/margins": 0.7556440830230713,
      "rewards/rejected": -0.8587337732315063,
      "step": 1419
    },
    {
      "epoch": 0.18581827103950274,
      "grad_norm": 7.318719863891602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03132244199514389,
      "logits/rejected": 0.29975494742393494,
      "logps/chosen": -598.7439575195312,
      "logps/rejected": -722.52294921875,
      "loss": 0.4965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28548452258110046,
      "rewards/margins": 0.5394769310951233,
      "rewards/rejected": -0.8249614834785461,
      "step": 1420
    },
    {
      "epoch": 0.1859491289768545,
      "grad_norm": 8.206186294555664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02281365543603897,
      "logits/rejected": -0.06856352835893631,
      "logps/chosen": -610.1513061523438,
      "logps/rejected": -587.128662109375,
      "loss": 0.5607,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5094627737998962,
      "rewards/margins": 0.5909896492958069,
      "rewards/rejected": -1.1004524230957031,
      "step": 1421
    },
    {
      "epoch": 0.18607998691420627,
      "grad_norm": 7.550895690917969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12991204857826233,
      "logits/rejected": 0.166792094707489,
      "logps/chosen": -448.1126403808594,
      "logps/rejected": -471.96527099609375,
      "loss": 0.5435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03161945939064026,
      "rewards/margins": 0.656450092792511,
      "rewards/rejected": -0.6248306632041931,
      "step": 1422
    },
    {
      "epoch": 0.18621084485155803,
      "grad_norm": 5.487965106964111,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2962810695171356,
      "logits/rejected": 0.22548356652259827,
      "logps/chosen": -484.75244140625,
      "logps/rejected": -342.69317626953125,
      "loss": 0.5295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08589822053909302,
      "rewards/margins": 0.44902098178863525,
      "rewards/rejected": -0.36312276124954224,
      "step": 1423
    },
    {
      "epoch": 0.1863417027889098,
      "grad_norm": 6.386390209197998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06357789039611816,
      "logits/rejected": 0.11118721961975098,
      "logps/chosen": -427.8117370605469,
      "logps/rejected": -413.9621887207031,
      "loss": 0.6198,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07447260618209839,
      "rewards/margins": 0.37172985076904297,
      "rewards/rejected": -0.44620245695114136,
      "step": 1424
    },
    {
      "epoch": 0.18647256072626156,
      "grad_norm": 7.123742580413818,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2491256296634674,
      "logits/rejected": 0.23967787623405457,
      "logps/chosen": -555.8401489257812,
      "logps/rejected": -532.9956665039062,
      "loss": 0.5831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5532563328742981,
      "rewards/margins": 0.8512923717498779,
      "rewards/rejected": -1.4045486450195312,
      "step": 1425
    },
    {
      "epoch": 0.18660341866361332,
      "grad_norm": 6.995690822601318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17202147841453552,
      "logits/rejected": 0.12528273463249207,
      "logps/chosen": -643.906982421875,
      "logps/rejected": -410.21636962890625,
      "loss": 0.4269,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.029837816953659058,
      "rewards/margins": 0.8501144647598267,
      "rewards/rejected": -0.82027667760849,
      "step": 1426
    },
    {
      "epoch": 0.1867342766009651,
      "grad_norm": 6.511109352111816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4586063325405121,
      "logits/rejected": 0.4606720209121704,
      "logps/chosen": -409.0434875488281,
      "logps/rejected": -406.0213623046875,
      "loss": 0.5729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3132522702217102,
      "rewards/margins": 0.5488975644111633,
      "rewards/rejected": -0.8621498346328735,
      "step": 1427
    },
    {
      "epoch": 0.18686513453831685,
      "grad_norm": 7.974130630493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1587199568748474,
      "logits/rejected": 0.17185017466545105,
      "logps/chosen": -368.2178955078125,
      "logps/rejected": -371.382568359375,
      "loss": 0.6306,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4552774429321289,
      "rewards/margins": 0.28603097796440125,
      "rewards/rejected": -0.7413083910942078,
      "step": 1428
    },
    {
      "epoch": 0.18699599247566862,
      "grad_norm": 6.369266033172607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1806536316871643,
      "logits/rejected": 0.2800854742527008,
      "logps/chosen": -347.6073303222656,
      "logps/rejected": -436.8675842285156,
      "loss": 0.6376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1487768143415451,
      "rewards/margins": 0.1881454586982727,
      "rewards/rejected": -0.336922287940979,
      "step": 1429
    },
    {
      "epoch": 0.18712685041302035,
      "grad_norm": 9.59298038482666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19636499881744385,
      "logits/rejected": 0.33400359749794006,
      "logps/chosen": -389.7686767578125,
      "logps/rejected": -390.7712097167969,
      "loss": 1.0188,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.28046452999115,
      "rewards/margins": -0.4341007173061371,
      "rewards/rejected": -0.8463638424873352,
      "step": 1430
    },
    {
      "epoch": 0.18725770835037212,
      "grad_norm": 4.729372501373291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5064786076545715,
      "logits/rejected": 0.4084514081478119,
      "logps/chosen": -523.8765869140625,
      "logps/rejected": -461.11956787109375,
      "loss": 0.442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38125649094581604,
      "rewards/margins": 0.8801425695419312,
      "rewards/rejected": -0.4988861083984375,
      "step": 1431
    },
    {
      "epoch": 0.18738856628772388,
      "grad_norm": 5.60236120223999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09136820584535599,
      "logits/rejected": 0.12228095531463623,
      "logps/chosen": -407.14312744140625,
      "logps/rejected": -395.74737548828125,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.028675831854343414,
      "rewards/margins": 0.9302463531494141,
      "rewards/rejected": -0.9589222073554993,
      "step": 1432
    },
    {
      "epoch": 0.18751942422507564,
      "grad_norm": 6.07200288772583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24082867801189423,
      "logits/rejected": 0.2864305377006531,
      "logps/chosen": -383.3518371582031,
      "logps/rejected": -443.1893310546875,
      "loss": 0.4344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16146928071975708,
      "rewards/margins": 0.957411527633667,
      "rewards/rejected": -1.1188808679580688,
      "step": 1433
    },
    {
      "epoch": 0.1876502821624274,
      "grad_norm": 5.068595886230469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35672903060913086,
      "logits/rejected": 0.4811750650405884,
      "logps/chosen": -430.53021240234375,
      "logps/rejected": -381.099609375,
      "loss": 0.4327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0808546245098114,
      "rewards/margins": 0.8305397033691406,
      "rewards/rejected": -0.9113943576812744,
      "step": 1434
    },
    {
      "epoch": 0.18778114009977917,
      "grad_norm": 8.552297592163086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0886697918176651,
      "logits/rejected": 0.24348053336143494,
      "logps/chosen": -538.1785888671875,
      "logps/rejected": -441.77996826171875,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14397239685058594,
      "rewards/margins": 0.3531153202056885,
      "rewards/rejected": -0.49708765745162964,
      "step": 1435
    },
    {
      "epoch": 0.18791199803713093,
      "grad_norm": 8.172479629516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32607170939445496,
      "logits/rejected": 0.006866931915283203,
      "logps/chosen": -763.884033203125,
      "logps/rejected": -710.1310424804688,
      "loss": 0.7497,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03231659159064293,
      "rewards/margins": 0.31711235642433167,
      "rewards/rejected": -0.3494289517402649,
      "step": 1436
    },
    {
      "epoch": 0.1880428559744827,
      "grad_norm": 5.8899359703063965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5443946123123169,
      "logits/rejected": 0.15535950660705566,
      "logps/chosen": -408.3439025878906,
      "logps/rejected": -423.1444091796875,
      "loss": 0.4879,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.44887953996658325,
      "rewards/margins": 0.7693731188774109,
      "rewards/rejected": -0.32049360871315,
      "step": 1437
    },
    {
      "epoch": 0.18817371391183446,
      "grad_norm": 4.939028739929199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18085983395576477,
      "logits/rejected": 0.1858862340450287,
      "logps/chosen": -458.6636657714844,
      "logps/rejected": -537.9986572265625,
      "loss": 0.4868,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5941157937049866,
      "rewards/margins": 0.5994998812675476,
      "rewards/rejected": -1.1936157941818237,
      "step": 1438
    },
    {
      "epoch": 0.18830457184918623,
      "grad_norm": 7.1362104415893555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22840633988380432,
      "logits/rejected": 0.2675928473472595,
      "logps/chosen": -384.55340576171875,
      "logps/rejected": -429.9285888671875,
      "loss": 0.68,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03551274910569191,
      "rewards/margins": 0.20264069736003876,
      "rewards/rejected": -0.23815345764160156,
      "step": 1439
    },
    {
      "epoch": 0.188435429786538,
      "grad_norm": 6.723074436187744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48400816321372986,
      "logits/rejected": 0.6379063725471497,
      "logps/chosen": -467.16485595703125,
      "logps/rejected": -373.5516052246094,
      "loss": 0.6079,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15323658287525177,
      "rewards/margins": 0.2838377356529236,
      "rewards/rejected": -0.4370742738246918,
      "step": 1440
    },
    {
      "epoch": 0.18856628772388975,
      "grad_norm": 6.109511375427246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1372872292995453,
      "logits/rejected": 0.09221937507390976,
      "logps/chosen": -593.1813354492188,
      "logps/rejected": -717.6236572265625,
      "loss": 0.6315,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6954383850097656,
      "rewards/margins": 0.31830501556396484,
      "rewards/rejected": -1.0137434005737305,
      "step": 1441
    },
    {
      "epoch": 0.18869714566124152,
      "grad_norm": 5.277354717254639,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35198289155960083,
      "logits/rejected": 0.5753973722457886,
      "logps/chosen": -441.41461181640625,
      "logps/rejected": -431.01123046875,
      "loss": 0.3915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0563032403588295,
      "rewards/margins": 1.0508021116256714,
      "rewards/rejected": -1.1071052551269531,
      "step": 1442
    },
    {
      "epoch": 0.18882800359859328,
      "grad_norm": 5.230937480926514,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5313243269920349,
      "logits/rejected": 0.5747338533401489,
      "logps/chosen": -388.922119140625,
      "logps/rejected": -338.0077819824219,
      "loss": 0.5598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16175080835819244,
      "rewards/margins": 0.6190342307090759,
      "rewards/rejected": -0.4572833776473999,
      "step": 1443
    },
    {
      "epoch": 0.18895886153594504,
      "grad_norm": 6.156514644622803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37888410687446594,
      "logits/rejected": 0.388234943151474,
      "logps/chosen": -479.77044677734375,
      "logps/rejected": -441.5209655761719,
      "loss": 0.5033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11294898390769958,
      "rewards/margins": 0.6809458136558533,
      "rewards/rejected": -0.7938947677612305,
      "step": 1444
    },
    {
      "epoch": 0.1890897194732968,
      "grad_norm": 7.06194543838501,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4309242367744446,
      "logits/rejected": 0.1949268877506256,
      "logps/chosen": -406.375,
      "logps/rejected": -380.1624755859375,
      "loss": 0.4754,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.00041083991527557373,
      "rewards/margins": 0.7050064206123352,
      "rewards/rejected": -0.7045955657958984,
      "step": 1445
    },
    {
      "epoch": 0.18922057741064857,
      "grad_norm": 7.875975131988525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18589524924755096,
      "logits/rejected": 0.47399577498435974,
      "logps/chosen": -317.24481201171875,
      "logps/rejected": -363.766845703125,
      "loss": 0.5811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7218714356422424,
      "rewards/margins": 0.5255346894264221,
      "rewards/rejected": -1.247406005859375,
      "step": 1446
    },
    {
      "epoch": 0.18935143534800034,
      "grad_norm": 4.294034481048584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4104401469230652,
      "logits/rejected": 0.24136072397232056,
      "logps/chosen": -730.5204467773438,
      "logps/rejected": -613.7267456054688,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20223960280418396,
      "rewards/margins": 1.4738754034042358,
      "rewards/rejected": -1.271635890007019,
      "step": 1447
    },
    {
      "epoch": 0.1894822932853521,
      "grad_norm": 3.6835696697235107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13910488784313202,
      "logits/rejected": 0.2441331148147583,
      "logps/chosen": -373.2566223144531,
      "logps/rejected": -341.932861328125,
      "loss": 0.3842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04782446473836899,
      "rewards/margins": 0.9406156539916992,
      "rewards/rejected": -0.9884401559829712,
      "step": 1448
    },
    {
      "epoch": 0.18961315122270386,
      "grad_norm": 3.4651846885681152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37675559520721436,
      "logits/rejected": 0.4201125502586365,
      "logps/chosen": -377.75750732421875,
      "logps/rejected": -300.4089050292969,
      "loss": 0.251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3786983788013458,
      "rewards/margins": 1.4898419380187988,
      "rewards/rejected": -1.8685400485992432,
      "step": 1449
    },
    {
      "epoch": 0.18974400916005563,
      "grad_norm": 10.677620887756348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2753536105155945,
      "logits/rejected": 0.35153377056121826,
      "logps/chosen": -507.01507568359375,
      "logps/rejected": -401.57122802734375,
      "loss": 0.4917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1618383675813675,
      "rewards/margins": 0.6253917813301086,
      "rewards/rejected": -0.7872301340103149,
      "step": 1450
    },
    {
      "epoch": 0.1898748670974074,
      "grad_norm": 7.950805187225342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05296008288860321,
      "logits/rejected": 0.093140609562397,
      "logps/chosen": -442.3695068359375,
      "logps/rejected": -365.27862548828125,
      "loss": 0.6696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5037716031074524,
      "rewards/margins": 0.33371931314468384,
      "rewards/rejected": -0.8374909162521362,
      "step": 1451
    },
    {
      "epoch": 0.19000572503475913,
      "grad_norm": 5.793763637542725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3700472116470337,
      "logits/rejected": 0.035295236855745316,
      "logps/chosen": -441.31646728515625,
      "logps/rejected": -382.7018127441406,
      "loss": 0.4444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18664930760860443,
      "rewards/margins": 0.6771951913833618,
      "rewards/rejected": -0.8638445734977722,
      "step": 1452
    },
    {
      "epoch": 0.1901365829721109,
      "grad_norm": 6.287108421325684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44170045852661133,
      "logits/rejected": 0.23433539271354675,
      "logps/chosen": -311.47119140625,
      "logps/rejected": -315.11627197265625,
      "loss": 0.5405,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6088117361068726,
      "rewards/margins": 0.46732842922210693,
      "rewards/rejected": -1.076140284538269,
      "step": 1453
    },
    {
      "epoch": 0.19026744090946265,
      "grad_norm": 4.910284042358398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16417643427848816,
      "logits/rejected": 0.1402568221092224,
      "logps/chosen": -385.05218505859375,
      "logps/rejected": -294.26934814453125,
      "loss": 0.4424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3665313422679901,
      "rewards/margins": 0.8070181608200073,
      "rewards/rejected": -1.1735495328903198,
      "step": 1454
    },
    {
      "epoch": 0.19039829884681442,
      "grad_norm": 8.371663093566895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2788989245891571,
      "logits/rejected": 0.29139992594718933,
      "logps/chosen": -438.6199645996094,
      "logps/rejected": -512.8232421875,
      "loss": 0.8387,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.8800838589668274,
      "rewards/margins": -0.00171661376953125,
      "rewards/rejected": -0.8783673048019409,
      "step": 1455
    },
    {
      "epoch": 0.19052915678416618,
      "grad_norm": 5.880861759185791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43328389525413513,
      "logits/rejected": 0.3015616536140442,
      "logps/chosen": -316.5831298828125,
      "logps/rejected": -341.6006774902344,
      "loss": 0.6041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35612279176712036,
      "rewards/margins": 0.357352614402771,
      "rewards/rejected": -0.7134754657745361,
      "step": 1456
    },
    {
      "epoch": 0.19066001472151795,
      "grad_norm": 6.159584045410156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23147264122962952,
      "logits/rejected": 0.14636699855327606,
      "logps/chosen": -448.124755859375,
      "logps/rejected": -322.7166748046875,
      "loss": 0.5653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31690290570259094,
      "rewards/margins": 0.6804491281509399,
      "rewards/rejected": -0.997352123260498,
      "step": 1457
    },
    {
      "epoch": 0.1907908726588697,
      "grad_norm": 6.098782539367676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4218682050704956,
      "logits/rejected": 0.16797152161598206,
      "logps/chosen": -528.1682739257812,
      "logps/rejected": -513.1668701171875,
      "loss": 0.5389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.029361329972743988,
      "rewards/margins": 0.535382091999054,
      "rewards/rejected": -0.5647433996200562,
      "step": 1458
    },
    {
      "epoch": 0.19092173059622147,
      "grad_norm": 4.3725152015686035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13277864456176758,
      "logits/rejected": 0.048418596386909485,
      "logps/chosen": -395.12896728515625,
      "logps/rejected": -369.6496887207031,
      "loss": 0.4274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1845703125,
      "rewards/margins": 0.7832136154174805,
      "rewards/rejected": -0.9677839279174805,
      "step": 1459
    },
    {
      "epoch": 0.19105258853357324,
      "grad_norm": 6.085221767425537,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5683110952377319,
      "logits/rejected": 0.5021497011184692,
      "logps/chosen": -441.8641357421875,
      "logps/rejected": -525.6557006835938,
      "loss": 0.5243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12175160646438599,
      "rewards/margins": 0.5631216168403625,
      "rewards/rejected": -0.6848732233047485,
      "step": 1460
    },
    {
      "epoch": 0.191183446470925,
      "grad_norm": 8.635608673095703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5323820114135742,
      "logits/rejected": 0.4517000615596771,
      "logps/chosen": -347.3861389160156,
      "logps/rejected": -432.47137451171875,
      "loss": 0.938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6940310001373291,
      "rewards/margins": -0.19482597708702087,
      "rewards/rejected": -0.4992050528526306,
      "step": 1461
    },
    {
      "epoch": 0.19131430440827676,
      "grad_norm": 5.8787665367126465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4923207759857178,
      "logits/rejected": 0.3821524381637573,
      "logps/chosen": -584.9556884765625,
      "logps/rejected": -385.59307861328125,
      "loss": 0.5055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17793312668800354,
      "rewards/margins": 0.9881430864334106,
      "rewards/rejected": -0.8102100491523743,
      "step": 1462
    },
    {
      "epoch": 0.19144516234562853,
      "grad_norm": 6.283586025238037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25476688146591187,
      "logits/rejected": 0.21328428387641907,
      "logps/chosen": -645.2772827148438,
      "logps/rejected": -647.343017578125,
      "loss": 0.6419,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.32619109749794006,
      "rewards/margins": 0.4839082360267639,
      "rewards/rejected": -0.15771710872650146,
      "step": 1463
    },
    {
      "epoch": 0.1915760202829803,
      "grad_norm": 5.259274482727051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4808656573295593,
      "logits/rejected": 0.19706079363822937,
      "logps/chosen": -349.5205993652344,
      "logps/rejected": -305.9451599121094,
      "loss": 0.477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015721138566732407,
      "rewards/margins": 1.1024452447891235,
      "rewards/rejected": -1.086724042892456,
      "step": 1464
    },
    {
      "epoch": 0.19170687822033206,
      "grad_norm": 6.669494152069092,
      "learning_rate": 5e-06,
      "logits/chosen": 0.166109099984169,
      "logits/rejected": 0.2555394768714905,
      "logps/chosen": -394.6505126953125,
      "logps/rejected": -486.67529296875,
      "loss": 0.5837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3609933853149414,
      "rewards/margins": 0.29650723934173584,
      "rewards/rejected": -0.657500684261322,
      "step": 1465
    },
    {
      "epoch": 0.19183773615768382,
      "grad_norm": 7.442462921142578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3933041989803314,
      "logits/rejected": 0.18848423659801483,
      "logps/chosen": -326.19573974609375,
      "logps/rejected": -269.83563232421875,
      "loss": 0.7421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0259582996368408,
      "rewards/margins": 0.0692722350358963,
      "rewards/rejected": -1.0952304601669312,
      "step": 1466
    },
    {
      "epoch": 0.19196859409503558,
      "grad_norm": 4.587414741516113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11280976235866547,
      "logits/rejected": 0.08762896060943604,
      "logps/chosen": -495.62249755859375,
      "logps/rejected": -503.14959716796875,
      "loss": 0.293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7545974254608154,
      "rewards/margins": 1.2557028532028198,
      "rewards/rejected": -0.5011053085327148,
      "step": 1467
    },
    {
      "epoch": 0.19209945203238735,
      "grad_norm": 9.414039611816406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21828648447990417,
      "logits/rejected": 0.48786216974258423,
      "logps/chosen": -521.9716796875,
      "logps/rejected": -461.3291320800781,
      "loss": 0.9883,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8421341180801392,
      "rewards/margins": -0.4114823341369629,
      "rewards/rejected": -0.4306517541408539,
      "step": 1468
    },
    {
      "epoch": 0.1922303099697391,
      "grad_norm": 4.792756080627441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22702741622924805,
      "logits/rejected": 0.20116207003593445,
      "logps/chosen": -445.20013427734375,
      "logps/rejected": -302.12347412109375,
      "loss": 0.2965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17160701751708984,
      "rewards/margins": 1.196284294128418,
      "rewards/rejected": -1.0246772766113281,
      "step": 1469
    },
    {
      "epoch": 0.19236116790709087,
      "grad_norm": 7.680704593658447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3391861915588379,
      "logits/rejected": 0.4172360599040985,
      "logps/chosen": -446.5043640136719,
      "logps/rejected": -396.7310485839844,
      "loss": 0.5308,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18927304446697235,
      "rewards/margins": 0.8089286684989929,
      "rewards/rejected": -0.9982017278671265,
      "step": 1470
    },
    {
      "epoch": 0.19249202584444264,
      "grad_norm": 7.234835624694824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06283707171678543,
      "logits/rejected": 0.02142450585961342,
      "logps/chosen": -602.7545776367188,
      "logps/rejected": -629.3095703125,
      "loss": 0.667,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3378709554672241,
      "rewards/margins": 0.2570013403892517,
      "rewards/rejected": -0.5948722958564758,
      "step": 1471
    },
    {
      "epoch": 0.1926228837817944,
      "grad_norm": 7.219685077667236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034302711486816406,
      "logits/rejected": -0.018553754314780235,
      "logps/chosen": -351.871337890625,
      "logps/rejected": -443.6197509765625,
      "loss": 0.7017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09625282883644104,
      "rewards/margins": 0.094525545835495,
      "rewards/rejected": -0.19077835977077484,
      "step": 1472
    },
    {
      "epoch": 0.19275374171914617,
      "grad_norm": 7.201086044311523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.455200731754303,
      "logits/rejected": 0.5051369667053223,
      "logps/chosen": -390.8583984375,
      "logps/rejected": -442.8631591796875,
      "loss": 0.7218,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0027486830949783325,
      "rewards/margins": 0.04921034723520279,
      "rewards/rejected": -0.046461671590805054,
      "step": 1473
    },
    {
      "epoch": 0.1928845996564979,
      "grad_norm": 9.217825889587402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3506474196910858,
      "logits/rejected": 0.20131689310073853,
      "logps/chosen": -417.1873779296875,
      "logps/rejected": -370.5143127441406,
      "loss": 0.7154,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6952712535858154,
      "rewards/margins": 0.1870870590209961,
      "rewards/rejected": -0.8823583722114563,
      "step": 1474
    },
    {
      "epoch": 0.19301545759384967,
      "grad_norm": 8.456788063049316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25491225719451904,
      "logits/rejected": 0.13480766117572784,
      "logps/chosen": -519.5387573242188,
      "logps/rejected": -525.8939208984375,
      "loss": 0.6903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3154006600379944,
      "rewards/margins": 0.46465688943862915,
      "rewards/rejected": -0.1492561399936676,
      "step": 1475
    },
    {
      "epoch": 0.19314631553120143,
      "grad_norm": 8.095511436462402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27022555470466614,
      "logits/rejected": 0.5517089366912842,
      "logps/chosen": -464.1978454589844,
      "logps/rejected": -669.7392578125,
      "loss": 0.6105,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7029049396514893,
      "rewards/margins": 0.41921156644821167,
      "rewards/rejected": -1.1221164464950562,
      "step": 1476
    },
    {
      "epoch": 0.1932771734685532,
      "grad_norm": 6.136693477630615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2690500020980835,
      "logits/rejected": 0.3261691927909851,
      "logps/chosen": -528.7014770507812,
      "logps/rejected": -476.27850341796875,
      "loss": 0.4633,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12198429554700851,
      "rewards/margins": 0.7126394510269165,
      "rewards/rejected": -0.8346236944198608,
      "step": 1477
    },
    {
      "epoch": 0.19340803140590496,
      "grad_norm": 7.213533401489258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42026978731155396,
      "logits/rejected": 0.21497470140457153,
      "logps/chosen": -310.65185546875,
      "logps/rejected": -385.2447509765625,
      "loss": 0.8004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4410203993320465,
      "rewards/margins": 0.12000119686126709,
      "rewards/rejected": -0.561021625995636,
      "step": 1478
    },
    {
      "epoch": 0.19353888934325672,
      "grad_norm": 3.7891836166381836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13710278272628784,
      "logits/rejected": 0.15090812742710114,
      "logps/chosen": -734.718994140625,
      "logps/rejected": -462.2588195800781,
      "loss": 0.2122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3639053702354431,
      "rewards/margins": 1.6915652751922607,
      "rewards/rejected": -1.327660083770752,
      "step": 1479
    },
    {
      "epoch": 0.19366974728060848,
      "grad_norm": 7.176054000854492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18186336755752563,
      "logits/rejected": -0.14751940965652466,
      "logps/chosen": -530.4100952148438,
      "logps/rejected": -414.11663818359375,
      "loss": 0.5822,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6880772113800049,
      "rewards/margins": 0.3968982696533203,
      "rewards/rejected": -1.0849754810333252,
      "step": 1480
    },
    {
      "epoch": 0.19380060521796025,
      "grad_norm": 4.643540382385254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17260080575942993,
      "logits/rejected": 0.07537662237882614,
      "logps/chosen": -457.8730773925781,
      "logps/rejected": -301.28253173828125,
      "loss": 0.3967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.041338153183460236,
      "rewards/margins": 0.959109902381897,
      "rewards/rejected": -1.0004479885101318,
      "step": 1481
    },
    {
      "epoch": 0.193931463155312,
      "grad_norm": 7.734600067138672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3079918324947357,
      "logits/rejected": 0.40347665548324585,
      "logps/chosen": -629.7025756835938,
      "logps/rejected": -593.64306640625,
      "loss": 0.5264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09140549600124359,
      "rewards/margins": 0.8005750179290771,
      "rewards/rejected": -0.7091696262359619,
      "step": 1482
    },
    {
      "epoch": 0.19406232109266378,
      "grad_norm": 6.776084899902344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23299039900302887,
      "logits/rejected": 0.2649746239185333,
      "logps/chosen": -626.1719970703125,
      "logps/rejected": -606.1198120117188,
      "loss": 0.4989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1075717955827713,
      "rewards/margins": 0.6839943528175354,
      "rewards/rejected": -0.7915660738945007,
      "step": 1483
    },
    {
      "epoch": 0.19419317903001554,
      "grad_norm": 6.05797815322876,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36369115114212036,
      "logits/rejected": 0.3751583397388458,
      "logps/chosen": -400.85003662109375,
      "logps/rejected": -402.6280517578125,
      "loss": 0.6959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39983177185058594,
      "rewards/margins": 0.22796648740768433,
      "rewards/rejected": -0.6277982592582703,
      "step": 1484
    },
    {
      "epoch": 0.1943240369673673,
      "grad_norm": 5.516050815582275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1004801094532013,
      "logits/rejected": 0.06584780663251877,
      "logps/chosen": -470.8399353027344,
      "logps/rejected": -438.8363342285156,
      "loss": 0.4859,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.422846257686615,
      "rewards/margins": 0.5544719696044922,
      "rewards/rejected": -0.9773181080818176,
      "step": 1485
    },
    {
      "epoch": 0.19445489490471907,
      "grad_norm": 7.378890514373779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2088608741760254,
      "logits/rejected": 0.03389094024896622,
      "logps/chosen": -389.23321533203125,
      "logps/rejected": -463.45703125,
      "loss": 0.6695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5800601840019226,
      "rewards/margins": 0.2704549729824066,
      "rewards/rejected": -0.8505151271820068,
      "step": 1486
    },
    {
      "epoch": 0.19458575284207083,
      "grad_norm": 7.241060733795166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24271012842655182,
      "logits/rejected": 0.026800895109772682,
      "logps/chosen": -771.72412109375,
      "logps/rejected": -731.76123046875,
      "loss": 0.4723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08422283083200455,
      "rewards/margins": 0.8296177387237549,
      "rewards/rejected": -0.9138405323028564,
      "step": 1487
    },
    {
      "epoch": 0.1947166107794226,
      "grad_norm": 6.6708149909973145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23772014677524567,
      "logits/rejected": -0.05348160117864609,
      "logps/chosen": -471.23992919921875,
      "logps/rejected": -442.7004089355469,
      "loss": 0.4802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9554720520973206,
      "rewards/margins": 0.6659361124038696,
      "rewards/rejected": -1.6214081048965454,
      "step": 1488
    },
    {
      "epoch": 0.19484746871677436,
      "grad_norm": 5.8694329261779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15837666392326355,
      "logits/rejected": 0.24375659227371216,
      "logps/chosen": -319.5304870605469,
      "logps/rejected": -494.9156799316406,
      "loss": 0.5002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8074987530708313,
      "rewards/margins": 0.5619430541992188,
      "rewards/rejected": -1.3694417476654053,
      "step": 1489
    },
    {
      "epoch": 0.19497832665412612,
      "grad_norm": 6.107609748840332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14084580540657043,
      "logits/rejected": 0.0850645899772644,
      "logps/chosen": -345.7321472167969,
      "logps/rejected": -360.2664489746094,
      "loss": 0.5115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4374597668647766,
      "rewards/margins": 0.5249162912368774,
      "rewards/rejected": -0.9623759984970093,
      "step": 1490
    },
    {
      "epoch": 0.19510918459147789,
      "grad_norm": 5.2606587409973145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5418625473976135,
      "logits/rejected": 0.2683466076850891,
      "logps/chosen": -353.11297607421875,
      "logps/rejected": -439.9118347167969,
      "loss": 0.3336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5898264050483704,
      "rewards/margins": 1.5859286785125732,
      "rewards/rejected": -2.175755023956299,
      "step": 1491
    },
    {
      "epoch": 0.19524004252882965,
      "grad_norm": 13.834721565246582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06315630674362183,
      "logits/rejected": 0.17454323172569275,
      "logps/chosen": -707.080322265625,
      "logps/rejected": -437.970458984375,
      "loss": 0.8877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.652479887008667,
      "rewards/margins": 0.07144203782081604,
      "rewards/rejected": -0.7239219546318054,
      "step": 1492
    },
    {
      "epoch": 0.1953709004661814,
      "grad_norm": 5.298254489898682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11027887463569641,
      "logits/rejected": 0.02895922213792801,
      "logps/chosen": -388.0087890625,
      "logps/rejected": -381.71966552734375,
      "loss": 0.4418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4241443872451782,
      "rewards/margins": 0.9230666756629944,
      "rewards/rejected": -1.3472110033035278,
      "step": 1493
    },
    {
      "epoch": 0.19550175840353318,
      "grad_norm": 13.252735137939453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2256334125995636,
      "logits/rejected": 0.14825013279914856,
      "logps/chosen": -553.9749755859375,
      "logps/rejected": -542.4268798828125,
      "loss": 0.9123,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0236115455627441,
      "rewards/margins": -0.049912258982658386,
      "rewards/rejected": -0.9736992120742798,
      "step": 1494
    },
    {
      "epoch": 0.19563261634088494,
      "grad_norm": 8.581405639648438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2665908634662628,
      "logits/rejected": 0.1900699883699417,
      "logps/chosen": -442.7811584472656,
      "logps/rejected": -462.10595703125,
      "loss": 0.6657,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015151962637901306,
      "rewards/margins": 0.1166364774107933,
      "rewards/rejected": -0.1317884624004364,
      "step": 1495
    },
    {
      "epoch": 0.19576347427823668,
      "grad_norm": 6.431162357330322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06574684381484985,
      "logits/rejected": -0.0803687646985054,
      "logps/chosen": -406.0054931640625,
      "logps/rejected": -392.25286865234375,
      "loss": 0.66,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45268210768699646,
      "rewards/margins": 0.14618094265460968,
      "rewards/rejected": -0.5988630652427673,
      "step": 1496
    },
    {
      "epoch": 0.19589433221558844,
      "grad_norm": 9.504630088806152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44192689657211304,
      "logits/rejected": 0.28108131885528564,
      "logps/chosen": -708.3389282226562,
      "logps/rejected": -550.1646728515625,
      "loss": 0.7336,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7694798111915588,
      "rewards/margins": 0.0808238834142685,
      "rewards/rejected": -0.8503036499023438,
      "step": 1497
    },
    {
      "epoch": 0.1960251901529402,
      "grad_norm": 6.172067165374756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07760321348905563,
      "logits/rejected": 0.013656561262905598,
      "logps/chosen": -271.0511779785156,
      "logps/rejected": -302.4821472167969,
      "loss": 0.6536,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8237857818603516,
      "rewards/margins": 0.4682005047798157,
      "rewards/rejected": -1.291986346244812,
      "step": 1498
    },
    {
      "epoch": 0.19615604809029197,
      "grad_norm": 4.959343910217285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22725023329257965,
      "logits/rejected": 0.404400497674942,
      "logps/chosen": -575.7108764648438,
      "logps/rejected": -569.1397094726562,
      "loss": 0.436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09238053113222122,
      "rewards/margins": 0.8909592628479004,
      "rewards/rejected": -0.798578679561615,
      "step": 1499
    },
    {
      "epoch": 0.19628690602764373,
      "grad_norm": 8.162095069885254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.308264821767807,
      "logits/rejected": 0.3750608563423157,
      "logps/chosen": -370.2229919433594,
      "logps/rejected": -415.9062194824219,
      "loss": 0.7514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6447247862815857,
      "rewards/margins": 0.39214932918548584,
      "rewards/rejected": -1.0368742942810059,
      "step": 1500
    },
    {
      "epoch": 0.1964177639649955,
      "grad_norm": 9.371994018554688,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42634811997413635,
      "logits/rejected": 0.41853439807891846,
      "logps/chosen": -326.0806884765625,
      "logps/rejected": -395.0132141113281,
      "loss": 1.1384,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5114168524742126,
      "rewards/margins": -0.5833175778388977,
      "rewards/rejected": 0.07190074771642685,
      "step": 1501
    },
    {
      "epoch": 0.19654862190234726,
      "grad_norm": 6.650393962860107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09400001168251038,
      "logits/rejected": 0.021442260593175888,
      "logps/chosen": -268.52777099609375,
      "logps/rejected": -391.5986633300781,
      "loss": 0.5845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08884851634502411,
      "rewards/margins": 0.43028658628463745,
      "rewards/rejected": -0.519135057926178,
      "step": 1502
    },
    {
      "epoch": 0.19667947983969902,
      "grad_norm": 8.145048141479492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03131544589996338,
      "logits/rejected": 0.08462119847536087,
      "logps/chosen": -510.739501953125,
      "logps/rejected": -547.6748657226562,
      "loss": 0.764,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3160904049873352,
      "rewards/margins": -0.024160772562026978,
      "rewards/rejected": -0.2919296622276306,
      "step": 1503
    },
    {
      "epoch": 0.1968103377770508,
      "grad_norm": 7.982932090759277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3681345582008362,
      "logits/rejected": 0.4140682518482208,
      "logps/chosen": -656.7000732421875,
      "logps/rejected": -530.6424560546875,
      "loss": 0.526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03417608141899109,
      "rewards/margins": 0.771670401096344,
      "rewards/rejected": -0.8058464527130127,
      "step": 1504
    },
    {
      "epoch": 0.19694119571440255,
      "grad_norm": 7.980602741241455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14049798250198364,
      "logits/rejected": -0.05676138773560524,
      "logps/chosen": -571.9048461914062,
      "logps/rejected": -495.87957763671875,
      "loss": 0.5551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14302502572536469,
      "rewards/margins": 0.6817833185195923,
      "rewards/rejected": -0.8248083591461182,
      "step": 1505
    },
    {
      "epoch": 0.19707205365175431,
      "grad_norm": 5.984898090362549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11216762661933899,
      "logits/rejected": 0.01688477396965027,
      "logps/chosen": -636.163818359375,
      "logps/rejected": -539.1488647460938,
      "loss": 0.4275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03825397044420242,
      "rewards/margins": 0.7764222621917725,
      "rewards/rejected": -0.8146763443946838,
      "step": 1506
    },
    {
      "epoch": 0.19720291158910608,
      "grad_norm": 9.109359741210938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34143272042274475,
      "logits/rejected": 0.18965941667556763,
      "logps/chosen": -442.7980041503906,
      "logps/rejected": -412.9026794433594,
      "loss": 0.5403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4621356725692749,
      "rewards/margins": 0.5668430924415588,
      "rewards/rejected": -1.0289788246154785,
      "step": 1507
    },
    {
      "epoch": 0.19733376952645784,
      "grad_norm": 7.371047496795654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3711216151714325,
      "logits/rejected": 0.3389611840248108,
      "logps/chosen": -491.8630676269531,
      "logps/rejected": -469.5267639160156,
      "loss": 0.6367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1029357761144638,
      "rewards/margins": 0.3005126714706421,
      "rewards/rejected": -0.1975768804550171,
      "step": 1508
    },
    {
      "epoch": 0.1974646274638096,
      "grad_norm": 13.978419303894043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24198681116104126,
      "logits/rejected": 0.37193596363067627,
      "logps/chosen": -679.4879150390625,
      "logps/rejected": -396.508544921875,
      "loss": 0.9014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9805115461349487,
      "rewards/margins": 0.2922893464565277,
      "rewards/rejected": -1.2728009223937988,
      "step": 1509
    },
    {
      "epoch": 0.19759548540116137,
      "grad_norm": 5.930739879608154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.303997278213501,
      "logits/rejected": 0.3339426815509796,
      "logps/chosen": -318.7220764160156,
      "logps/rejected": -324.1798400878906,
      "loss": 0.4577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5376676917076111,
      "rewards/margins": 0.7145012021064758,
      "rewards/rejected": -1.252168893814087,
      "step": 1510
    },
    {
      "epoch": 0.19772634333851313,
      "grad_norm": 5.977047443389893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2805873155593872,
      "logits/rejected": 0.24764537811279297,
      "logps/chosen": -291.16009521484375,
      "logps/rejected": -389.4228515625,
      "loss": 0.6634,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6752164363861084,
      "rewards/margins": 0.234685480594635,
      "rewards/rejected": -0.909902036190033,
      "step": 1511
    },
    {
      "epoch": 0.1978572012758649,
      "grad_norm": 6.1111602783203125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3082197904586792,
      "logits/rejected": 0.21432732045650482,
      "logps/chosen": -614.1582641601562,
      "logps/rejected": -628.063232421875,
      "loss": 0.4272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08341195434331894,
      "rewards/margins": 0.9310683012008667,
      "rewards/rejected": -1.0144803524017334,
      "step": 1512
    },
    {
      "epoch": 0.19798805921321666,
      "grad_norm": 7.548707485198975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01672777161002159,
      "logits/rejected": 0.08287689089775085,
      "logps/chosen": -362.887939453125,
      "logps/rejected": -579.2698364257812,
      "loss": 0.4795,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3267820477485657,
      "rewards/margins": 0.8787750601768494,
      "rewards/rejected": -1.205557107925415,
      "step": 1513
    },
    {
      "epoch": 0.19811891715056842,
      "grad_norm": 7.731341361999512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21580882370471954,
      "logits/rejected": -0.06492134183645248,
      "logps/chosen": -528.22509765625,
      "logps/rejected": -307.9245300292969,
      "loss": 0.6373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43823373317718506,
      "rewards/margins": 0.2360767424106598,
      "rewards/rejected": -0.6743105053901672,
      "step": 1514
    },
    {
      "epoch": 0.1982497750879202,
      "grad_norm": 11.020035743713379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3723672032356262,
      "logits/rejected": 0.3666578233242035,
      "logps/chosen": -474.99554443359375,
      "logps/rejected": -440.9884338378906,
      "loss": 0.7935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6726354360580444,
      "rewards/margins": 0.017262689769268036,
      "rewards/rejected": -0.6898981332778931,
      "step": 1515
    },
    {
      "epoch": 0.19838063302527195,
      "grad_norm": 6.776141166687012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19645769894123077,
      "logits/rejected": 0.2041974663734436,
      "logps/chosen": -392.3638916015625,
      "logps/rejected": -434.5881042480469,
      "loss": 0.5944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.540930986404419,
      "rewards/margins": 0.6295899748802185,
      "rewards/rejected": -1.1705210208892822,
      "step": 1516
    },
    {
      "epoch": 0.1985114909626237,
      "grad_norm": 9.49674129486084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3292206823825836,
      "logits/rejected": 0.4595646262168884,
      "logps/chosen": -350.38482666015625,
      "logps/rejected": -327.89215087890625,
      "loss": 0.8349,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.5369644165039062,
      "rewards/margins": -0.0660480260848999,
      "rewards/rejected": -1.470916509628296,
      "step": 1517
    },
    {
      "epoch": 0.19864234889997545,
      "grad_norm": 6.447634696960449,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10342254489660263,
      "logits/rejected": 0.04262041300535202,
      "logps/chosen": -343.9677734375,
      "logps/rejected": -511.115234375,
      "loss": 0.5476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45298004150390625,
      "rewards/margins": 0.6675640344619751,
      "rewards/rejected": -1.1205440759658813,
      "step": 1518
    },
    {
      "epoch": 0.19877320683732722,
      "grad_norm": 5.590062618255615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3046593964099884,
      "logits/rejected": 0.12157832086086273,
      "logps/chosen": -380.45294189453125,
      "logps/rejected": -309.2688903808594,
      "loss": 0.3633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1381380259990692,
      "rewards/margins": 1.223543405532837,
      "rewards/rejected": -1.361681342124939,
      "step": 1519
    },
    {
      "epoch": 0.19890406477467898,
      "grad_norm": 4.990044593811035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17320726811885834,
      "logits/rejected": 0.09395720809698105,
      "logps/chosen": -479.2425537109375,
      "logps/rejected": -356.0357360839844,
      "loss": 0.3671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05163116008043289,
      "rewards/margins": 0.9719741940498352,
      "rewards/rejected": -0.9203430414199829,
      "step": 1520
    },
    {
      "epoch": 0.19903492271203074,
      "grad_norm": 4.158878326416016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13773491978645325,
      "logits/rejected": 0.1028248518705368,
      "logps/chosen": -352.3724365234375,
      "logps/rejected": -356.204833984375,
      "loss": 0.3349,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2780449092388153,
      "rewards/margins": 1.2327080965042114,
      "rewards/rejected": -1.5107531547546387,
      "step": 1521
    },
    {
      "epoch": 0.1991657806493825,
      "grad_norm": 5.041606426239014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27189573645591736,
      "logits/rejected": 0.4028882086277008,
      "logps/chosen": -417.5672607421875,
      "logps/rejected": -372.8102111816406,
      "loss": 0.4938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20052415132522583,
      "rewards/margins": 0.6848406195640564,
      "rewards/rejected": -0.8853647708892822,
      "step": 1522
    },
    {
      "epoch": 0.19929663858673427,
      "grad_norm": 4.472341537475586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1233854740858078,
      "logits/rejected": -0.00986778736114502,
      "logps/chosen": -463.8456726074219,
      "logps/rejected": -629.339599609375,
      "loss": 0.3678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07469920814037323,
      "rewards/margins": 1.4833483695983887,
      "rewards/rejected": -1.4086490869522095,
      "step": 1523
    },
    {
      "epoch": 0.19942749652408603,
      "grad_norm": 10.51034164428711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21971499919891357,
      "logits/rejected": 0.10668349266052246,
      "logps/chosen": -611.4494018554688,
      "logps/rejected": -524.8128662109375,
      "loss": 0.7695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09849052131175995,
      "rewards/margins": 0.32867586612701416,
      "rewards/rejected": -0.4271663725376129,
      "step": 1524
    },
    {
      "epoch": 0.1995583544614378,
      "grad_norm": 4.265204906463623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26786044239997864,
      "logits/rejected": 0.2952324151992798,
      "logps/chosen": -365.0628662109375,
      "logps/rejected": -439.43804931640625,
      "loss": 0.4444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3351452052593231,
      "rewards/margins": 1.4408632516860962,
      "rewards/rejected": -1.7760084867477417,
      "step": 1525
    },
    {
      "epoch": 0.19968921239878956,
      "grad_norm": 6.305601596832275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28847557306289673,
      "logits/rejected": 0.33624595403671265,
      "logps/chosen": -507.3763427734375,
      "logps/rejected": -438.8899230957031,
      "loss": 0.6092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4148866832256317,
      "rewards/margins": 0.5266435742378235,
      "rewards/rejected": -0.941530168056488,
      "step": 1526
    },
    {
      "epoch": 0.19982007033614133,
      "grad_norm": 9.427597045898438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05121816322207451,
      "logits/rejected": 0.16127634048461914,
      "logps/chosen": -419.81427001953125,
      "logps/rejected": -367.510498046875,
      "loss": 0.9392,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1124423742294312,
      "rewards/margins": -0.23785053193569183,
      "rewards/rejected": -0.8745918273925781,
      "step": 1527
    },
    {
      "epoch": 0.1999509282734931,
      "grad_norm": 7.615073204040527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3396840989589691,
      "logits/rejected": 0.3713061511516571,
      "logps/chosen": -666.452392578125,
      "logps/rejected": -597.2042846679688,
      "loss": 0.5642,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13829536736011505,
      "rewards/margins": 0.8112598657608032,
      "rewards/rejected": -0.9495551586151123,
      "step": 1528
    },
    {
      "epoch": 0.20008178621084485,
      "grad_norm": 7.495523929595947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5150149464607239,
      "logits/rejected": 0.5156888961791992,
      "logps/chosen": -604.286376953125,
      "logps/rejected": -625.38232421875,
      "loss": 0.7152,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04222603142261505,
      "rewards/margins": 0.2406417578458786,
      "rewards/rejected": -0.19841572642326355,
      "step": 1529
    },
    {
      "epoch": 0.20021264414819662,
      "grad_norm": 6.417133808135986,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3457128703594208,
      "logits/rejected": 0.5145444869995117,
      "logps/chosen": -346.70391845703125,
      "logps/rejected": -499.89398193359375,
      "loss": 0.4528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5782915353775024,
      "rewards/margins": 0.830655574798584,
      "rewards/rejected": -1.408947229385376,
      "step": 1530
    },
    {
      "epoch": 0.20034350208554838,
      "grad_norm": 3.4485089778900146,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11706662178039551,
      "logits/rejected": 0.1578521430492401,
      "logps/chosen": -329.5772705078125,
      "logps/rejected": -337.5687561035156,
      "loss": 0.3064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44900569319725037,
      "rewards/margins": 1.226346731185913,
      "rewards/rejected": -0.7773410677909851,
      "step": 1531
    },
    {
      "epoch": 0.20047436002290014,
      "grad_norm": 8.008111000061035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14823558926582336,
      "logits/rejected": 0.08590270578861237,
      "logps/chosen": -312.3904113769531,
      "logps/rejected": -381.58734130859375,
      "loss": 0.7625,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0930837392807007,
      "rewards/margins": 0.30069246888160706,
      "rewards/rejected": -1.3937761783599854,
      "step": 1532
    },
    {
      "epoch": 0.2006052179602519,
      "grad_norm": 6.042459487915039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2912200093269348,
      "logits/rejected": 0.3427152633666992,
      "logps/chosen": -525.0509033203125,
      "logps/rejected": -473.08966064453125,
      "loss": 0.5655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2907610237598419,
      "rewards/margins": 0.5582658648490906,
      "rewards/rejected": -0.8490268588066101,
      "step": 1533
    },
    {
      "epoch": 0.20073607589760367,
      "grad_norm": 5.948034286499023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3013802468776703,
      "logits/rejected": 0.48268091678619385,
      "logps/chosen": -338.1727600097656,
      "logps/rejected": -387.1832275390625,
      "loss": 0.5222,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03529128432273865,
      "rewards/margins": 0.7810369729995728,
      "rewards/rejected": -0.7457456588745117,
      "step": 1534
    },
    {
      "epoch": 0.20086693383495544,
      "grad_norm": 6.287509918212891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33591428399086,
      "logits/rejected": 0.19964860379695892,
      "logps/chosen": -425.2501220703125,
      "logps/rejected": -441.361328125,
      "loss": 0.422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16898994147777557,
      "rewards/margins": 1.3235315084457397,
      "rewards/rejected": -1.1545416116714478,
      "step": 1535
    },
    {
      "epoch": 0.2009977917723072,
      "grad_norm": 5.4692792892456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11768732964992523,
      "logits/rejected": -0.18067960441112518,
      "logps/chosen": -827.7406616210938,
      "logps/rejected": -517.0037231445312,
      "loss": 0.3946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06065371632575989,
      "rewards/margins": 1.3296278715133667,
      "rewards/rejected": -1.2689740657806396,
      "step": 1536
    },
    {
      "epoch": 0.20112864970965896,
      "grad_norm": 6.131885051727295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3478911221027374,
      "logits/rejected": 0.37806034088134766,
      "logps/chosen": -437.65966796875,
      "logps/rejected": -449.42596435546875,
      "loss": 0.6505,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09740085154771805,
      "rewards/margins": 0.6153404712677002,
      "rewards/rejected": -0.7127412557601929,
      "step": 1537
    },
    {
      "epoch": 0.20125950764701073,
      "grad_norm": 5.938581943511963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3394330143928528,
      "logits/rejected": 0.04701196402311325,
      "logps/chosen": -454.812744140625,
      "logps/rejected": -361.5082702636719,
      "loss": 0.4648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5681412220001221,
      "rewards/margins": 0.6704646944999695,
      "rewards/rejected": -1.2386058568954468,
      "step": 1538
    },
    {
      "epoch": 0.20139036558436246,
      "grad_norm": 6.863369464874268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5608338117599487,
      "logits/rejected": 0.1400374323129654,
      "logps/chosen": -387.26202392578125,
      "logps/rejected": -428.1656799316406,
      "loss": 0.5987,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7912700176239014,
      "rewards/margins": 0.42048323154449463,
      "rewards/rejected": -1.211753249168396,
      "step": 1539
    },
    {
      "epoch": 0.20152122352171423,
      "grad_norm": 8.077468872070312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2095264345407486,
      "logits/rejected": 0.24441872537136078,
      "logps/chosen": -539.9713134765625,
      "logps/rejected": -472.5371398925781,
      "loss": 0.6526,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8899992108345032,
      "rewards/margins": 0.3077799081802368,
      "rewards/rejected": -1.1977791786193848,
      "step": 1540
    },
    {
      "epoch": 0.201652081459066,
      "grad_norm": 5.1826491355896,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3232819437980652,
      "logits/rejected": 0.265228271484375,
      "logps/chosen": -360.03399658203125,
      "logps/rejected": -393.93994140625,
      "loss": 0.4433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5415537357330322,
      "rewards/margins": 0.6567649841308594,
      "rewards/rejected": -1.198318600654602,
      "step": 1541
    },
    {
      "epoch": 0.20178293939641775,
      "grad_norm": 10.093147277832031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34088608622550964,
      "logits/rejected": 0.44535350799560547,
      "logps/chosen": -521.1168212890625,
      "logps/rejected": -580.2416381835938,
      "loss": 1.0062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6927775144577026,
      "rewards/margins": -0.08716034889221191,
      "rewards/rejected": -0.6056171655654907,
      "step": 1542
    },
    {
      "epoch": 0.20191379733376952,
      "grad_norm": 5.138392925262451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3969796895980835,
      "logits/rejected": 0.1635083109140396,
      "logps/chosen": -454.6839599609375,
      "logps/rejected": -346.322021484375,
      "loss": 0.4773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30891379714012146,
      "rewards/margins": 0.8585010170936584,
      "rewards/rejected": -1.167414903640747,
      "step": 1543
    },
    {
      "epoch": 0.20204465527112128,
      "grad_norm": 5.143338203430176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5662461519241333,
      "logits/rejected": 0.23733073472976685,
      "logps/chosen": -445.93475341796875,
      "logps/rejected": -581.2528686523438,
      "loss": 0.4515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07220669090747833,
      "rewards/margins": 1.927796721458435,
      "rewards/rejected": -1.8555899858474731,
      "step": 1544
    },
    {
      "epoch": 0.20217551320847305,
      "grad_norm": 5.953774452209473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4667975902557373,
      "logits/rejected": 0.28325241804122925,
      "logps/chosen": -533.91162109375,
      "logps/rejected": -638.0672607421875,
      "loss": 0.4061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.036355406045913696,
      "rewards/margins": 0.9393362402915955,
      "rewards/rejected": -0.9029808640480042,
      "step": 1545
    },
    {
      "epoch": 0.2023063711458248,
      "grad_norm": 5.288307189941406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2672271132469177,
      "logits/rejected": 0.2104547768831253,
      "logps/chosen": -459.7061462402344,
      "logps/rejected": -487.54486083984375,
      "loss": 0.5314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6691257953643799,
      "rewards/margins": 0.6088086366653442,
      "rewards/rejected": -1.2779345512390137,
      "step": 1546
    },
    {
      "epoch": 0.20243722908317657,
      "grad_norm": 5.880424976348877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40055331587791443,
      "logits/rejected": 0.27582287788391113,
      "logps/chosen": -457.57818603515625,
      "logps/rejected": -488.49542236328125,
      "loss": 0.5112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36034148931503296,
      "rewards/margins": 0.7182297110557556,
      "rewards/rejected": -1.0785712003707886,
      "step": 1547
    },
    {
      "epoch": 0.20256808702052834,
      "grad_norm": 5.488514423370361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27031582593917847,
      "logits/rejected": 0.2987692058086395,
      "logps/chosen": -445.725830078125,
      "logps/rejected": -387.8257751464844,
      "loss": 0.4596,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47891342639923096,
      "rewards/margins": 0.8654338121414185,
      "rewards/rejected": -1.3443472385406494,
      "step": 1548
    },
    {
      "epoch": 0.2026989449578801,
      "grad_norm": 5.65243673324585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29719674587249756,
      "logits/rejected": 0.23058530688285828,
      "logps/chosen": -488.9651184082031,
      "logps/rejected": -480.6865234375,
      "loss": 0.5273,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5398765802383423,
      "rewards/margins": 0.7670799493789673,
      "rewards/rejected": -1.3069565296173096,
      "step": 1549
    },
    {
      "epoch": 0.20282980289523186,
      "grad_norm": 6.252970218658447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11425594985485077,
      "logits/rejected": 0.10512211918830872,
      "logps/chosen": -496.97235107421875,
      "logps/rejected": -356.9266357421875,
      "loss": 0.6019,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5007709264755249,
      "rewards/margins": 0.39949214458465576,
      "rewards/rejected": -0.9002630710601807,
      "step": 1550
    },
    {
      "epoch": 0.20296066083258363,
      "grad_norm": 5.4082231521606445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16186216473579407,
      "logits/rejected": 0.011991966515779495,
      "logps/chosen": -572.5079345703125,
      "logps/rejected": -512.228271484375,
      "loss": 0.4564,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1726524531841278,
      "rewards/margins": 0.7224029302597046,
      "rewards/rejected": -0.549750566482544,
      "step": 1551
    },
    {
      "epoch": 0.2030915187699354,
      "grad_norm": 6.474915504455566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18477042019367218,
      "logits/rejected": 0.1819101721048355,
      "logps/chosen": -305.58551025390625,
      "logps/rejected": -341.8258056640625,
      "loss": 0.4814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5021181702613831,
      "rewards/margins": 0.8517202734947205,
      "rewards/rejected": -1.3538384437561035,
      "step": 1552
    },
    {
      "epoch": 0.20322237670728716,
      "grad_norm": 5.293283462524414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2295430302619934,
      "logits/rejected": 0.02418157085776329,
      "logps/chosen": -356.2518005371094,
      "logps/rejected": -352.75048828125,
      "loss": 0.4623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7672539949417114,
      "rewards/margins": 0.671733021736145,
      "rewards/rejected": -1.4389870166778564,
      "step": 1553
    },
    {
      "epoch": 0.20335323464463892,
      "grad_norm": 4.742854118347168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20020879805088043,
      "logits/rejected": 0.13328592479228973,
      "logps/chosen": -776.4041748046875,
      "logps/rejected": -670.6566772460938,
      "loss": 0.3167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05886650085449219,
      "rewards/margins": 1.3156582117080688,
      "rewards/rejected": -1.256791591644287,
      "step": 1554
    },
    {
      "epoch": 0.20348409258199068,
      "grad_norm": 5.688377380371094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3189910650253296,
      "logits/rejected": -0.18435703217983246,
      "logps/chosen": -368.5418701171875,
      "logps/rejected": -471.8758239746094,
      "loss": 0.3535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27503547072410583,
      "rewards/margins": 1.1671431064605713,
      "rewards/rejected": -1.4421786069869995,
      "step": 1555
    },
    {
      "epoch": 0.20361495051934245,
      "grad_norm": 9.249809265136719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3044687807559967,
      "logits/rejected": 0.33241426944732666,
      "logps/chosen": -552.6856689453125,
      "logps/rejected": -445.5600280761719,
      "loss": 0.6748,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5088071823120117,
      "rewards/margins": 0.1612936109304428,
      "rewards/rejected": -0.6701008081436157,
      "step": 1556
    },
    {
      "epoch": 0.2037458084566942,
      "grad_norm": 7.125090599060059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21561893820762634,
      "logits/rejected": 0.368299663066864,
      "logps/chosen": -449.40618896484375,
      "logps/rejected": -533.4061279296875,
      "loss": 0.5442,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1333286166191101,
      "rewards/margins": 0.8248959183692932,
      "rewards/rejected": -0.9582245349884033,
      "step": 1557
    },
    {
      "epoch": 0.20387666639404597,
      "grad_norm": 3.880776882171631,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04124479740858078,
      "logits/rejected": 0.26396483182907104,
      "logps/chosen": -405.01708984375,
      "logps/rejected": -430.0631103515625,
      "loss": 0.3823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10781954228878021,
      "rewards/margins": 1.0708746910095215,
      "rewards/rejected": -1.1786941289901733,
      "step": 1558
    },
    {
      "epoch": 0.20400752433139774,
      "grad_norm": 7.215897083282471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3595485985279083,
      "logits/rejected": -0.14481644332408905,
      "logps/chosen": -539.742919921875,
      "logps/rejected": -593.7314453125,
      "loss": 0.5355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0246529579162598,
      "rewards/margins": 0.7164119482040405,
      "rewards/rejected": -1.7410647869110107,
      "step": 1559
    },
    {
      "epoch": 0.2041383822687495,
      "grad_norm": 4.421838760375977,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28662729263305664,
      "logits/rejected": 0.27916574478149414,
      "logps/chosen": -462.9450988769531,
      "logps/rejected": -468.4219665527344,
      "loss": 0.2675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30178412795066833,
      "rewards/margins": 1.4341795444488525,
      "rewards/rejected": -1.7359635829925537,
      "step": 1560
    },
    {
      "epoch": 0.20426924020610124,
      "grad_norm": 9.16181468963623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21978792548179626,
      "logits/rejected": 0.2597184479236603,
      "logps/chosen": -322.88818359375,
      "logps/rejected": -355.35845947265625,
      "loss": 0.9306,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.46765953302383423,
      "rewards/margins": -0.31518620252609253,
      "rewards/rejected": -0.15247337520122528,
      "step": 1561
    },
    {
      "epoch": 0.204400098143453,
      "grad_norm": 7.033092975616455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08661462366580963,
      "logits/rejected": 0.04780120402574539,
      "logps/chosen": -618.5977783203125,
      "logps/rejected": -467.745361328125,
      "loss": 0.5487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9181075096130371,
      "rewards/margins": 0.605186939239502,
      "rewards/rejected": -1.523294448852539,
      "step": 1562
    },
    {
      "epoch": 0.20453095608080477,
      "grad_norm": 6.580048084259033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3743572235107422,
      "logits/rejected": 0.3395662009716034,
      "logps/chosen": -318.7887878417969,
      "logps/rejected": -466.9486999511719,
      "loss": 0.4662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9838077425956726,
      "rewards/margins": 1.2847702503204346,
      "rewards/rejected": -2.268578052520752,
      "step": 1563
    },
    {
      "epoch": 0.20466181401815653,
      "grad_norm": 8.407154083251953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5330454111099243,
      "logits/rejected": 0.6319204568862915,
      "logps/chosen": -465.48248291015625,
      "logps/rejected": -406.9346618652344,
      "loss": 0.5896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8018136620521545,
      "rewards/margins": 0.593074381351471,
      "rewards/rejected": -1.394888162612915,
      "step": 1564
    },
    {
      "epoch": 0.2047926719555083,
      "grad_norm": 6.719120502471924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3846761882305145,
      "logits/rejected": 0.25800615549087524,
      "logps/chosen": -545.48974609375,
      "logps/rejected": -442.8037109375,
      "loss": 0.4656,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31670284271240234,
      "rewards/margins": 0.8734702467918396,
      "rewards/rejected": -1.1901731491088867,
      "step": 1565
    },
    {
      "epoch": 0.20492352989286006,
      "grad_norm": 5.080894470214844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2020304650068283,
      "logits/rejected": -0.012432865798473358,
      "logps/chosen": -476.310546875,
      "logps/rejected": -403.7491760253906,
      "loss": 0.4252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6079787015914917,
      "rewards/margins": 0.8590721487998962,
      "rewards/rejected": -1.4670507907867432,
      "step": 1566
    },
    {
      "epoch": 0.20505438783021182,
      "grad_norm": 4.741890907287598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07713872939348221,
      "logits/rejected": -0.03166995197534561,
      "logps/chosen": -428.0339660644531,
      "logps/rejected": -479.4508056640625,
      "loss": 0.307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28962990641593933,
      "rewards/margins": 1.1543718576431274,
      "rewards/rejected": -1.4440016746520996,
      "step": 1567
    },
    {
      "epoch": 0.20518524576756358,
      "grad_norm": 6.127986431121826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07720937579870224,
      "logits/rejected": 0.14549699425697327,
      "logps/chosen": -406.449462890625,
      "logps/rejected": -513.9782104492188,
      "loss": 0.5569,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6899347901344299,
      "rewards/margins": 0.7407395839691162,
      "rewards/rejected": -1.4306743144989014,
      "step": 1568
    },
    {
      "epoch": 0.20531610370491535,
      "grad_norm": 3.2816579341888428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4748232364654541,
      "logits/rejected": 0.30835509300231934,
      "logps/chosen": -440.1314392089844,
      "logps/rejected": -425.6125183105469,
      "loss": 0.2984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06886137276887894,
      "rewards/margins": 1.486051321029663,
      "rewards/rejected": -1.554912805557251,
      "step": 1569
    },
    {
      "epoch": 0.2054469616422671,
      "grad_norm": 4.355599403381348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24386216700077057,
      "logits/rejected": -0.005222693085670471,
      "logps/chosen": -323.560302734375,
      "logps/rejected": -436.2895202636719,
      "loss": 0.4467,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8861028552055359,
      "rewards/margins": 0.9358911514282227,
      "rewards/rejected": -1.8219940662384033,
      "step": 1570
    },
    {
      "epoch": 0.20557781957961888,
      "grad_norm": 6.37306547164917,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31876787543296814,
      "logits/rejected": 0.3452886939048767,
      "logps/chosen": -454.825927734375,
      "logps/rejected": -475.03326416015625,
      "loss": 0.4764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0439602136611938,
      "rewards/margins": 1.1018307209014893,
      "rewards/rejected": -2.1457908153533936,
      "step": 1571
    },
    {
      "epoch": 0.20570867751697064,
      "grad_norm": 9.4723539352417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11756429821252823,
      "logits/rejected": 0.07875314354896545,
      "logps/chosen": -445.5096130371094,
      "logps/rejected": -423.97955322265625,
      "loss": 0.6615,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1303688287734985,
      "rewards/margins": 0.2285444438457489,
      "rewards/rejected": -1.3589133024215698,
      "step": 1572
    },
    {
      "epoch": 0.2058395354543224,
      "grad_norm": 9.367728233337402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0826757550239563,
      "logits/rejected": 0.347042977809906,
      "logps/chosen": -577.868896484375,
      "logps/rejected": -513.971923828125,
      "loss": 0.8149,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0553977489471436,
      "rewards/margins": -0.06125161051750183,
      "rewards/rejected": -0.9941461682319641,
      "step": 1573
    },
    {
      "epoch": 0.20597039339167417,
      "grad_norm": 7.316382884979248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22960057854652405,
      "logits/rejected": 0.2347000539302826,
      "logps/chosen": -440.88092041015625,
      "logps/rejected": -400.70458984375,
      "loss": 0.7166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0279302597045898,
      "rewards/margins": 0.023649990558624268,
      "rewards/rejected": -1.0515803098678589,
      "step": 1574
    },
    {
      "epoch": 0.20610125132902593,
      "grad_norm": 4.878828048706055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1325310319662094,
      "logits/rejected": 0.23657535016536713,
      "logps/chosen": -490.23333740234375,
      "logps/rejected": -376.4044189453125,
      "loss": 0.4048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8347702026367188,
      "rewards/margins": 1.577048659324646,
      "rewards/rejected": -2.4118189811706543,
      "step": 1575
    },
    {
      "epoch": 0.2062321092663777,
      "grad_norm": 5.165226936340332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28976571559906006,
      "logits/rejected": 0.23418673872947693,
      "logps/chosen": -400.9249572753906,
      "logps/rejected": -386.1745910644531,
      "loss": 0.3812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09222392737865448,
      "rewards/margins": 1.0535188913345337,
      "rewards/rejected": -1.1457427740097046,
      "step": 1576
    },
    {
      "epoch": 0.20636296720372946,
      "grad_norm": 4.383170127868652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09165871143341064,
      "logits/rejected": 0.016041353344917297,
      "logps/chosen": -339.1672058105469,
      "logps/rejected": -299.76422119140625,
      "loss": 0.3135,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17506182193756104,
      "rewards/margins": 1.3370896577835083,
      "rewards/rejected": -1.5121513605117798,
      "step": 1577
    },
    {
      "epoch": 0.20649382514108122,
      "grad_norm": 8.732390403747559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11982771754264832,
      "logits/rejected": 0.15874533355236053,
      "logps/chosen": -494.6857604980469,
      "logps/rejected": -371.9920349121094,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9011848568916321,
      "rewards/margins": 0.2341962456703186,
      "rewards/rejected": -1.1353811025619507,
      "step": 1578
    },
    {
      "epoch": 0.20662468307843299,
      "grad_norm": 5.751011848449707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3559342324733734,
      "logits/rejected": 0.47938108444213867,
      "logps/chosen": -324.9043273925781,
      "logps/rejected": -385.3706359863281,
      "loss": 0.4677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9557621479034424,
      "rewards/margins": 0.7080206274986267,
      "rewards/rejected": -1.6637828350067139,
      "step": 1579
    },
    {
      "epoch": 0.20675554101578475,
      "grad_norm": 16.031827926635742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13968655467033386,
      "logits/rejected": 0.0026218444108963013,
      "logps/chosen": -813.3671264648438,
      "logps/rejected": -579.9671020507812,
      "loss": 0.6627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9263565540313721,
      "rewards/margins": 0.6022592782974243,
      "rewards/rejected": -1.5286157131195068,
      "step": 1580
    },
    {
      "epoch": 0.2068863989531365,
      "grad_norm": 5.537709712982178,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20131368935108185,
      "logits/rejected": 0.10978446900844574,
      "logps/chosen": -313.7537841796875,
      "logps/rejected": -256.5167236328125,
      "loss": 0.534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0424009561538696,
      "rewards/margins": 0.44390618801116943,
      "rewards/rejected": -1.486307144165039,
      "step": 1581
    },
    {
      "epoch": 0.20701725689048828,
      "grad_norm": 12.396231651306152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3932986557483673,
      "logits/rejected": 0.0763639509677887,
      "logps/chosen": -309.294921875,
      "logps/rejected": -283.3524169921875,
      "loss": 1.0562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0099294185638428,
      "rewards/margins": 0.26123738288879395,
      "rewards/rejected": -1.2711668014526367,
      "step": 1582
    },
    {
      "epoch": 0.20714811482784,
      "grad_norm": 7.589815616607666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29249444603919983,
      "logits/rejected": 0.3239692747592926,
      "logps/chosen": -464.845947265625,
      "logps/rejected": -359.64923095703125,
      "loss": 0.5662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5618060827255249,
      "rewards/margins": 0.6440456509590149,
      "rewards/rejected": -1.205851674079895,
      "step": 1583
    },
    {
      "epoch": 0.20727897276519178,
      "grad_norm": 7.807676315307617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1355227530002594,
      "logits/rejected": 0.11186105757951736,
      "logps/chosen": -560.6228637695312,
      "logps/rejected": -594.5902099609375,
      "loss": 0.4476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4187747836112976,
      "rewards/margins": 1.0839622020721436,
      "rewards/rejected": -1.502737045288086,
      "step": 1584
    },
    {
      "epoch": 0.20740983070254354,
      "grad_norm": 10.927306175231934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2773277759552002,
      "logits/rejected": 0.12611068785190582,
      "logps/chosen": -524.6721801757812,
      "logps/rejected": -519.7127685546875,
      "loss": 0.8026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3003493547439575,
      "rewards/margins": 0.5736602544784546,
      "rewards/rejected": -1.874009609222412,
      "step": 1585
    },
    {
      "epoch": 0.2075406886398953,
      "grad_norm": 8.043726921081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22001348435878754,
      "logits/rejected": 0.16483455896377563,
      "logps/chosen": -328.40057373046875,
      "logps/rejected": -379.327392578125,
      "loss": 0.6411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8440300226211548,
      "rewards/margins": 0.4560428559780121,
      "rewards/rejected": -1.3000729084014893,
      "step": 1586
    },
    {
      "epoch": 0.20767154657724707,
      "grad_norm": 9.040068626403809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41407445073127747,
      "logits/rejected": 0.19221076369285583,
      "logps/chosen": -442.7354736328125,
      "logps/rejected": -398.6744384765625,
      "loss": 0.9843,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.548477292060852,
      "rewards/margins": -0.11692225188016891,
      "rewards/rejected": -0.43155500292778015,
      "step": 1587
    },
    {
      "epoch": 0.20780240451459883,
      "grad_norm": 7.641262531280518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24977873265743256,
      "logits/rejected": 0.22527393698692322,
      "logps/chosen": -418.3211669921875,
      "logps/rejected": -348.56109619140625,
      "loss": 0.8075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9404798746109009,
      "rewards/margins": 0.5345156192779541,
      "rewards/rejected": -1.4749953746795654,
      "step": 1588
    },
    {
      "epoch": 0.2079332624519506,
      "grad_norm": 5.056636810302734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2699360251426697,
      "logits/rejected": 0.16220974922180176,
      "logps/chosen": -474.0017395019531,
      "logps/rejected": -389.14093017578125,
      "loss": 0.4156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4134496748447418,
      "rewards/margins": 0.8536818027496338,
      "rewards/rejected": -1.2671313285827637,
      "step": 1589
    },
    {
      "epoch": 0.20806412038930236,
      "grad_norm": 6.194265842437744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4571373164653778,
      "logits/rejected": 0.34608739614486694,
      "logps/chosen": -365.82257080078125,
      "logps/rejected": -349.271728515625,
      "loss": 0.7005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2345608472824097,
      "rewards/margins": 0.22068868577480316,
      "rewards/rejected": -1.4552494287490845,
      "step": 1590
    },
    {
      "epoch": 0.20819497832665412,
      "grad_norm": 10.929823875427246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019942624494433403,
      "logits/rejected": 0.050103943794965744,
      "logps/chosen": -433.530517578125,
      "logps/rejected": -419.32879638671875,
      "loss": 0.9803,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.488647699356079,
      "rewards/margins": -0.12631511688232422,
      "rewards/rejected": -1.3623325824737549,
      "step": 1591
    },
    {
      "epoch": 0.2083258362640059,
      "grad_norm": 7.700056552886963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4322414994239807,
      "logits/rejected": 0.19366243481636047,
      "logps/chosen": -507.32476806640625,
      "logps/rejected": -319.9943542480469,
      "loss": 0.7205,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9674495458602905,
      "rewards/margins": 0.09086383879184723,
      "rewards/rejected": -1.0583133697509766,
      "step": 1592
    },
    {
      "epoch": 0.20845669420135765,
      "grad_norm": 4.0321364402771,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15968330204486847,
      "logits/rejected": 0.45370611548423767,
      "logps/chosen": -332.1723327636719,
      "logps/rejected": -381.41668701171875,
      "loss": 0.3492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2648313343524933,
      "rewards/margins": 1.2365975379943848,
      "rewards/rejected": -1.5014288425445557,
      "step": 1593
    },
    {
      "epoch": 0.20858755213870941,
      "grad_norm": 9.496339797973633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35209205746650696,
      "logits/rejected": 0.5961999297142029,
      "logps/chosen": -597.02392578125,
      "logps/rejected": -610.4136962890625,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5250808596611023,
      "rewards/margins": 0.2638639509677887,
      "rewards/rejected": -0.7889448404312134,
      "step": 1594
    },
    {
      "epoch": 0.20871841007606118,
      "grad_norm": 7.773284435272217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09026151150465012,
      "logits/rejected": 0.06524793803691864,
      "logps/chosen": -360.5002136230469,
      "logps/rejected": -438.4873046875,
      "loss": 0.5791,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.825872004032135,
      "rewards/margins": 0.8025181293487549,
      "rewards/rejected": -1.6283901929855347,
      "step": 1595
    },
    {
      "epoch": 0.20884926801341294,
      "grad_norm": 4.587086200714111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002755149267613888,
      "logits/rejected": 0.10376100987195969,
      "logps/chosen": -519.8883666992188,
      "logps/rejected": -478.0086364746094,
      "loss": 0.3885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04799196124076843,
      "rewards/margins": 0.857520341873169,
      "rewards/rejected": -0.9055122137069702,
      "step": 1596
    },
    {
      "epoch": 0.2089801259507647,
      "grad_norm": 6.286421775817871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0010221563279628754,
      "logits/rejected": 0.03215833008289337,
      "logps/chosen": -576.7210083007812,
      "logps/rejected": -516.210205078125,
      "loss": 0.3746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1159870624542236,
      "rewards/margins": 1.3644661903381348,
      "rewards/rejected": -2.4804530143737793,
      "step": 1597
    },
    {
      "epoch": 0.20911098388811647,
      "grad_norm": 7.312832355499268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.339354932308197,
      "logits/rejected": 0.19738037884235382,
      "logps/chosen": -382.05523681640625,
      "logps/rejected": -378.3344421386719,
      "loss": 0.616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8687306046485901,
      "rewards/margins": 0.3622409701347351,
      "rewards/rejected": -1.2309715747833252,
      "step": 1598
    },
    {
      "epoch": 0.20924184182546823,
      "grad_norm": 6.29341983795166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32624679803848267,
      "logits/rejected": 0.29898762702941895,
      "logps/chosen": -364.26983642578125,
      "logps/rejected": -681.158203125,
      "loss": 0.5197,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4442865550518036,
      "rewards/margins": 0.6191421151161194,
      "rewards/rejected": -1.0634287595748901,
      "step": 1599
    },
    {
      "epoch": 0.20937269976282,
      "grad_norm": 8.435253143310547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08844953775405884,
      "logits/rejected": 0.07852086424827576,
      "logps/chosen": -534.7936401367188,
      "logps/rejected": -443.9052429199219,
      "loss": 0.6384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6678626537322998,
      "rewards/margins": 0.5248807668685913,
      "rewards/rejected": -1.1927435398101807,
      "step": 1600
    },
    {
      "epoch": 0.20950355770017176,
      "grad_norm": 6.301697731018066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2535991668701172,
      "logits/rejected": 0.11351954936981201,
      "logps/chosen": -467.1095886230469,
      "logps/rejected": -404.6781005859375,
      "loss": 0.4863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09004439413547516,
      "rewards/margins": 0.8226194381713867,
      "rewards/rejected": -0.9126638174057007,
      "step": 1601
    },
    {
      "epoch": 0.20963441563752352,
      "grad_norm": 7.4021897315979,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5108079314231873,
      "logits/rejected": 0.46529144048690796,
      "logps/chosen": -502.0069580078125,
      "logps/rejected": -517.94580078125,
      "loss": 0.707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3284038305282593,
      "rewards/margins": 0.707588791847229,
      "rewards/rejected": -1.0359926223754883,
      "step": 1602
    },
    {
      "epoch": 0.2097652735748753,
      "grad_norm": 5.901860237121582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15693466365337372,
      "logits/rejected": 0.2674543559551239,
      "logps/chosen": -369.67633056640625,
      "logps/rejected": -464.297119140625,
      "loss": 0.5371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8696567416191101,
      "rewards/margins": 0.6883842349052429,
      "rewards/rejected": -1.5580410957336426,
      "step": 1603
    },
    {
      "epoch": 0.20989613151222705,
      "grad_norm": 7.221285343170166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3380684554576874,
      "logits/rejected": 0.3562631607055664,
      "logps/chosen": -696.7348022460938,
      "logps/rejected": -474.4405517578125,
      "loss": 0.3872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4631752073764801,
      "rewards/margins": 1.2322512865066528,
      "rewards/rejected": -0.7690761685371399,
      "step": 1604
    },
    {
      "epoch": 0.2100269894495788,
      "grad_norm": 5.671371936798096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20164261758327484,
      "logits/rejected": 0.25978386402130127,
      "logps/chosen": -428.90179443359375,
      "logps/rejected": -344.2623596191406,
      "loss": 0.4032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32353633642196655,
      "rewards/margins": 1.026577115058899,
      "rewards/rejected": -1.3501135110855103,
      "step": 1605
    },
    {
      "epoch": 0.21015784738693055,
      "grad_norm": 7.739414215087891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3689330518245697,
      "logits/rejected": 0.3632383644580841,
      "logps/chosen": -488.3548583984375,
      "logps/rejected": -378.1583251953125,
      "loss": 0.5181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7690792083740234,
      "rewards/margins": 0.584874153137207,
      "rewards/rejected": -1.3539533615112305,
      "step": 1606
    },
    {
      "epoch": 0.21028870532428232,
      "grad_norm": 5.883974552154541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2742452621459961,
      "logits/rejected": 0.3003138601779938,
      "logps/chosen": -488.80352783203125,
      "logps/rejected": -433.23046875,
      "loss": 0.4589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8822544813156128,
      "rewards/margins": 1.051621913909912,
      "rewards/rejected": -1.9338765144348145,
      "step": 1607
    },
    {
      "epoch": 0.21041956326163408,
      "grad_norm": 8.433554649353027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20190931856632233,
      "logits/rejected": 0.20102892816066742,
      "logps/chosen": -701.1825561523438,
      "logps/rejected": -588.4834594726562,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43493443727493286,
      "rewards/margins": 0.3748186230659485,
      "rewards/rejected": -0.8097530603408813,
      "step": 1608
    },
    {
      "epoch": 0.21055042119898584,
      "grad_norm": 9.130517959594727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11116665601730347,
      "logits/rejected": 0.08732821047306061,
      "logps/chosen": -507.3130798339844,
      "logps/rejected": -516.7645874023438,
      "loss": 0.7185,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9906408786773682,
      "rewards/margins": 0.21520408987998962,
      "rewards/rejected": -1.2058448791503906,
      "step": 1609
    },
    {
      "epoch": 0.2106812791363376,
      "grad_norm": 5.54408073425293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34286847710609436,
      "logits/rejected": 0.2267928272485733,
      "logps/chosen": -373.869873046875,
      "logps/rejected": -325.3022766113281,
      "loss": 0.4607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19146235287189484,
      "rewards/margins": 0.9853065013885498,
      "rewards/rejected": -1.1767688989639282,
      "step": 1610
    },
    {
      "epoch": 0.21081213707368937,
      "grad_norm": 5.728946208953857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12007563561201096,
      "logits/rejected": 0.21423539519309998,
      "logps/chosen": -402.12530517578125,
      "logps/rejected": -407.9708251953125,
      "loss": 0.4839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6868332028388977,
      "rewards/margins": 0.6614193320274353,
      "rewards/rejected": -1.348252534866333,
      "step": 1611
    },
    {
      "epoch": 0.21094299501104113,
      "grad_norm": 4.961996078491211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03428387641906738,
      "logits/rejected": 0.18080244958400726,
      "logps/chosen": -528.983642578125,
      "logps/rejected": -496.2413024902344,
      "loss": 0.4461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.004155355505645275,
      "rewards/margins": 0.7046451568603516,
      "rewards/rejected": -0.70048987865448,
      "step": 1612
    },
    {
      "epoch": 0.2110738529483929,
      "grad_norm": 5.522514343261719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09474870562553406,
      "logits/rejected": 0.00221291184425354,
      "logps/chosen": -471.51531982421875,
      "logps/rejected": -537.6749267578125,
      "loss": 0.3797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05862225964665413,
      "rewards/margins": 0.9175705313682556,
      "rewards/rejected": -0.9761928915977478,
      "step": 1613
    },
    {
      "epoch": 0.21120471088574466,
      "grad_norm": 6.198129653930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15743954479694366,
      "logits/rejected": 0.13675960898399353,
      "logps/chosen": -307.5985107421875,
      "logps/rejected": -327.9822998046875,
      "loss": 0.5792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35103586316108704,
      "rewards/margins": 0.8054441809654236,
      "rewards/rejected": -1.156480073928833,
      "step": 1614
    },
    {
      "epoch": 0.21133556882309643,
      "grad_norm": 9.774362564086914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2045830488204956,
      "logits/rejected": -0.022477850317955017,
      "logps/chosen": -428.7860412597656,
      "logps/rejected": -542.2938232421875,
      "loss": 0.6108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6838917136192322,
      "rewards/margins": 0.7717254161834717,
      "rewards/rejected": -1.4556171894073486,
      "step": 1615
    },
    {
      "epoch": 0.2114664267604482,
      "grad_norm": 6.948112487792969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3149155378341675,
      "logits/rejected": 0.28832724690437317,
      "logps/chosen": -358.4129638671875,
      "logps/rejected": -387.50799560546875,
      "loss": 0.5847,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9689842462539673,
      "rewards/margins": 0.37528282403945923,
      "rewards/rejected": -1.3442670106887817,
      "step": 1616
    },
    {
      "epoch": 0.21159728469779995,
      "grad_norm": 5.286250114440918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1286749690771103,
      "logits/rejected": 0.16485749185085297,
      "logps/chosen": -534.2615966796875,
      "logps/rejected": -418.4979248046875,
      "loss": 0.4126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.019670113921165466,
      "rewards/margins": 1.0920826196670532,
      "rewards/rejected": -1.0724124908447266,
      "step": 1617
    },
    {
      "epoch": 0.21172814263515172,
      "grad_norm": 5.3654632568359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24440309405326843,
      "logits/rejected": 0.2481827735900879,
      "logps/chosen": -353.986572265625,
      "logps/rejected": -471.03985595703125,
      "loss": 0.4114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6875451803207397,
      "rewards/margins": 0.9017126560211182,
      "rewards/rejected": -1.589257836341858,
      "step": 1618
    },
    {
      "epoch": 0.21185900057250348,
      "grad_norm": 6.594242095947266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028023716062307358,
      "logits/rejected": 0.04789309948682785,
      "logps/chosen": -404.6414489746094,
      "logps/rejected": -375.9303283691406,
      "loss": 0.6743,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9347617626190186,
      "rewards/margins": 0.1488981544971466,
      "rewards/rejected": -1.0836598873138428,
      "step": 1619
    },
    {
      "epoch": 0.21198985850985524,
      "grad_norm": 9.30884838104248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23251329362392426,
      "logits/rejected": -0.4228062629699707,
      "logps/chosen": -607.5587158203125,
      "logps/rejected": -554.1197509765625,
      "loss": 0.692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.222337245941162,
      "rewards/margins": 0.6368224024772644,
      "rewards/rejected": -1.8591597080230713,
      "step": 1620
    },
    {
      "epoch": 0.212120716447207,
      "grad_norm": 5.583690166473389,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1285465657711029,
      "logits/rejected": 0.11891863495111465,
      "logps/chosen": -317.6957702636719,
      "logps/rejected": -521.2423706054688,
      "loss": 0.4554,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8952924013137817,
      "rewards/margins": 1.866515874862671,
      "rewards/rejected": -2.761808156967163,
      "step": 1621
    },
    {
      "epoch": 0.21225157438455877,
      "grad_norm": 5.654390811920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1628165990114212,
      "logits/rejected": 0.31508398056030273,
      "logps/chosen": -395.84649658203125,
      "logps/rejected": -453.36529541015625,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04272383451461792,
      "rewards/margins": 0.9200608730316162,
      "rewards/rejected": -0.9627847671508789,
      "step": 1622
    },
    {
      "epoch": 0.21238243232191054,
      "grad_norm": 5.862757682800293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3825269341468811,
      "logits/rejected": 0.2638261020183563,
      "logps/chosen": -440.15533447265625,
      "logps/rejected": -494.2318115234375,
      "loss": 0.4601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015020555816590786,
      "rewards/margins": 1.1969863176345825,
      "rewards/rejected": -1.181965708732605,
      "step": 1623
    },
    {
      "epoch": 0.2125132902592623,
      "grad_norm": 13.932238578796387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04848424345254898,
      "logits/rejected": 0.14511163532733917,
      "logps/chosen": -401.37115478515625,
      "logps/rejected": -405.0115966796875,
      "loss": 0.8112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1659469604492188,
      "rewards/margins": 0.5495541095733643,
      "rewards/rejected": -1.7155009508132935,
      "step": 1624
    },
    {
      "epoch": 0.21264414819661406,
      "grad_norm": 7.111850738525391,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21612149477005005,
      "logits/rejected": 0.2632131576538086,
      "logps/chosen": -420.79681396484375,
      "logps/rejected": -316.1080017089844,
      "loss": 0.6171,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42801934480667114,
      "rewards/margins": 0.4943537414073944,
      "rewards/rejected": -0.9223730564117432,
      "step": 1625
    },
    {
      "epoch": 0.21277500613396583,
      "grad_norm": 7.1706743240356445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30566105246543884,
      "logits/rejected": 0.37977445125579834,
      "logps/chosen": -444.616943359375,
      "logps/rejected": -514.6507568359375,
      "loss": 0.5464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5413579940795898,
      "rewards/margins": 0.6456273198127747,
      "rewards/rejected": -1.1869852542877197,
      "step": 1626
    },
    {
      "epoch": 0.21290586407131756,
      "grad_norm": 7.960244655609131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09996230900287628,
      "logits/rejected": -0.12258090078830719,
      "logps/chosen": -483.69891357421875,
      "logps/rejected": -456.865234375,
      "loss": 0.6076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7454981207847595,
      "rewards/margins": 0.4889829754829407,
      "rewards/rejected": -1.2344809770584106,
      "step": 1627
    },
    {
      "epoch": 0.21303672200866933,
      "grad_norm": 4.041797161102295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4590802490711212,
      "logits/rejected": 0.31464165449142456,
      "logps/chosen": -705.507080078125,
      "logps/rejected": -700.0499267578125,
      "loss": 0.2287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3791106939315796,
      "rewards/margins": 1.6135905981063843,
      "rewards/rejected": -1.2344799041748047,
      "step": 1628
    },
    {
      "epoch": 0.2131675799460211,
      "grad_norm": 10.533961296081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024657994508743286,
      "logits/rejected": 0.1075294017791748,
      "logps/chosen": -370.7739562988281,
      "logps/rejected": -248.12875366210938,
      "loss": 0.9098,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3043195009231567,
      "rewards/margins": -0.2903346121311188,
      "rewards/rejected": -1.0139849185943604,
      "step": 1629
    },
    {
      "epoch": 0.21329843788337285,
      "grad_norm": 11.426027297973633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08694683015346527,
      "logits/rejected": 0.007182471454143524,
      "logps/chosen": -525.23974609375,
      "logps/rejected": -452.71630859375,
      "loss": 0.8403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3091894388198853,
      "rewards/margins": 0.3828660845756531,
      "rewards/rejected": -1.692055583000183,
      "step": 1630
    },
    {
      "epoch": 0.21342929582072462,
      "grad_norm": 4.474829196929932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08147133141756058,
      "logits/rejected": 0.08155303448438644,
      "logps/chosen": -417.1247863769531,
      "logps/rejected": -418.38958740234375,
      "loss": 0.3371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18479004502296448,
      "rewards/margins": 1.2004523277282715,
      "rewards/rejected": -1.3852423429489136,
      "step": 1631
    },
    {
      "epoch": 0.21356015375807638,
      "grad_norm": 9.556066513061523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37801241874694824,
      "logits/rejected": 0.20876803994178772,
      "logps/chosen": -450.6112060546875,
      "logps/rejected": -539.4384155273438,
      "loss": 0.6377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0395748615264893,
      "rewards/margins": 0.5736525654792786,
      "rewards/rejected": -1.613227367401123,
      "step": 1632
    },
    {
      "epoch": 0.21369101169542815,
      "grad_norm": 10.448709487915039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31627774238586426,
      "logits/rejected": 0.1004558578133583,
      "logps/chosen": -681.4701538085938,
      "logps/rejected": -498.37908935546875,
      "loss": 0.6027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7789748311042786,
      "rewards/margins": 0.4284585118293762,
      "rewards/rejected": -1.2074333429336548,
      "step": 1633
    },
    {
      "epoch": 0.2138218696327799,
      "grad_norm": 4.362568378448486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2404317557811737,
      "logits/rejected": 0.18075920641422272,
      "logps/chosen": -479.3970031738281,
      "logps/rejected": -452.2054748535156,
      "loss": 0.3043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.008731856942176819,
      "rewards/margins": 1.2182807922363281,
      "rewards/rejected": -1.2095489501953125,
      "step": 1634
    },
    {
      "epoch": 0.21395272757013167,
      "grad_norm": 6.215017318725586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3104928135871887,
      "logits/rejected": 0.2677726745605469,
      "logps/chosen": -546.5416870117188,
      "logps/rejected": -412.4031982421875,
      "loss": 0.4343,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20458507537841797,
      "rewards/margins": 0.9584537744522095,
      "rewards/rejected": -1.1630388498306274,
      "step": 1635
    },
    {
      "epoch": 0.21408358550748344,
      "grad_norm": 6.511715412139893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31164073944091797,
      "logits/rejected": 0.26881787180900574,
      "logps/chosen": -508.9486389160156,
      "logps/rejected": -562.5485229492188,
      "loss": 0.4895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5540740489959717,
      "rewards/margins": 0.7819083333015442,
      "rewards/rejected": -1.3359824419021606,
      "step": 1636
    },
    {
      "epoch": 0.2142144434448352,
      "grad_norm": 7.971854209899902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14111416041851044,
      "logits/rejected": 0.1463652104139328,
      "logps/chosen": -452.8062744140625,
      "logps/rejected": -423.91375732421875,
      "loss": 0.7291,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8157141208648682,
      "rewards/margins": 0.5033010244369507,
      "rewards/rejected": -1.3190151453018188,
      "step": 1637
    },
    {
      "epoch": 0.21434530138218696,
      "grad_norm": 7.575528144836426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03607616573572159,
      "logits/rejected": 0.022849878296256065,
      "logps/chosen": -532.6900024414062,
      "logps/rejected": -403.9815673828125,
      "loss": 0.5566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5131458044052124,
      "rewards/margins": 1.0346986055374146,
      "rewards/rejected": -1.5478445291519165,
      "step": 1638
    },
    {
      "epoch": 0.21447615931953873,
      "grad_norm": 5.916806221008301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6826627254486084,
      "logits/rejected": 0.46658051013946533,
      "logps/chosen": -347.4947814941406,
      "logps/rejected": -410.8005676269531,
      "loss": 0.5847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7305238842964172,
      "rewards/margins": 0.2751157879829407,
      "rewards/rejected": -1.005639672279358,
      "step": 1639
    },
    {
      "epoch": 0.2146070172568905,
      "grad_norm": 43.53545379638672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0017764531075954437,
      "logits/rejected": 0.10922863334417343,
      "logps/chosen": -487.99188232421875,
      "logps/rejected": -451.1552734375,
      "loss": 1.4931,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6166320443153381,
      "rewards/margins": -0.3283044397830963,
      "rewards/rejected": -0.28832757472991943,
      "step": 1640
    },
    {
      "epoch": 0.21473787519424226,
      "grad_norm": 5.652933120727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23042592406272888,
      "logits/rejected": 0.10204850882291794,
      "logps/chosen": -461.501953125,
      "logps/rejected": -455.005126953125,
      "loss": 0.5171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21855488419532776,
      "rewards/margins": 0.45820295810699463,
      "rewards/rejected": -0.6767578125,
      "step": 1641
    },
    {
      "epoch": 0.21486873313159402,
      "grad_norm": 3.917626142501831,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1827574372291565,
      "logits/rejected": 0.12007354199886322,
      "logps/chosen": -308.0165100097656,
      "logps/rejected": -333.0552062988281,
      "loss": 0.3836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07766211032867432,
      "rewards/margins": 0.8702009916305542,
      "rewards/rejected": -0.9478631019592285,
      "step": 1642
    },
    {
      "epoch": 0.21499959106894578,
      "grad_norm": 6.640174388885498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20007523894309998,
      "logits/rejected": 0.3113424777984619,
      "logps/chosen": -659.4553833007812,
      "logps/rejected": -518.0218505859375,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4054733216762543,
      "rewards/margins": 0.9647846221923828,
      "rewards/rejected": -0.5593112707138062,
      "step": 1643
    },
    {
      "epoch": 0.21513044900629755,
      "grad_norm": 6.010119438171387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2777329385280609,
      "logits/rejected": 0.20451602339744568,
      "logps/chosen": -487.10302734375,
      "logps/rejected": -422.9057312011719,
      "loss": 0.509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5639187097549438,
      "rewards/margins": 0.6468015313148499,
      "rewards/rejected": -1.2107203006744385,
      "step": 1644
    },
    {
      "epoch": 0.2152613069436493,
      "grad_norm": 4.755863189697266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13338249921798706,
      "logits/rejected": 0.03975139930844307,
      "logps/chosen": -779.9381713867188,
      "logps/rejected": -581.58544921875,
      "loss": 0.2507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6488776803016663,
      "rewards/margins": 1.5514366626739502,
      "rewards/rejected": -0.9025588631629944,
      "step": 1645
    },
    {
      "epoch": 0.21539216488100107,
      "grad_norm": 5.042920112609863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34074148535728455,
      "logits/rejected": 0.041599005460739136,
      "logps/chosen": -396.14678955078125,
      "logps/rejected": -504.9680480957031,
      "loss": 0.3408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3815496563911438,
      "rewards/margins": 1.4993020296096802,
      "rewards/rejected": -1.8808516263961792,
      "step": 1646
    },
    {
      "epoch": 0.21552302281835284,
      "grad_norm": 8.38774299621582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2962248623371124,
      "logits/rejected": 0.3069953918457031,
      "logps/chosen": -620.1808471679688,
      "logps/rejected": -462.3519592285156,
      "loss": 0.6786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26823365688323975,
      "rewards/margins": 0.3363143503665924,
      "rewards/rejected": -0.6045480966567993,
      "step": 1647
    },
    {
      "epoch": 0.21565388075570457,
      "grad_norm": 5.354034900665283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14337760210037231,
      "logits/rejected": 0.2733061909675598,
      "logps/chosen": -252.68679809570312,
      "logps/rejected": -234.44529724121094,
      "loss": 0.5615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5360078811645508,
      "rewards/margins": 0.44593220949172974,
      "rewards/rejected": -0.9819400906562805,
      "step": 1648
    },
    {
      "epoch": 0.21578473869305634,
      "grad_norm": 6.607044696807861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08592820912599564,
      "logits/rejected": 0.18559181690216064,
      "logps/chosen": -304.5953369140625,
      "logps/rejected": -453.8773193359375,
      "loss": 0.4667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8092058300971985,
      "rewards/margins": 1.167389988899231,
      "rewards/rejected": -1.9765957593917847,
      "step": 1649
    },
    {
      "epoch": 0.2159155966304081,
      "grad_norm": 6.067573547363281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13132363557815552,
      "logits/rejected": 0.12216441333293915,
      "logps/chosen": -500.90142822265625,
      "logps/rejected": -380.2140197753906,
      "loss": 0.623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2790553867816925,
      "rewards/margins": 0.2913793623447418,
      "rewards/rejected": -0.5704348087310791,
      "step": 1650
    },
    {
      "epoch": 0.21604645456775987,
      "grad_norm": 5.935676574707031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6402627229690552,
      "logits/rejected": 0.5800061225891113,
      "logps/chosen": -493.454345703125,
      "logps/rejected": -435.8642272949219,
      "loss": 0.5848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3758412003517151,
      "rewards/margins": 0.4612613916397095,
      "rewards/rejected": -0.837102472782135,
      "step": 1651
    },
    {
      "epoch": 0.21617731250511163,
      "grad_norm": 8.047917366027832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12163212895393372,
      "logits/rejected": 0.45859494805336,
      "logps/chosen": -399.07183837890625,
      "logps/rejected": -411.23187255859375,
      "loss": 0.5781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34460896253585815,
      "rewards/margins": 0.6784237623214722,
      "rewards/rejected": -1.023032784461975,
      "step": 1652
    },
    {
      "epoch": 0.2163081704424634,
      "grad_norm": 7.9587016105651855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22173267602920532,
      "logits/rejected": 0.3026830852031708,
      "logps/chosen": -399.0670471191406,
      "logps/rejected": -389.7202453613281,
      "loss": 0.6108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5713676810264587,
      "rewards/margins": 0.5942567586898804,
      "rewards/rejected": -1.1656243801116943,
      "step": 1653
    },
    {
      "epoch": 0.21643902837981516,
      "grad_norm": 3.858980655670166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2056870460510254,
      "logits/rejected": 0.12869787216186523,
      "logps/chosen": -553.537353515625,
      "logps/rejected": -439.04461669921875,
      "loss": 0.2515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03760489076375961,
      "rewards/margins": 1.4793410301208496,
      "rewards/rejected": -1.441736102104187,
      "step": 1654
    },
    {
      "epoch": 0.21656988631716692,
      "grad_norm": 7.26546049118042,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0538022518157959,
      "logits/rejected": 0.2735772132873535,
      "logps/chosen": -541.739501953125,
      "logps/rejected": -502.4229431152344,
      "loss": 0.5112,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1637079268693924,
      "rewards/margins": 0.7566641569137573,
      "rewards/rejected": -0.5929561853408813,
      "step": 1655
    },
    {
      "epoch": 0.21670074425451868,
      "grad_norm": 3.7568323612213135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3671375513076782,
      "logits/rejected": 0.14162468910217285,
      "logps/chosen": -467.11883544921875,
      "logps/rejected": -397.3216247558594,
      "loss": 0.3618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09517231583595276,
      "rewards/margins": 1.0346548557281494,
      "rewards/rejected": -0.939482569694519,
      "step": 1656
    },
    {
      "epoch": 0.21683160219187045,
      "grad_norm": 10.063644409179688,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21553678810596466,
      "logits/rejected": 0.25181999802589417,
      "logps/chosen": -535.9171752929688,
      "logps/rejected": -423.6600341796875,
      "loss": 0.8063,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6809993386268616,
      "rewards/margins": 0.31726667284965515,
      "rewards/rejected": -0.9982660412788391,
      "step": 1657
    },
    {
      "epoch": 0.2169624601292222,
      "grad_norm": 10.845205307006836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06562883406877518,
      "logits/rejected": 0.16944685578346252,
      "logps/chosen": -929.9224853515625,
      "logps/rejected": -589.514404296875,
      "loss": 0.8074,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6261085867881775,
      "rewards/margins": 0.3601623475551605,
      "rewards/rejected": -0.9862710237503052,
      "step": 1658
    },
    {
      "epoch": 0.21709331806657398,
      "grad_norm": 6.399144172668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3287394344806671,
      "logits/rejected": 0.14150312542915344,
      "logps/chosen": -456.0863037109375,
      "logps/rejected": -412.739013671875,
      "loss": 0.5547,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2900732159614563,
      "rewards/margins": 0.5525670647621155,
      "rewards/rejected": -0.8426402807235718,
      "step": 1659
    },
    {
      "epoch": 0.21722417600392574,
      "grad_norm": 5.278007507324219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11323430389165878,
      "logits/rejected": -0.2694090008735657,
      "logps/chosen": -311.91741943359375,
      "logps/rejected": -446.32025146484375,
      "loss": 0.4653,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4011414051055908,
      "rewards/margins": 0.8031086921691895,
      "rewards/rejected": -1.2042499780654907,
      "step": 1660
    },
    {
      "epoch": 0.2173550339412775,
      "grad_norm": 7.577236175537109,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12983743846416473,
      "logits/rejected": 0.19535444676876068,
      "logps/chosen": -518.6820678710938,
      "logps/rejected": -509.4823913574219,
      "loss": 0.5975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18822498619556427,
      "rewards/margins": 0.6936967372894287,
      "rewards/rejected": -0.8819217681884766,
      "step": 1661
    },
    {
      "epoch": 0.21748589187862927,
      "grad_norm": 4.015509128570557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39229997992515564,
      "logits/rejected": 0.30366450548171997,
      "logps/chosen": -302.7938537597656,
      "logps/rejected": -447.4166564941406,
      "loss": 0.3139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3494970202445984,
      "rewards/margins": 1.4781917333602905,
      "rewards/rejected": -1.8276888132095337,
      "step": 1662
    },
    {
      "epoch": 0.21761674981598103,
      "grad_norm": 6.118520736694336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16217568516731262,
      "logits/rejected": -0.05253474414348602,
      "logps/chosen": -329.05670166015625,
      "logps/rejected": -366.0255126953125,
      "loss": 0.6472,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47107601165771484,
      "rewards/margins": 0.1531130075454712,
      "rewards/rejected": -0.624189019203186,
      "step": 1663
    },
    {
      "epoch": 0.2177476077533328,
      "grad_norm": 6.281745910644531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5340157151222229,
      "logits/rejected": 0.5557842254638672,
      "logps/chosen": -374.31390380859375,
      "logps/rejected": -406.13623046875,
      "loss": 0.6089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15413837134838104,
      "rewards/margins": 0.5393557548522949,
      "rewards/rejected": -0.6934942007064819,
      "step": 1664
    },
    {
      "epoch": 0.21787846569068456,
      "grad_norm": 7.825385093688965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09687265753746033,
      "logits/rejected": 0.11386194825172424,
      "logps/chosen": -560.6324462890625,
      "logps/rejected": -526.26171875,
      "loss": 0.4453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24219150841236115,
      "rewards/margins": 0.926503598690033,
      "rewards/rejected": -1.1686949729919434,
      "step": 1665
    },
    {
      "epoch": 0.21800932362803632,
      "grad_norm": 14.343656539916992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2388559877872467,
      "logits/rejected": 0.2892872989177704,
      "logps/chosen": -559.2474365234375,
      "logps/rejected": -375.6137390136719,
      "loss": 1.2385,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8643258810043335,
      "rewards/margins": -0.5475195050239563,
      "rewards/rejected": -0.3168064057826996,
      "step": 1666
    },
    {
      "epoch": 0.21814018156538809,
      "grad_norm": 11.58520793914795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16171978414058685,
      "logits/rejected": 0.29217424988746643,
      "logps/chosen": -473.9385986328125,
      "logps/rejected": -659.4904174804688,
      "loss": 1.0391,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.49363023042678833,
      "rewards/margins": -0.03064781427383423,
      "rewards/rejected": -0.4629823565483093,
      "step": 1667
    },
    {
      "epoch": 0.21827103950273985,
      "grad_norm": 5.7326579093933105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3069572150707245,
      "logits/rejected": 0.3089054524898529,
      "logps/chosen": -457.9103698730469,
      "logps/rejected": -439.69512939453125,
      "loss": 0.4813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04681815579533577,
      "rewards/margins": 1.206281065940857,
      "rewards/rejected": -1.1594629287719727,
      "step": 1668
    },
    {
      "epoch": 0.2184018974400916,
      "grad_norm": 5.780554294586182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30003857612609863,
      "logits/rejected": 0.12394239008426666,
      "logps/chosen": -391.6451721191406,
      "logps/rejected": -311.14703369140625,
      "loss": 0.3862,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.042255982756614685,
      "rewards/margins": 1.3734489679336548,
      "rewards/rejected": -1.3311930894851685,
      "step": 1669
    },
    {
      "epoch": 0.21853275537744335,
      "grad_norm": 4.2033233642578125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4038130044937134,
      "logits/rejected": 0.48628243803977966,
      "logps/chosen": -374.34515380859375,
      "logps/rejected": -298.187744140625,
      "loss": 0.4223,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11807535588741302,
      "rewards/margins": 0.8281496167182922,
      "rewards/rejected": -0.9462249279022217,
      "step": 1670
    },
    {
      "epoch": 0.2186636133147951,
      "grad_norm": 6.333818435668945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1600198745727539,
      "logits/rejected": 0.3019341826438904,
      "logps/chosen": -383.10540771484375,
      "logps/rejected": -394.4757080078125,
      "loss": 0.5178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07701723277568817,
      "rewards/margins": 0.6423267722129822,
      "rewards/rejected": -0.5653095245361328,
      "step": 1671
    },
    {
      "epoch": 0.21879447125214688,
      "grad_norm": 5.556875705718994,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0013714693486690521,
      "logits/rejected": -0.05046062916517258,
      "logps/chosen": -458.3090515136719,
      "logps/rejected": -456.5562438964844,
      "loss": 0.4964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1507532298564911,
      "rewards/margins": 1.3545845746994019,
      "rewards/rejected": -1.2038313150405884,
      "step": 1672
    },
    {
      "epoch": 0.21892532918949864,
      "grad_norm": 6.732783794403076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0013893768191337585,
      "logits/rejected": 0.09470491111278534,
      "logps/chosen": -360.8177795410156,
      "logps/rejected": -321.0475158691406,
      "loss": 0.5929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4468717873096466,
      "rewards/margins": 0.37159329652786255,
      "rewards/rejected": -0.8184650540351868,
      "step": 1673
    },
    {
      "epoch": 0.2190561871268504,
      "grad_norm": 5.410154819488525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31724199652671814,
      "logits/rejected": 0.2106943428516388,
      "logps/chosen": -372.89654541015625,
      "logps/rejected": -341.66461181640625,
      "loss": 0.5192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5547583699226379,
      "rewards/margins": 0.44067496061325073,
      "rewards/rejected": -0.9954333305358887,
      "step": 1674
    },
    {
      "epoch": 0.21918704506420217,
      "grad_norm": 7.269657135009766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.174192875623703,
      "logits/rejected": 0.06387554854154587,
      "logps/chosen": -380.1673583984375,
      "logps/rejected": -348.14111328125,
      "loss": 0.7149,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05277883633971214,
      "rewards/margins": 0.9673674702644348,
      "rewards/rejected": -1.0201462507247925,
      "step": 1675
    },
    {
      "epoch": 0.21931790300155393,
      "grad_norm": 8.26873779296875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20843815803527832,
      "logits/rejected": 0.1701023280620575,
      "logps/chosen": -582.790771484375,
      "logps/rejected": -538.5366821289062,
      "loss": 0.6599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.27245691418647766,
      "rewards/margins": 0.401398628950119,
      "rewards/rejected": -0.12894174456596375,
      "step": 1676
    },
    {
      "epoch": 0.2194487609389057,
      "grad_norm": 7.869601726531982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11414848268032074,
      "logits/rejected": 0.07903007417917252,
      "logps/chosen": -375.681884765625,
      "logps/rejected": -387.0340270996094,
      "loss": 0.7833,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.45366421341896057,
      "rewards/margins": 0.022804051637649536,
      "rewards/rejected": -0.4764682948589325,
      "step": 1677
    },
    {
      "epoch": 0.21957961887625746,
      "grad_norm": 9.407851219177246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03338949382305145,
      "logits/rejected": 0.08458542078733444,
      "logps/chosen": -409.4870300292969,
      "logps/rejected": -404.961669921875,
      "loss": 0.7453,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6429897546768188,
      "rewards/margins": 0.5915800929069519,
      "rewards/rejected": -1.2345696687698364,
      "step": 1678
    },
    {
      "epoch": 0.21971047681360922,
      "grad_norm": 6.883566856384277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16100522875785828,
      "logits/rejected": 0.13060598075389862,
      "logps/chosen": -480.6205139160156,
      "logps/rejected": -506.13104248046875,
      "loss": 0.5783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13210278749465942,
      "rewards/margins": 0.5194457769393921,
      "rewards/rejected": -0.6515486240386963,
      "step": 1679
    },
    {
      "epoch": 0.219841334750961,
      "grad_norm": 11.384342193603516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25647780299186707,
      "logits/rejected": 0.3068268299102783,
      "logps/chosen": -545.342529296875,
      "logps/rejected": -349.5874328613281,
      "loss": 0.6816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8701772093772888,
      "rewards/margins": 0.8688867092132568,
      "rewards/rejected": -1.73906409740448,
      "step": 1680
    },
    {
      "epoch": 0.21997219268831275,
      "grad_norm": 5.470468521118164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45843270421028137,
      "logits/rejected": 0.5040415525436401,
      "logps/chosen": -312.19793701171875,
      "logps/rejected": -388.3525695800781,
      "loss": 0.4602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24105492234230042,
      "rewards/margins": 1.052730679512024,
      "rewards/rejected": -1.2937856912612915,
      "step": 1681
    },
    {
      "epoch": 0.22010305062566451,
      "grad_norm": 6.223266124725342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10922296345233917,
      "logits/rejected": 0.22114448249340057,
      "logps/chosen": -395.7216796875,
      "logps/rejected": -259.80938720703125,
      "loss": 0.714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21059340238571167,
      "rewards/margins": 0.38261914253234863,
      "rewards/rejected": -0.5932124853134155,
      "step": 1682
    },
    {
      "epoch": 0.22023390856301628,
      "grad_norm": 10.214818000793457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5917356610298157,
      "logits/rejected": 0.3311323821544647,
      "logps/chosen": -528.0548095703125,
      "logps/rejected": -493.677490234375,
      "loss": 0.868,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.32125914096832275,
      "rewards/margins": -0.07823580503463745,
      "rewards/rejected": -0.24302329123020172,
      "step": 1683
    },
    {
      "epoch": 0.22036476650036804,
      "grad_norm": 7.863504409790039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0828767865896225,
      "logits/rejected": -0.07871311902999878,
      "logps/chosen": -364.90264892578125,
      "logps/rejected": -424.343994140625,
      "loss": 0.5981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5174646377563477,
      "rewards/margins": 0.654949426651001,
      "rewards/rejected": -1.172413945198059,
      "step": 1684
    },
    {
      "epoch": 0.2204956244377198,
      "grad_norm": 4.959905624389648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.49040836095809937,
      "logits/rejected": 0.26119866967201233,
      "logps/chosen": -444.3937683105469,
      "logps/rejected": -458.4254150390625,
      "loss": 0.375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29972362518310547,
      "rewards/margins": 0.945780873298645,
      "rewards/rejected": -0.6460573673248291,
      "step": 1685
    },
    {
      "epoch": 0.22062648237507157,
      "grad_norm": 6.352564334869385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11088956892490387,
      "logits/rejected": 0.22136163711547852,
      "logps/chosen": -516.9503173828125,
      "logps/rejected": -596.5020141601562,
      "loss": 0.5023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4667154550552368,
      "rewards/margins": 0.9270684719085693,
      "rewards/rejected": -0.46035313606262207,
      "step": 1686
    },
    {
      "epoch": 0.22075734031242333,
      "grad_norm": 7.703583717346191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2979975640773773,
      "logits/rejected": 0.35999906063079834,
      "logps/chosen": -465.1432800292969,
      "logps/rejected": -625.4176025390625,
      "loss": 0.6128,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15179137885570526,
      "rewards/margins": 0.5596696734428406,
      "rewards/rejected": -0.711461067199707,
      "step": 1687
    },
    {
      "epoch": 0.2208881982497751,
      "grad_norm": 7.710210800170898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11882266402244568,
      "logits/rejected": 0.28600746393203735,
      "logps/chosen": -356.365234375,
      "logps/rejected": -490.2442321777344,
      "loss": 0.6727,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8439223766326904,
      "rewards/margins": 0.5217065811157227,
      "rewards/rejected": -1.365628957748413,
      "step": 1688
    },
    {
      "epoch": 0.22101905618712686,
      "grad_norm": 7.9581780433654785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03854667395353317,
      "logits/rejected": 0.03410838544368744,
      "logps/chosen": -703.9486083984375,
      "logps/rejected": -636.64990234375,
      "loss": 0.6465,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11371193826198578,
      "rewards/margins": 0.4089735150337219,
      "rewards/rejected": -0.5226854085922241,
      "step": 1689
    },
    {
      "epoch": 0.22114991412447862,
      "grad_norm": 5.797741413116455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2556196451187134,
      "logits/rejected": 0.19386696815490723,
      "logps/chosen": -627.1239013671875,
      "logps/rejected": -555.7346801757812,
      "loss": 0.5073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.367536723613739,
      "rewards/margins": 0.5333036780357361,
      "rewards/rejected": -0.1657669097185135,
      "step": 1690
    },
    {
      "epoch": 0.2212807720618304,
      "grad_norm": 8.559316635131836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5502846837043762,
      "logits/rejected": 0.29403969645500183,
      "logps/chosen": -370.2813415527344,
      "logps/rejected": -413.4990234375,
      "loss": 0.7644,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0628015995025635,
      "rewards/margins": 0.5348228812217712,
      "rewards/rejected": -1.59762442111969,
      "step": 1691
    },
    {
      "epoch": 0.22141162999918212,
      "grad_norm": 6.991509437561035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40561041235923767,
      "logits/rejected": 0.34775325655937195,
      "logps/chosen": -382.5825500488281,
      "logps/rejected": -408.33367919921875,
      "loss": 0.5981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.023652836680412292,
      "rewards/margins": 0.4453312158584595,
      "rewards/rejected": -0.46898406744003296,
      "step": 1692
    },
    {
      "epoch": 0.2215424879365339,
      "grad_norm": 6.659853458404541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1353517770767212,
      "logits/rejected": 0.04153784364461899,
      "logps/chosen": -305.78436279296875,
      "logps/rejected": -407.0186462402344,
      "loss": 0.6295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19191224873065948,
      "rewards/margins": 0.3234882354736328,
      "rewards/rejected": -0.5154004096984863,
      "step": 1693
    },
    {
      "epoch": 0.22167334587388565,
      "grad_norm": 5.074869155883789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42705461382865906,
      "logits/rejected": 0.15526485443115234,
      "logps/chosen": -382.1955871582031,
      "logps/rejected": -269.853271484375,
      "loss": 0.4496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7254676818847656,
      "rewards/margins": 0.6124802827835083,
      "rewards/rejected": -1.337947964668274,
      "step": 1694
    },
    {
      "epoch": 0.22180420381123742,
      "grad_norm": 5.552333354949951,
      "learning_rate": 5e-06,
      "logits/chosen": 0.241560161113739,
      "logits/rejected": 0.2876150608062744,
      "logps/chosen": -371.0569152832031,
      "logps/rejected": -411.697509765625,
      "loss": 0.4441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.045707885175943375,
      "rewards/margins": 0.7191892862319946,
      "rewards/rejected": -0.7648971676826477,
      "step": 1695
    },
    {
      "epoch": 0.22193506174858918,
      "grad_norm": 5.088972091674805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2895563840866089,
      "logits/rejected": 0.056412793695926666,
      "logps/chosen": -413.2218322753906,
      "logps/rejected": -445.68951416015625,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23275279998779297,
      "rewards/margins": 0.9658027291297913,
      "rewards/rejected": -1.198555588722229,
      "step": 1696
    },
    {
      "epoch": 0.22206591968594094,
      "grad_norm": 7.298109531402588,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12005575746297836,
      "logits/rejected": 0.14220522344112396,
      "logps/chosen": -444.80645751953125,
      "logps/rejected": -385.5367126464844,
      "loss": 0.5253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5622532367706299,
      "rewards/margins": 0.4373774826526642,
      "rewards/rejected": -0.9996307492256165,
      "step": 1697
    },
    {
      "epoch": 0.2221967776232927,
      "grad_norm": 7.954137325286865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07283832132816315,
      "logits/rejected": 0.14285919070243835,
      "logps/chosen": -398.9107971191406,
      "logps/rejected": -405.26312255859375,
      "loss": 0.8255,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0898258239030838,
      "rewards/margins": -0.11975383758544922,
      "rewards/rejected": 0.029928013682365417,
      "step": 1698
    },
    {
      "epoch": 0.22232763556064447,
      "grad_norm": 7.303761959075928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3152901530265808,
      "logits/rejected": 0.29463493824005127,
      "logps/chosen": -484.95294189453125,
      "logps/rejected": -585.3377685546875,
      "loss": 0.5917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.268968790769577,
      "rewards/margins": 1.162014365196228,
      "rewards/rejected": -0.8930456042289734,
      "step": 1699
    },
    {
      "epoch": 0.22245849349799623,
      "grad_norm": 6.311108589172363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1943436563014984,
      "logits/rejected": 0.09421301633119583,
      "logps/chosen": -482.1326904296875,
      "logps/rejected": -453.69793701171875,
      "loss": 0.416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03613719344139099,
      "rewards/margins": 1.0949667692184448,
      "rewards/rejected": -1.0588295459747314,
      "step": 1700
    },
    {
      "epoch": 0.222589351435348,
      "grad_norm": 7.272604942321777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24435293674468994,
      "logits/rejected": 0.11605319380760193,
      "logps/chosen": -442.4707336425781,
      "logps/rejected": -444.34088134765625,
      "loss": 0.6145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2592460513114929,
      "rewards/margins": 0.31238654255867004,
      "rewards/rejected": -0.5716325044631958,
      "step": 1701
    },
    {
      "epoch": 0.22272020937269976,
      "grad_norm": 5.760636806488037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.234989196062088,
      "logits/rejected": 0.13241010904312134,
      "logps/chosen": -468.4344482421875,
      "logps/rejected": -365.6312561035156,
      "loss": 0.6281,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18850985169410706,
      "rewards/margins": 0.392386257648468,
      "rewards/rejected": -0.5808961391448975,
      "step": 1702
    },
    {
      "epoch": 0.22285106731005153,
      "grad_norm": 6.4932637214660645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22511912882328033,
      "logits/rejected": 0.28684183955192566,
      "logps/chosen": -534.6930541992188,
      "logps/rejected": -544.098876953125,
      "loss": 0.5259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23223364353179932,
      "rewards/margins": 0.5066017508506775,
      "rewards/rejected": -0.2743681073188782,
      "step": 1703
    },
    {
      "epoch": 0.2229819252474033,
      "grad_norm": 5.882894515991211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07461148500442505,
      "logits/rejected": -0.01933959126472473,
      "logps/chosen": -381.3753967285156,
      "logps/rejected": -423.071533203125,
      "loss": 0.5077,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11844003200531006,
      "rewards/margins": 0.7041747570037842,
      "rewards/rejected": -0.8226147890090942,
      "step": 1704
    },
    {
      "epoch": 0.22311278318475505,
      "grad_norm": 5.786091327667236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5225993394851685,
      "logits/rejected": 0.3038467466831207,
      "logps/chosen": -486.0830078125,
      "logps/rejected": -370.328125,
      "loss": 0.4584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0893852487206459,
      "rewards/margins": 0.9233183264732361,
      "rewards/rejected": -0.8339331150054932,
      "step": 1705
    },
    {
      "epoch": 0.22324364112210682,
      "grad_norm": 6.848543167114258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20658046007156372,
      "logits/rejected": -0.14299960434436798,
      "logps/chosen": -476.007568359375,
      "logps/rejected": -462.5966796875,
      "loss": 0.4557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3763856589794159,
      "rewards/margins": 0.9052846431732178,
      "rewards/rejected": -1.281670331954956,
      "step": 1706
    },
    {
      "epoch": 0.22337449905945858,
      "grad_norm": 3.9845762252807617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19557932019233704,
      "logits/rejected": 0.14745503664016724,
      "logps/chosen": -454.5065612792969,
      "logps/rejected": -362.0915222167969,
      "loss": 0.4871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13674888014793396,
      "rewards/margins": 0.7494045495986938,
      "rewards/rejected": -0.8861533999443054,
      "step": 1707
    },
    {
      "epoch": 0.22350535699681034,
      "grad_norm": 7.435206890106201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33844202756881714,
      "logits/rejected": 0.24341243505477905,
      "logps/chosen": -389.92822265625,
      "logps/rejected": -417.07757568359375,
      "loss": 0.671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4370868504047394,
      "rewards/margins": 0.2700643837451935,
      "rewards/rejected": -0.7071511745452881,
      "step": 1708
    },
    {
      "epoch": 0.2236362149341621,
      "grad_norm": 8.333525657653809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15121853351593018,
      "logits/rejected": 0.2828275263309479,
      "logps/chosen": -486.9812316894531,
      "logps/rejected": -563.5028076171875,
      "loss": 0.6295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20219960808753967,
      "rewards/margins": 0.40844228863716125,
      "rewards/rejected": -0.6106418371200562,
      "step": 1709
    },
    {
      "epoch": 0.22376707287151387,
      "grad_norm": 7.5650634765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21519190073013306,
      "logits/rejected": 0.11419431865215302,
      "logps/chosen": -623.1126098632812,
      "logps/rejected": -688.2308349609375,
      "loss": 0.356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0705690085887909,
      "rewards/margins": 1.140525460243225,
      "rewards/rejected": -1.0699565410614014,
      "step": 1710
    },
    {
      "epoch": 0.22389793080886564,
      "grad_norm": 5.278459548950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3704374134540558,
      "logits/rejected": 0.33127328753471375,
      "logps/chosen": -555.8128051757812,
      "logps/rejected": -491.72308349609375,
      "loss": 0.4319,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3414871394634247,
      "rewards/margins": 0.6460617184638977,
      "rewards/rejected": -0.3045746088027954,
      "step": 1711
    },
    {
      "epoch": 0.2240287887462174,
      "grad_norm": 6.012864112854004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.418867290019989,
      "logits/rejected": 0.3061768710613251,
      "logps/chosen": -493.12042236328125,
      "logps/rejected": -592.183837890625,
      "loss": 0.4172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5897789001464844,
      "rewards/margins": 1.518479824066162,
      "rewards/rejected": -0.9287008047103882,
      "step": 1712
    },
    {
      "epoch": 0.22415964668356916,
      "grad_norm": 6.478544235229492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005857698619365692,
      "logits/rejected": 0.15853938460350037,
      "logps/chosen": -310.3514404296875,
      "logps/rejected": -349.01080322265625,
      "loss": 0.5816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3906286358833313,
      "rewards/margins": 0.5146322250366211,
      "rewards/rejected": -0.9052608609199524,
      "step": 1713
    },
    {
      "epoch": 0.2242905046209209,
      "grad_norm": 6.809568405151367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2974485456943512,
      "logits/rejected": 0.4959803819656372,
      "logps/chosen": -436.9240417480469,
      "logps/rejected": -471.73748779296875,
      "loss": 0.6879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06655711680650711,
      "rewards/margins": 0.30643558502197266,
      "rewards/rejected": -0.23987846076488495,
      "step": 1714
    },
    {
      "epoch": 0.22442136255827266,
      "grad_norm": 5.948470592498779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25978028774261475,
      "logits/rejected": 0.16617320477962494,
      "logps/chosen": -428.8611145019531,
      "logps/rejected": -795.9888916015625,
      "loss": 0.5061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13684654235839844,
      "rewards/margins": 1.0847057104110718,
      "rewards/rejected": -1.2215522527694702,
      "step": 1715
    },
    {
      "epoch": 0.22455222049562443,
      "grad_norm": 5.176716327667236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23376822471618652,
      "logits/rejected": 0.148521289229393,
      "logps/chosen": -704.325927734375,
      "logps/rejected": -483.3296203613281,
      "loss": 0.4684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22551459074020386,
      "rewards/margins": 0.93939608335495,
      "rewards/rejected": -0.7138814926147461,
      "step": 1716
    },
    {
      "epoch": 0.2246830784329762,
      "grad_norm": 5.653931140899658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36736437678337097,
      "logits/rejected": 0.45974355936050415,
      "logps/chosen": -426.0749206542969,
      "logps/rejected": -546.70458984375,
      "loss": 0.4293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5036695003509521,
      "rewards/margins": 1.1837337017059326,
      "rewards/rejected": -0.6800640821456909,
      "step": 1717
    },
    {
      "epoch": 0.22481393637032795,
      "grad_norm": 11.999320030212402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5424268841743469,
      "logits/rejected": 0.4763328433036804,
      "logps/chosen": -374.5647888183594,
      "logps/rejected": -651.6453247070312,
      "loss": 0.6508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.016242220997810364,
      "rewards/margins": 0.3002157211303711,
      "rewards/rejected": -0.31645795702934265,
      "step": 1718
    },
    {
      "epoch": 0.22494479430767972,
      "grad_norm": 5.229811191558838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2818775773048401,
      "logits/rejected": 0.06656938791275024,
      "logps/chosen": -395.10174560546875,
      "logps/rejected": -484.6288757324219,
      "loss": 0.55,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3529024124145508,
      "rewards/margins": 0.6286472082138062,
      "rewards/rejected": -0.9815496802330017,
      "step": 1719
    },
    {
      "epoch": 0.22507565224503148,
      "grad_norm": 5.927804946899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16558530926704407,
      "logits/rejected": -0.021242689341306686,
      "logps/chosen": -477.33624267578125,
      "logps/rejected": -515.8212890625,
      "loss": 0.5293,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4481344223022461,
      "rewards/margins": 0.5616590976715088,
      "rewards/rejected": -0.11352463066577911,
      "step": 1720
    },
    {
      "epoch": 0.22520651018238325,
      "grad_norm": 6.072899341583252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2539941072463989,
      "logits/rejected": -0.045734670013189316,
      "logps/chosen": -541.9059448242188,
      "logps/rejected": -455.11083984375,
      "loss": 0.468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1362888216972351,
      "rewards/margins": 0.7862598896026611,
      "rewards/rejected": -0.6499710083007812,
      "step": 1721
    },
    {
      "epoch": 0.225337368119735,
      "grad_norm": 6.8328166007995605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1159520223736763,
      "logits/rejected": 0.13392400741577148,
      "logps/chosen": -643.8640747070312,
      "logps/rejected": -589.744873046875,
      "loss": 0.5824,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.252547025680542,
      "rewards/margins": 0.5532799363136292,
      "rewards/rejected": -0.3007327914237976,
      "step": 1722
    },
    {
      "epoch": 0.22546822605708677,
      "grad_norm": 4.64708948135376,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3888375759124756,
      "logits/rejected": 0.43962153792381287,
      "logps/chosen": -518.8128051757812,
      "logps/rejected": -579.2567138671875,
      "loss": 0.3839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11708357185125351,
      "rewards/margins": 1.1204801797866821,
      "rewards/rejected": -1.0033965110778809,
      "step": 1723
    },
    {
      "epoch": 0.22559908399443854,
      "grad_norm": 4.677742004394531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024098645895719528,
      "logits/rejected": 0.23090516030788422,
      "logps/chosen": -359.62042236328125,
      "logps/rejected": -312.93035888671875,
      "loss": 0.4982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.029261790215969086,
      "rewards/margins": 0.7666976451873779,
      "rewards/rejected": -0.79595947265625,
      "step": 1724
    },
    {
      "epoch": 0.2257299419317903,
      "grad_norm": 4.029548168182373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37253424525260925,
      "logits/rejected": 0.16374269127845764,
      "logps/chosen": -422.2431640625,
      "logps/rejected": -444.56329345703125,
      "loss": 0.4017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4153755307197571,
      "rewards/margins": 1.2444212436676025,
      "rewards/rejected": -0.8290457725524902,
      "step": 1725
    },
    {
      "epoch": 0.22586079986914206,
      "grad_norm": 6.710729598999023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1936747431755066,
      "logits/rejected": 0.17746375501155853,
      "logps/chosen": -521.061767578125,
      "logps/rejected": -350.0470886230469,
      "loss": 0.4482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4560268521308899,
      "rewards/margins": 0.9063667058944702,
      "rewards/rejected": -1.3623934984207153,
      "step": 1726
    },
    {
      "epoch": 0.22599165780649383,
      "grad_norm": 6.597015380859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4400686025619507,
      "logits/rejected": 0.343932569026947,
      "logps/chosen": -534.8429565429688,
      "logps/rejected": -476.9243469238281,
      "loss": 0.438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08581450581550598,
      "rewards/margins": 1.0276638269424438,
      "rewards/rejected": -1.1134783029556274,
      "step": 1727
    },
    {
      "epoch": 0.2261225157438456,
      "grad_norm": 8.330093383789062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10024682432413101,
      "logits/rejected": 0.10245610773563385,
      "logps/chosen": -720.8172607421875,
      "logps/rejected": -581.2401123046875,
      "loss": 0.8436,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.5478153228759766,
      "rewards/margins": 0.3710654675960541,
      "rewards/rejected": 0.1767498105764389,
      "step": 1728
    },
    {
      "epoch": 0.22625337368119736,
      "grad_norm": 7.773916244506836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043719835579395294,
      "logits/rejected": 0.09573385119438171,
      "logps/chosen": -391.3524169921875,
      "logps/rejected": -473.59234619140625,
      "loss": 0.7786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10099545121192932,
      "rewards/margins": 0.2348221242427826,
      "rewards/rejected": -0.13382665812969208,
      "step": 1729
    },
    {
      "epoch": 0.22638423161854912,
      "grad_norm": 9.30239486694336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2915472388267517,
      "logits/rejected": 0.17699021100997925,
      "logps/chosen": -536.7254638671875,
      "logps/rejected": -391.28619384765625,
      "loss": 0.6849,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28551292419433594,
      "rewards/margins": 0.5264248847961426,
      "rewards/rejected": -0.811937689781189,
      "step": 1730
    },
    {
      "epoch": 0.22651508955590088,
      "grad_norm": 6.232593536376953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4264772832393646,
      "logits/rejected": 0.40230298042297363,
      "logps/chosen": -569.7313842773438,
      "logps/rejected": -457.0898742675781,
      "loss": 0.5575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6955736875534058,
      "rewards/margins": 0.5997041463851929,
      "rewards/rejected": 0.09586947411298752,
      "step": 1731
    },
    {
      "epoch": 0.22664594749325265,
      "grad_norm": 7.627089977264404,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05821319669485092,
      "logits/rejected": 0.23372787237167358,
      "logps/chosen": -431.7962951660156,
      "logps/rejected": -425.4089050292969,
      "loss": 0.6772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08353214710950851,
      "rewards/margins": 0.14835360646247864,
      "rewards/rejected": -0.23188574612140656,
      "step": 1732
    },
    {
      "epoch": 0.2267768054306044,
      "grad_norm": 5.6016364097595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0614248588681221,
      "logits/rejected": 0.04409002512693405,
      "logps/chosen": -272.15887451171875,
      "logps/rejected": -321.02716064453125,
      "loss": 0.4857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3177158534526825,
      "rewards/margins": 0.6128857135772705,
      "rewards/rejected": -0.2951698303222656,
      "step": 1733
    },
    {
      "epoch": 0.22690766336795617,
      "grad_norm": 6.689657688140869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12332318723201752,
      "logits/rejected": 0.12946484982967377,
      "logps/chosen": -331.086669921875,
      "logps/rejected": -386.2695617675781,
      "loss": 0.6154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29305019974708557,
      "rewards/margins": 0.42315083742141724,
      "rewards/rejected": -0.13010065257549286,
      "step": 1734
    },
    {
      "epoch": 0.22703852130530794,
      "grad_norm": 5.911105155944824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1416422724723816,
      "logits/rejected": 0.0034214891493320465,
      "logps/chosen": -568.0919189453125,
      "logps/rejected": -409.5581359863281,
      "loss": 0.4832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.028060350567102432,
      "rewards/margins": 1.0717151165008545,
      "rewards/rejected": -1.0436549186706543,
      "step": 1735
    },
    {
      "epoch": 0.22716937924265967,
      "grad_norm": 8.6429443359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38625168800354004,
      "logits/rejected": 0.2547076344490051,
      "logps/chosen": -517.7623291015625,
      "logps/rejected": -446.71026611328125,
      "loss": 0.5654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15071049332618713,
      "rewards/margins": 0.6660838723182678,
      "rewards/rejected": -0.5153734087944031,
      "step": 1736
    },
    {
      "epoch": 0.22730023718001144,
      "grad_norm": 3.7929561138153076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26669842004776,
      "logits/rejected": 0.24019025266170502,
      "logps/chosen": -334.0394287109375,
      "logps/rejected": -543.6724853515625,
      "loss": 0.2988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48773959279060364,
      "rewards/margins": 1.2362216711044312,
      "rewards/rejected": -0.7484821677207947,
      "step": 1737
    },
    {
      "epoch": 0.2274310951173632,
      "grad_norm": 6.383087635040283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2019968330860138,
      "logits/rejected": 0.011424943804740906,
      "logps/chosen": -355.0568542480469,
      "logps/rejected": -461.53216552734375,
      "loss": 0.5166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7352316975593567,
      "rewards/margins": 0.8967244029045105,
      "rewards/rejected": -1.6319561004638672,
      "step": 1738
    },
    {
      "epoch": 0.22756195305471497,
      "grad_norm": 7.477145671844482,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4733836054801941,
      "logits/rejected": 0.38328421115875244,
      "logps/chosen": -288.2322082519531,
      "logps/rejected": -339.9881896972656,
      "loss": 0.8241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.177568256855011,
      "rewards/margins": 0.03352278470993042,
      "rewards/rejected": -0.2110910266637802,
      "step": 1739
    },
    {
      "epoch": 0.22769281099206673,
      "grad_norm": 6.614190578460693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3187955617904663,
      "logits/rejected": 0.2682269513607025,
      "logps/chosen": -479.50244140625,
      "logps/rejected": -432.26678466796875,
      "loss": 0.5185,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10688479244709015,
      "rewards/margins": 0.6880568265914917,
      "rewards/rejected": -0.5811719298362732,
      "step": 1740
    },
    {
      "epoch": 0.2278236689294185,
      "grad_norm": 6.973458290100098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19458609819412231,
      "logits/rejected": 0.22325244545936584,
      "logps/chosen": -431.3463134765625,
      "logps/rejected": -639.1978759765625,
      "loss": 0.4642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.26748645305633545,
      "rewards/margins": 1.4203954935073853,
      "rewards/rejected": -1.6878819465637207,
      "step": 1741
    },
    {
      "epoch": 0.22795452686677026,
      "grad_norm": 5.575541973114014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018458526581525803,
      "logits/rejected": 0.04210401326417923,
      "logps/chosen": -328.1390075683594,
      "logps/rejected": -304.87371826171875,
      "loss": 0.4352,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1171148344874382,
      "rewards/margins": 0.9813970327377319,
      "rewards/rejected": -0.8642822504043579,
      "step": 1742
    },
    {
      "epoch": 0.22808538480412202,
      "grad_norm": 6.2239251136779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4428233504295349,
      "logits/rejected": 0.27703896164894104,
      "logps/chosen": -537.2098999023438,
      "logps/rejected": -523.2114868164062,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08387604355812073,
      "rewards/margins": 0.7361888885498047,
      "rewards/rejected": -0.820064902305603,
      "step": 1743
    },
    {
      "epoch": 0.22821624274147378,
      "grad_norm": 6.286464214324951,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14170247316360474,
      "logits/rejected": 0.13191460072994232,
      "logps/chosen": -693.6576538085938,
      "logps/rejected": -582.9989013671875,
      "loss": 0.5115,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04101179167628288,
      "rewards/margins": 0.8167517185211182,
      "rewards/rejected": -0.7757399082183838,
      "step": 1744
    },
    {
      "epoch": 0.22834710067882555,
      "grad_norm": 6.294265270233154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5213720798492432,
      "logits/rejected": 0.32191264629364014,
      "logps/chosen": -378.2535095214844,
      "logps/rejected": -458.8750305175781,
      "loss": 0.5319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2596141993999481,
      "rewards/margins": 0.5536587238311768,
      "rewards/rejected": -0.29404449462890625,
      "step": 1745
    },
    {
      "epoch": 0.2284779586161773,
      "grad_norm": 7.256325721740723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0769893005490303,
      "logits/rejected": 0.015662062913179398,
      "logps/chosen": -439.1903381347656,
      "logps/rejected": -431.93341064453125,
      "loss": 0.6497,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07875880599021912,
      "rewards/margins": 0.4353385865688324,
      "rewards/rejected": -0.3565797805786133,
      "step": 1746
    },
    {
      "epoch": 0.22860881655352908,
      "grad_norm": 7.204935550689697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25276219844818115,
      "logits/rejected": 0.3686021566390991,
      "logps/chosen": -506.27838134765625,
      "logps/rejected": -426.27606201171875,
      "loss": 0.687,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19195041060447693,
      "rewards/margins": 0.34354689717292786,
      "rewards/rejected": -0.53549724817276,
      "step": 1747
    },
    {
      "epoch": 0.22873967449088084,
      "grad_norm": 11.090071678161621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3506650924682617,
      "logits/rejected": 0.27639949321746826,
      "logps/chosen": -406.1876525878906,
      "logps/rejected": -361.312255859375,
      "loss": 0.6815,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5941940546035767,
      "rewards/margins": 0.4814760386943817,
      "rewards/rejected": -1.0756700038909912,
      "step": 1748
    },
    {
      "epoch": 0.2288705324282326,
      "grad_norm": 5.500150680541992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11224271357059479,
      "logits/rejected": 0.3123946189880371,
      "logps/chosen": -540.7978515625,
      "logps/rejected": -421.84222412109375,
      "loss": 0.4839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5715230703353882,
      "rewards/margins": 0.7428361177444458,
      "rewards/rejected": -1.3143590688705444,
      "step": 1749
    },
    {
      "epoch": 0.22900139036558437,
      "grad_norm": 4.471704959869385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14974164962768555,
      "logits/rejected": 0.34970512986183167,
      "logps/chosen": -537.3339233398438,
      "logps/rejected": -471.6633605957031,
      "loss": 0.3269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36783942580223083,
      "rewards/margins": 1.192330241203308,
      "rewards/rejected": -0.8244907259941101,
      "step": 1750
    },
    {
      "epoch": 0.22913224830293613,
      "grad_norm": 7.179708957672119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21190328896045685,
      "logits/rejected": 0.09922875463962555,
      "logps/chosen": -531.3319091796875,
      "logps/rejected": -585.9899291992188,
      "loss": 0.6666,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6163480877876282,
      "rewards/margins": 0.29371145367622375,
      "rewards/rejected": -0.9100595116615295,
      "step": 1751
    },
    {
      "epoch": 0.2292631062402879,
      "grad_norm": 4.33146333694458,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09123047441244125,
      "logits/rejected": -0.0017012357711791992,
      "logps/chosen": -324.4208679199219,
      "logps/rejected": -358.67138671875,
      "loss": 0.3803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6024526357650757,
      "rewards/margins": 0.9752598404884338,
      "rewards/rejected": -1.5777124166488647,
      "step": 1752
    },
    {
      "epoch": 0.22939396417763966,
      "grad_norm": 6.532271385192871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24570713937282562,
      "logits/rejected": 0.25422975420951843,
      "logps/chosen": -440.3226318359375,
      "logps/rejected": -399.3235168457031,
      "loss": 0.6166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4434933066368103,
      "rewards/margins": 0.4019859731197357,
      "rewards/rejected": -0.8454792499542236,
      "step": 1753
    },
    {
      "epoch": 0.22952482211499142,
      "grad_norm": 6.066154479980469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5386610627174377,
      "logits/rejected": 0.3867686092853546,
      "logps/chosen": -634.6727905273438,
      "logps/rejected": -537.8616943359375,
      "loss": 0.4089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05945663899183273,
      "rewards/margins": 1.0583431720733643,
      "rewards/rejected": -0.9988864660263062,
      "step": 1754
    },
    {
      "epoch": 0.2296556800523432,
      "grad_norm": 4.134363174438477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008165317587554455,
      "logits/rejected": 0.021882183849811554,
      "logps/chosen": -486.1466369628906,
      "logps/rejected": -401.12109375,
      "loss": 0.296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.017811676487326622,
      "rewards/margins": 1.5627591609954834,
      "rewards/rejected": -1.5449475049972534,
      "step": 1755
    },
    {
      "epoch": 0.22978653798969495,
      "grad_norm": 4.532172203063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03328729420900345,
      "logits/rejected": 0.004963701590895653,
      "logps/chosen": -313.18487548828125,
      "logps/rejected": -282.3588562011719,
      "loss": 0.3841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.204376682639122,
      "rewards/margins": 1.2451682090759277,
      "rewards/rejected": -1.449544906616211,
      "step": 1756
    },
    {
      "epoch": 0.2299173959270467,
      "grad_norm": 3.998260974884033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20289091765880585,
      "logits/rejected": 0.13526862859725952,
      "logps/chosen": -412.13494873046875,
      "logps/rejected": -396.9419860839844,
      "loss": 0.309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.030286595225334167,
      "rewards/margins": 1.1894654035568237,
      "rewards/rejected": -1.2197519540786743,
      "step": 1757
    },
    {
      "epoch": 0.23004825386439845,
      "grad_norm": 7.222767353057861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07969065010547638,
      "logits/rejected": -0.18426886200904846,
      "logps/chosen": -497.9449768066406,
      "logps/rejected": -323.36181640625,
      "loss": 0.5374,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4627651274204254,
      "rewards/margins": 0.9188574552536011,
      "rewards/rejected": -1.381622552871704,
      "step": 1758
    },
    {
      "epoch": 0.2301791118017502,
      "grad_norm": 5.5887837409973145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1073841005563736,
      "logits/rejected": 0.08672774583101273,
      "logps/chosen": -384.02313232421875,
      "logps/rejected": -400.97515869140625,
      "loss": 0.4107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.021162986755371094,
      "rewards/margins": 0.9704199433326721,
      "rewards/rejected": -0.9492568969726562,
      "step": 1759
    },
    {
      "epoch": 0.23030996973910198,
      "grad_norm": 5.523127555847168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2157078981399536,
      "logits/rejected": 0.23885411024093628,
      "logps/chosen": -466.8106384277344,
      "logps/rejected": -410.56634521484375,
      "loss": 0.4916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5095966458320618,
      "rewards/margins": 0.628750205039978,
      "rewards/rejected": -1.1383469104766846,
      "step": 1760
    },
    {
      "epoch": 0.23044082767645374,
      "grad_norm": 5.449763298034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28626877069473267,
      "logits/rejected": 0.2638603746891022,
      "logps/chosen": -634.9109497070312,
      "logps/rejected": -542.0255126953125,
      "loss": 0.337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5506747364997864,
      "rewards/margins": 1.244559645652771,
      "rewards/rejected": -0.6938848495483398,
      "step": 1761
    },
    {
      "epoch": 0.2305716856138055,
      "grad_norm": 4.998621463775635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22301003336906433,
      "logits/rejected": 0.1347247064113617,
      "logps/chosen": -481.0263671875,
      "logps/rejected": -294.36346435546875,
      "loss": 0.3232,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.041354745626449585,
      "rewards/margins": 1.2174042463302612,
      "rewards/rejected": -1.2587589025497437,
      "step": 1762
    },
    {
      "epoch": 0.23070254355115727,
      "grad_norm": 6.808145999908447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0051474012434482574,
      "logits/rejected": 0.2367914468050003,
      "logps/chosen": -402.9121398925781,
      "logps/rejected": -353.99444580078125,
      "loss": 0.6114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19550704956054688,
      "rewards/margins": 0.46226900815963745,
      "rewards/rejected": -0.6577761173248291,
      "step": 1763
    },
    {
      "epoch": 0.23083340148850903,
      "grad_norm": 8.556662559509277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21683728694915771,
      "logits/rejected": 0.12379900366067886,
      "logps/chosen": -448.2236022949219,
      "logps/rejected": -387.3316345214844,
      "loss": 0.609,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.993920087814331,
      "rewards/margins": 0.6611137390136719,
      "rewards/rejected": -1.6550335884094238,
      "step": 1764
    },
    {
      "epoch": 0.2309642594258608,
      "grad_norm": 6.31689453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2661704123020172,
      "logits/rejected": 0.5317291021347046,
      "logps/chosen": -428.1229553222656,
      "logps/rejected": -451.8993835449219,
      "loss": 0.5414,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11939773708581924,
      "rewards/margins": 0.834229052066803,
      "rewards/rejected": -0.7148313522338867,
      "step": 1765
    },
    {
      "epoch": 0.23109511736321256,
      "grad_norm": 4.470777988433838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1540852040052414,
      "logits/rejected": 0.15520982444286346,
      "logps/chosen": -414.9942321777344,
      "logps/rejected": -512.7005004882812,
      "loss": 0.3616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24849680066108704,
      "rewards/margins": 1.0563068389892578,
      "rewards/rejected": -1.304803729057312,
      "step": 1766
    },
    {
      "epoch": 0.23122597530056432,
      "grad_norm": 15.959622383117676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2566829025745392,
      "logits/rejected": 0.06709956377744675,
      "logps/chosen": -604.289794921875,
      "logps/rejected": -529.8833618164062,
      "loss": 0.9487,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18795469403266907,
      "rewards/margins": 0.08197976648807526,
      "rewards/rejected": -0.26993444561958313,
      "step": 1767
    },
    {
      "epoch": 0.2313568332379161,
      "grad_norm": 9.433938980102539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45126205682754517,
      "logits/rejected": 0.4622620940208435,
      "logps/chosen": -496.1396484375,
      "logps/rejected": -366.585205078125,
      "loss": 0.8986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4925723969936371,
      "rewards/margins": 0.10622631013393402,
      "rewards/rejected": -0.5987987518310547,
      "step": 1768
    },
    {
      "epoch": 0.23148769117526785,
      "grad_norm": 7.535093307495117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40624648332595825,
      "logits/rejected": 0.2647148370742798,
      "logps/chosen": -450.31610107421875,
      "logps/rejected": -499.93951416015625,
      "loss": 0.5438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4242936968803406,
      "rewards/margins": 1.4000846147537231,
      "rewards/rejected": -1.8243781328201294,
      "step": 1769
    },
    {
      "epoch": 0.23161854911261961,
      "grad_norm": 7.467041492462158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09616018086671829,
      "logits/rejected": 0.09739822149276733,
      "logps/chosen": -619.3714599609375,
      "logps/rejected": -545.0886840820312,
      "loss": 0.4524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1436714231967926,
      "rewards/margins": 1.0101581811904907,
      "rewards/rejected": -1.153829574584961,
      "step": 1770
    },
    {
      "epoch": 0.23174940704997138,
      "grad_norm": 6.321193218231201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1603545993566513,
      "logits/rejected": 0.05325592681765556,
      "logps/chosen": -347.084228515625,
      "logps/rejected": -364.5274963378906,
      "loss": 0.672,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4361054301261902,
      "rewards/margins": 0.2675336003303528,
      "rewards/rejected": -0.703639030456543,
      "step": 1771
    },
    {
      "epoch": 0.23188026498732314,
      "grad_norm": 8.793963432312012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09188481420278549,
      "logits/rejected": -0.10818245261907578,
      "logps/chosen": -476.84375,
      "logps/rejected": -327.7112731933594,
      "loss": 0.5892,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31016674637794495,
      "rewards/margins": 0.6992744207382202,
      "rewards/rejected": -1.0094412565231323,
      "step": 1772
    },
    {
      "epoch": 0.2320111229246749,
      "grad_norm": 7.028232097625732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06666646897792816,
      "logits/rejected": -0.03166676312685013,
      "logps/chosen": -714.9258422851562,
      "logps/rejected": -455.06964111328125,
      "loss": 0.4767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4189308285713196,
      "rewards/margins": 1.0870606899261475,
      "rewards/rejected": -0.6681297421455383,
      "step": 1773
    },
    {
      "epoch": 0.23214198086202667,
      "grad_norm": 6.132767677307129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48019495606422424,
      "logits/rejected": 0.5653670430183411,
      "logps/chosen": -392.94366455078125,
      "logps/rejected": -439.217529296875,
      "loss": 0.4649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1168128103017807,
      "rewards/margins": 0.9482297301292419,
      "rewards/rejected": -1.065042495727539,
      "step": 1774
    },
    {
      "epoch": 0.23227283879937843,
      "grad_norm": 9.298087120056152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18311528861522675,
      "logits/rejected": 0.0867839902639389,
      "logps/chosen": -435.29754638671875,
      "logps/rejected": -463.9228515625,
      "loss": 0.8241,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20946982502937317,
      "rewards/margins": -0.05670320987701416,
      "rewards/rejected": -0.15276658535003662,
      "step": 1775
    },
    {
      "epoch": 0.2324036967367302,
      "grad_norm": 6.645636558532715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3066650927066803,
      "logits/rejected": 0.05458151176571846,
      "logps/chosen": -497.1794738769531,
      "logps/rejected": -671.4661254882812,
      "loss": 0.5558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13987864553928375,
      "rewards/margins": 0.9469162225723267,
      "rewards/rejected": -1.0867949724197388,
      "step": 1776
    },
    {
      "epoch": 0.23253455467408196,
      "grad_norm": 5.902970790863037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41001126170158386,
      "logits/rejected": 0.42031317949295044,
      "logps/chosen": -481.6437072753906,
      "logps/rejected": -447.271484375,
      "loss": 0.5433,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08984144032001495,
      "rewards/margins": 0.6265178322792053,
      "rewards/rejected": -0.5366764068603516,
      "step": 1777
    },
    {
      "epoch": 0.23266541261143373,
      "grad_norm": 6.985084533691406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33336517214775085,
      "logits/rejected": 0.23128479719161987,
      "logps/chosen": -482.39141845703125,
      "logps/rejected": -356.6439514160156,
      "loss": 0.7278,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03836366534233093,
      "rewards/margins": 0.31259119510650635,
      "rewards/rejected": -0.3509548008441925,
      "step": 1778
    },
    {
      "epoch": 0.23279627054878546,
      "grad_norm": 5.512653350830078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015057988464832306,
      "logits/rejected": -0.006844494491815567,
      "logps/chosen": -577.8395385742188,
      "logps/rejected": -455.99554443359375,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11999226361513138,
      "rewards/margins": 0.8715692162513733,
      "rewards/rejected": -0.9915614724159241,
      "step": 1779
    },
    {
      "epoch": 0.23292712848613722,
      "grad_norm": 7.166416645050049,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027767017483711243,
      "logits/rejected": -0.06665803492069244,
      "logps/chosen": -397.046142578125,
      "logps/rejected": -429.4203796386719,
      "loss": 0.5173,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.49249058961868286,
      "rewards/margins": 0.7884628176689148,
      "rewards/rejected": -1.2809535264968872,
      "step": 1780
    },
    {
      "epoch": 0.233057986423489,
      "grad_norm": 3.8479573726654053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07316387444734573,
      "logits/rejected": 0.01578143984079361,
      "logps/chosen": -444.7004699707031,
      "logps/rejected": -411.39453125,
      "loss": 0.232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11844749748706818,
      "rewards/margins": 1.465620756149292,
      "rewards/rejected": -1.5840681791305542,
      "step": 1781
    },
    {
      "epoch": 0.23318884436084075,
      "grad_norm": 4.454925537109375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24999135732650757,
      "logits/rejected": 0.3679002523422241,
      "logps/chosen": -320.1946105957031,
      "logps/rejected": -322.55474853515625,
      "loss": 0.3956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1296464204788208,
      "rewards/margins": 0.9026976823806763,
      "rewards/rejected": -0.7730513215065002,
      "step": 1782
    },
    {
      "epoch": 0.23331970229819252,
      "grad_norm": 7.097326278686523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22392895817756653,
      "logits/rejected": 0.17131128907203674,
      "logps/chosen": -610.7686767578125,
      "logps/rejected": -579.1268920898438,
      "loss": 0.5069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10794715583324432,
      "rewards/margins": 0.5177064538002014,
      "rewards/rejected": -0.6256536245346069,
      "step": 1783
    },
    {
      "epoch": 0.23345056023554428,
      "grad_norm": 8.041457176208496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1975451409816742,
      "logits/rejected": 0.24102017283439636,
      "logps/chosen": -414.47198486328125,
      "logps/rejected": -555.5799560546875,
      "loss": 0.6865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47368165850639343,
      "rewards/margins": 0.44156935811042786,
      "rewards/rejected": -0.9152510166168213,
      "step": 1784
    },
    {
      "epoch": 0.23358141817289604,
      "grad_norm": 5.651119709014893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1757161170244217,
      "logits/rejected": 0.3190400302410126,
      "logps/chosen": -677.48291015625,
      "logps/rejected": -686.0277099609375,
      "loss": 0.3263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7067554593086243,
      "rewards/margins": 1.2692482471466064,
      "rewards/rejected": -0.5624927878379822,
      "step": 1785
    },
    {
      "epoch": 0.2337122761102478,
      "grad_norm": 8.928364753723145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034261178225278854,
      "logits/rejected": 0.005985669791698456,
      "logps/chosen": -410.49505615234375,
      "logps/rejected": -471.2867431640625,
      "loss": 0.865,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8373867273330688,
      "rewards/margins": 0.14677312970161438,
      "rewards/rejected": -0.9841598272323608,
      "step": 1786
    },
    {
      "epoch": 0.23384313404759957,
      "grad_norm": 7.119853973388672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19297270476818085,
      "logits/rejected": 0.21201352775096893,
      "logps/chosen": -305.3565979003906,
      "logps/rejected": -359.48187255859375,
      "loss": 0.7546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1871301829814911,
      "rewards/margins": 0.10613517463207245,
      "rewards/rejected": -0.29326534271240234,
      "step": 1787
    },
    {
      "epoch": 0.23397399198495133,
      "grad_norm": 6.5984787940979,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11045821756124496,
      "logits/rejected": 0.11898957192897797,
      "logps/chosen": -489.89422607421875,
      "logps/rejected": -571.8414306640625,
      "loss": 0.4587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08096103370189667,
      "rewards/margins": 0.7116479873657227,
      "rewards/rejected": -0.7926089763641357,
      "step": 1788
    },
    {
      "epoch": 0.2341048499223031,
      "grad_norm": 3.636362075805664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3250330090522766,
      "logits/rejected": 0.26897042989730835,
      "logps/chosen": -396.3511657714844,
      "logps/rejected": -470.2737731933594,
      "loss": 0.3066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2056797444820404,
      "rewards/margins": 1.3353739976882935,
      "rewards/rejected": -1.5410536527633667,
      "step": 1789
    },
    {
      "epoch": 0.23423570785965486,
      "grad_norm": 8.088043212890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4465788006782532,
      "logits/rejected": 0.5713315010070801,
      "logps/chosen": -426.6556396484375,
      "logps/rejected": -404.7339782714844,
      "loss": 0.692,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.34044018387794495,
      "rewards/margins": 0.2861335575580597,
      "rewards/rejected": 0.05430661141872406,
      "step": 1790
    },
    {
      "epoch": 0.23436656579700663,
      "grad_norm": 3.583214282989502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32951486110687256,
      "logits/rejected": 0.3100269138813019,
      "logps/chosen": -307.1022033691406,
      "logps/rejected": -325.0086669921875,
      "loss": 0.3045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3093871474266052,
      "rewards/margins": 1.3712371587753296,
      "rewards/rejected": -1.68062424659729,
      "step": 1791
    },
    {
      "epoch": 0.2344974237343584,
      "grad_norm": 8.675310134887695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25739380717277527,
      "logits/rejected": 0.24769219756126404,
      "logps/chosen": -437.15252685546875,
      "logps/rejected": -514.3195190429688,
      "loss": 0.565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1607416272163391,
      "rewards/margins": 1.2442353963851929,
      "rewards/rejected": -1.0834938287734985,
      "step": 1792
    },
    {
      "epoch": 0.23462828167171015,
      "grad_norm": 7.185945987701416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15639495849609375,
      "logits/rejected": 0.00978206004947424,
      "logps/chosen": -445.53070068359375,
      "logps/rejected": -377.0885314941406,
      "loss": 0.6213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5480091571807861,
      "rewards/margins": 0.6774505972862244,
      "rewards/rejected": -1.2254596948623657,
      "step": 1793
    },
    {
      "epoch": 0.23475913960906192,
      "grad_norm": 6.196441173553467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1546458899974823,
      "logits/rejected": 0.2567691206932068,
      "logps/chosen": -323.4769592285156,
      "logps/rejected": -324.60943603515625,
      "loss": 0.4807,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3901485800743103,
      "rewards/margins": 0.9773842096328735,
      "rewards/rejected": -1.367532730102539,
      "step": 1794
    },
    {
      "epoch": 0.23488999754641368,
      "grad_norm": 4.521017074584961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11214402318000793,
      "logits/rejected": 0.057592958211898804,
      "logps/chosen": -403.98309326171875,
      "logps/rejected": -475.93914794921875,
      "loss": 0.3657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5891959071159363,
      "rewards/margins": 1.090664029121399,
      "rewards/rejected": -1.67985999584198,
      "step": 1795
    },
    {
      "epoch": 0.23502085548376545,
      "grad_norm": 5.60481071472168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12579630315303802,
      "logits/rejected": 0.09271392226219177,
      "logps/chosen": -555.7069091796875,
      "logps/rejected": -425.64886474609375,
      "loss": 0.5301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15362432599067688,
      "rewards/margins": 0.8041160702705383,
      "rewards/rejected": -0.6504917144775391,
      "step": 1796
    },
    {
      "epoch": 0.2351517134211172,
      "grad_norm": 6.551993370056152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3138294219970703,
      "logits/rejected": 0.23952533304691315,
      "logps/chosen": -405.0849914550781,
      "logps/rejected": -306.26605224609375,
      "loss": 0.5151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13119827210903168,
      "rewards/margins": 0.8903136253356934,
      "rewards/rejected": -1.0215119123458862,
      "step": 1797
    },
    {
      "epoch": 0.23528257135846897,
      "grad_norm": 3.616219997406006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23399150371551514,
      "logits/rejected": 0.24592983722686768,
      "logps/chosen": -397.24664306640625,
      "logps/rejected": -498.24713134765625,
      "loss": 0.3339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4895218014717102,
      "rewards/margins": 1.3447576761245728,
      "rewards/rejected": -1.8342794179916382,
      "step": 1798
    },
    {
      "epoch": 0.23541342929582074,
      "grad_norm": 7.156924247741699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3559640347957611,
      "logits/rejected": 0.17063768208026886,
      "logps/chosen": -345.98980712890625,
      "logps/rejected": -630.8815307617188,
      "loss": 0.6215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23035067319869995,
      "rewards/margins": 1.0213160514831543,
      "rewards/rejected": -1.251666784286499,
      "step": 1799
    },
    {
      "epoch": 0.2355442872331725,
      "grad_norm": 6.755992889404297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3954288959503174,
      "logits/rejected": 0.3696810305118561,
      "logps/chosen": -396.9964599609375,
      "logps/rejected": -398.80731201171875,
      "loss": 0.6235,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5839999914169312,
      "rewards/margins": 0.42721065878868103,
      "rewards/rejected": -1.0112106800079346,
      "step": 1800
    },
    {
      "epoch": 0.23567514517052424,
      "grad_norm": 4.665604591369629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3687519133090973,
      "logits/rejected": 0.5217946767807007,
      "logps/chosen": -436.4066467285156,
      "logps/rejected": -459.27117919921875,
      "loss": 0.3156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38892877101898193,
      "rewards/margins": 1.9643943309783936,
      "rewards/rejected": -2.353323221206665,
      "step": 1801
    },
    {
      "epoch": 0.235806003107876,
      "grad_norm": 9.325478553771973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21711724996566772,
      "logits/rejected": 0.30305784940719604,
      "logps/chosen": -637.920654296875,
      "logps/rejected": -537.8544311523438,
      "loss": 0.7387,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2940473556518555,
      "rewards/margins": 0.06792905926704407,
      "rewards/rejected": -1.3619765043258667,
      "step": 1802
    },
    {
      "epoch": 0.23593686104522776,
      "grad_norm": 9.610884666442871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2752998471260071,
      "logits/rejected": 0.24480438232421875,
      "logps/chosen": -433.3258972167969,
      "logps/rejected": -529.4094848632812,
      "loss": 0.7931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.49922072887420654,
      "rewards/margins": 0.028843507170677185,
      "rewards/rejected": -0.5280641317367554,
      "step": 1803
    },
    {
      "epoch": 0.23606771898257953,
      "grad_norm": 9.36738395690918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0908651351928711,
      "logits/rejected": 0.08466768264770508,
      "logps/chosen": -579.3322143554688,
      "logps/rejected": -590.0909423828125,
      "loss": 0.5258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6010828018188477,
      "rewards/margins": 1.0531744956970215,
      "rewards/rejected": -1.6542572975158691,
      "step": 1804
    },
    {
      "epoch": 0.2361985769199313,
      "grad_norm": 7.814882278442383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23203682899475098,
      "logits/rejected": 0.2514495253562927,
      "logps/chosen": -484.20574951171875,
      "logps/rejected": -347.96563720703125,
      "loss": 0.6673,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07550984621047974,
      "rewards/margins": 0.29467374086380005,
      "rewards/rejected": -0.21916386485099792,
      "step": 1805
    },
    {
      "epoch": 0.23632943485728305,
      "grad_norm": 3.707007646560669,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16719841957092285,
      "logits/rejected": -0.10365208983421326,
      "logps/chosen": -441.17218017578125,
      "logps/rejected": -658.6678466796875,
      "loss": 0.2335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13201045989990234,
      "rewards/margins": 2.1602749824523926,
      "rewards/rejected": -2.292285680770874,
      "step": 1806
    },
    {
      "epoch": 0.23646029279463482,
      "grad_norm": 4.929722309112549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2920316159725189,
      "logits/rejected": 0.1532694697380066,
      "logps/chosen": -531.8081665039062,
      "logps/rejected": -624.0685424804688,
      "loss": 0.2774,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16665954887866974,
      "rewards/margins": 1.5443145036697388,
      "rewards/rejected": -1.7109739780426025,
      "step": 1807
    },
    {
      "epoch": 0.23659115073198658,
      "grad_norm": 4.619592666625977,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12036196887493134,
      "logits/rejected": 0.4956052601337433,
      "logps/chosen": -394.0687561035156,
      "logps/rejected": -351.4327087402344,
      "loss": 0.3545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45954591035842896,
      "rewards/margins": 1.1177111864089966,
      "rewards/rejected": -1.5772571563720703,
      "step": 1808
    },
    {
      "epoch": 0.23672200866933835,
      "grad_norm": 6.312192916870117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1972719430923462,
      "logits/rejected": 0.17894938588142395,
      "logps/chosen": -305.5194396972656,
      "logps/rejected": -357.344482421875,
      "loss": 0.6215,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20540304481983185,
      "rewards/margins": 0.25825244188308716,
      "rewards/rejected": -0.4636554718017578,
      "step": 1809
    },
    {
      "epoch": 0.2368528666066901,
      "grad_norm": 6.857192039489746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.437918484210968,
      "logits/rejected": 0.38786593079566956,
      "logps/chosen": -467.7341613769531,
      "logps/rejected": -571.4649658203125,
      "loss": 0.55,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021779820322990417,
      "rewards/margins": 0.7666418552398682,
      "rewards/rejected": -0.7448619604110718,
      "step": 1810
    },
    {
      "epoch": 0.23698372454404187,
      "grad_norm": 10.210664749145508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18361787497997284,
      "logits/rejected": 0.173667311668396,
      "logps/chosen": -528.1363525390625,
      "logps/rejected": -488.1273193359375,
      "loss": 0.3827,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07718735933303833,
      "rewards/margins": 0.9166185259819031,
      "rewards/rejected": -0.9938058853149414,
      "step": 1811
    },
    {
      "epoch": 0.23711458248139364,
      "grad_norm": 11.064899444580078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2477077692747116,
      "logits/rejected": 0.27363064885139465,
      "logps/chosen": -639.1403198242188,
      "logps/rejected": -677.554443359375,
      "loss": 0.7536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5979881286621094,
      "rewards/margins": 0.10538068413734436,
      "rewards/rejected": -0.7033687829971313,
      "step": 1812
    },
    {
      "epoch": 0.2372454404187454,
      "grad_norm": 7.832789421081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037429772317409515,
      "logits/rejected": 0.11436308920383453,
      "logps/chosen": -679.7517700195312,
      "logps/rejected": -559.7954711914062,
      "loss": 0.5933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5630239844322205,
      "rewards/margins": 0.7906591892242432,
      "rewards/rejected": -0.22763529419898987,
      "step": 1813
    },
    {
      "epoch": 0.23737629835609717,
      "grad_norm": 5.492949962615967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2495911419391632,
      "logits/rejected": 0.4769449830055237,
      "logps/chosen": -610.8551635742188,
      "logps/rejected": -516.926025390625,
      "loss": 0.4067,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24709492921829224,
      "rewards/margins": 1.061843752861023,
      "rewards/rejected": -0.8147488236427307,
      "step": 1814
    },
    {
      "epoch": 0.23750715629344893,
      "grad_norm": 7.385339736938477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23181042075157166,
      "logits/rejected": 0.24198436737060547,
      "logps/chosen": -519.6825561523438,
      "logps/rejected": -419.03790283203125,
      "loss": 0.5747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6462118029594421,
      "rewards/margins": 0.649059534072876,
      "rewards/rejected": -1.2952712774276733,
      "step": 1815
    },
    {
      "epoch": 0.2376380142308007,
      "grad_norm": 15.484493255615234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019482845440506935,
      "logits/rejected": 0.1446433961391449,
      "logps/chosen": -497.27520751953125,
      "logps/rejected": -330.4830322265625,
      "loss": 1.1677,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2944550514221191,
      "rewards/margins": -0.42314234375953674,
      "rewards/rejected": -0.8713127374649048,
      "step": 1816
    },
    {
      "epoch": 0.23776887216815246,
      "grad_norm": 5.378864288330078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2705172300338745,
      "logits/rejected": 0.10415995866060257,
      "logps/chosen": -430.53326416015625,
      "logps/rejected": -436.6410217285156,
      "loss": 0.4373,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06835345923900604,
      "rewards/margins": 1.345360279083252,
      "rewards/rejected": -1.4137139320373535,
      "step": 1817
    },
    {
      "epoch": 0.23789973010550422,
      "grad_norm": 7.155630111694336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054896481335163116,
      "logits/rejected": 0.17242099344730377,
      "logps/chosen": -596.1614990234375,
      "logps/rejected": -803.493408203125,
      "loss": 0.515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28520795702934265,
      "rewards/margins": 0.6691632866859436,
      "rewards/rejected": -0.9543712139129639,
      "step": 1818
    },
    {
      "epoch": 0.23803058804285598,
      "grad_norm": 7.238985538482666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3234867453575134,
      "logits/rejected": 0.08071400225162506,
      "logps/chosen": -468.87908935546875,
      "logps/rejected": -457.431640625,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15038128197193146,
      "rewards/margins": 0.6026357412338257,
      "rewards/rejected": -0.7530171275138855,
      "step": 1819
    },
    {
      "epoch": 0.23816144598020775,
      "grad_norm": 6.83044958114624,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42803293466567993,
      "logits/rejected": 0.540795087814331,
      "logps/chosen": -405.0876770019531,
      "logps/rejected": -381.491455078125,
      "loss": 0.5678,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.074737787246704,
      "rewards/margins": 0.6071069836616516,
      "rewards/rejected": -1.681844711303711,
      "step": 1820
    },
    {
      "epoch": 0.2382923039175595,
      "grad_norm": 7.954569339752197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00756758451461792,
      "logits/rejected": 0.1317482888698578,
      "logps/chosen": -496.024658203125,
      "logps/rejected": -681.1331787109375,
      "loss": 0.7505,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0883556455373764,
      "rewards/margins": 0.04202613979578018,
      "rewards/rejected": 0.046329498291015625,
      "step": 1821
    },
    {
      "epoch": 0.23842316185491128,
      "grad_norm": 4.185693740844727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3612350523471832,
      "logits/rejected": 0.2954903841018677,
      "logps/chosen": -353.4019775390625,
      "logps/rejected": -573.912353515625,
      "loss": 0.3554,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.010550878942012787,
      "rewards/margins": 2.259882688522339,
      "rewards/rejected": -2.2493319511413574,
      "step": 1822
    },
    {
      "epoch": 0.238554019792263,
      "grad_norm": 5.372060775756836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07433120906352997,
      "logits/rejected": 0.11946038901805878,
      "logps/chosen": -509.57989501953125,
      "logps/rejected": -509.59344482421875,
      "loss": 0.3619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08185500651597977,
      "rewards/margins": 1.0558868646621704,
      "rewards/rejected": -1.1377418041229248,
      "step": 1823
    },
    {
      "epoch": 0.23868487772961477,
      "grad_norm": 4.214189052581787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19328618049621582,
      "logits/rejected": 0.10267520695924759,
      "logps/chosen": -479.68359375,
      "logps/rejected": -477.3899230957031,
      "loss": 0.3358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1496339738368988,
      "rewards/margins": 1.0719902515411377,
      "rewards/rejected": -1.2216241359710693,
      "step": 1824
    },
    {
      "epoch": 0.23881573566696654,
      "grad_norm": 9.070154190063477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08559545874595642,
      "logits/rejected": 0.11253324151039124,
      "logps/chosen": -501.5193176269531,
      "logps/rejected": -398.2399597167969,
      "loss": 0.5821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.473160982131958,
      "rewards/margins": 0.38488027453422546,
      "rewards/rejected": -0.8580411672592163,
      "step": 1825
    },
    {
      "epoch": 0.2389465936043183,
      "grad_norm": 6.593343257904053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09546204656362534,
      "logits/rejected": -0.05247844383120537,
      "logps/chosen": -425.9954833984375,
      "logps/rejected": -624.6893920898438,
      "loss": 0.3773,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47053906321525574,
      "rewards/margins": 1.348744511604309,
      "rewards/rejected": -1.8192836046218872,
      "step": 1826
    },
    {
      "epoch": 0.23907745154167007,
      "grad_norm": 11.51846694946289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07865423709154129,
      "logits/rejected": 0.07582805305719376,
      "logps/chosen": -357.6891784667969,
      "logps/rejected": -307.6396179199219,
      "loss": 0.8185,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9432657361030579,
      "rewards/margins": 0.18507422506809235,
      "rewards/rejected": -1.1283398866653442,
      "step": 1827
    },
    {
      "epoch": 0.23920830947902183,
      "grad_norm": 7.304340362548828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13035236299037933,
      "logits/rejected": 0.2591882348060608,
      "logps/chosen": -451.90740966796875,
      "logps/rejected": -530.665283203125,
      "loss": 0.5271,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23984448611736298,
      "rewards/margins": 1.1241912841796875,
      "rewards/rejected": -1.3640358448028564,
      "step": 1828
    },
    {
      "epoch": 0.2393391674163736,
      "grad_norm": 8.040098190307617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1397152543067932,
      "logits/rejected": -0.21610033512115479,
      "logps/chosen": -447.95880126953125,
      "logps/rejected": -417.19488525390625,
      "loss": 0.575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5152910351753235,
      "rewards/margins": 0.8679029941558838,
      "rewards/rejected": -1.383194088935852,
      "step": 1829
    },
    {
      "epoch": 0.23947002535372536,
      "grad_norm": 5.941047191619873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04567534476518631,
      "logits/rejected": 0.10569992661476135,
      "logps/chosen": -547.3184204101562,
      "logps/rejected": -527.5465698242188,
      "loss": 0.3472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30730578303337097,
      "rewards/margins": 1.096360445022583,
      "rewards/rejected": -0.7890546321868896,
      "step": 1830
    },
    {
      "epoch": 0.23960088329107712,
      "grad_norm": 7.63017463684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13663560152053833,
      "logits/rejected": -0.1459924578666687,
      "logps/chosen": -482.11529541015625,
      "logps/rejected": -478.85845947265625,
      "loss": 0.4845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4373279809951782,
      "rewards/margins": 1.0415157079696655,
      "rewards/rejected": -1.4788434505462646,
      "step": 1831
    },
    {
      "epoch": 0.23973174122842889,
      "grad_norm": 6.821032524108887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3711889684200287,
      "logits/rejected": 0.22409147024154663,
      "logps/chosen": -433.652587890625,
      "logps/rejected": -318.1828918457031,
      "loss": 0.6098,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0752859115600586,
      "rewards/margins": 0.41621553897857666,
      "rewards/rejected": -0.3409295678138733,
      "step": 1832
    },
    {
      "epoch": 0.23986259916578065,
      "grad_norm": 8.837540626525879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26057183742523193,
      "logits/rejected": 0.3590448498725891,
      "logps/chosen": -463.8232116699219,
      "logps/rejected": -371.8603210449219,
      "loss": 0.6764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5515074729919434,
      "rewards/margins": 0.41925734281539917,
      "rewards/rejected": -0.9707647562026978,
      "step": 1833
    },
    {
      "epoch": 0.2399934571031324,
      "grad_norm": 8.301823616027832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11780790984630585,
      "logits/rejected": 0.2120116502046585,
      "logps/chosen": -575.066650390625,
      "logps/rejected": -664.5654907226562,
      "loss": 0.7422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.017119035124778748,
      "rewards/margins": 0.37600478529930115,
      "rewards/rejected": -0.3931238353252411,
      "step": 1834
    },
    {
      "epoch": 0.24012431504048418,
      "grad_norm": 10.434847831726074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028549201786518097,
      "logits/rejected": -0.02325006015598774,
      "logps/chosen": -598.7369384765625,
      "logps/rejected": -560.5721435546875,
      "loss": 0.7258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48220962285995483,
      "rewards/margins": 0.7851622700691223,
      "rewards/rejected": -1.2673718929290771,
      "step": 1835
    },
    {
      "epoch": 0.24025517297783594,
      "grad_norm": 10.592743873596191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23418304324150085,
      "logits/rejected": 0.15535643696784973,
      "logps/chosen": -497.44500732421875,
      "logps/rejected": -464.6914367675781,
      "loss": 0.4397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1991020292043686,
      "rewards/margins": 0.8700868487358093,
      "rewards/rejected": -1.0691888332366943,
      "step": 1836
    },
    {
      "epoch": 0.2403860309151877,
      "grad_norm": 4.662222862243652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20294660329818726,
      "logits/rejected": -0.06101372838020325,
      "logps/chosen": -335.5934143066406,
      "logps/rejected": -383.5933837890625,
      "loss": 0.3073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6424753069877625,
      "rewards/margins": 1.3569931983947754,
      "rewards/rejected": -1.9994688034057617,
      "step": 1837
    },
    {
      "epoch": 0.24051688885253947,
      "grad_norm": 5.079294204711914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43017899990081787,
      "logits/rejected": 0.25774896144866943,
      "logps/chosen": -510.843505859375,
      "logps/rejected": -625.6672973632812,
      "loss": 0.2744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24741479754447937,
      "rewards/margins": 1.844451904296875,
      "rewards/rejected": -1.5970369577407837,
      "step": 1838
    },
    {
      "epoch": 0.24064774678989123,
      "grad_norm": 4.606470584869385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38245725631713867,
      "logits/rejected": 0.16244864463806152,
      "logps/chosen": -403.5300598144531,
      "logps/rejected": -395.45269775390625,
      "loss": 0.3932,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0473155677318573,
      "rewards/margins": 1.1882368326187134,
      "rewards/rejected": -1.235552191734314,
      "step": 1839
    },
    {
      "epoch": 0.240778604727243,
      "grad_norm": 5.6078386306762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1719754934310913,
      "logits/rejected": -0.20736274123191833,
      "logps/chosen": -521.3642578125,
      "logps/rejected": -385.7306823730469,
      "loss": 0.4134,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.021124830469489098,
      "rewards/margins": 0.9174124598503113,
      "rewards/rejected": -0.9385372400283813,
      "step": 1840
    },
    {
      "epoch": 0.24090946266459476,
      "grad_norm": 6.343100547790527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21263466775417328,
      "logits/rejected": 0.2372792810201645,
      "logps/chosen": -425.3912048339844,
      "logps/rejected": -480.11669921875,
      "loss": 0.363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3809967041015625,
      "rewards/margins": 1.1816469430923462,
      "rewards/rejected": -1.5626437664031982,
      "step": 1841
    },
    {
      "epoch": 0.24104032060194652,
      "grad_norm": 4.016286373138428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23423662781715393,
      "logits/rejected": 0.17391622066497803,
      "logps/chosen": -404.3524475097656,
      "logps/rejected": -427.8624572753906,
      "loss": 0.2366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5563352704048157,
      "rewards/margins": 1.5730202198028564,
      "rewards/rejected": -1.0166847705841064,
      "step": 1842
    },
    {
      "epoch": 0.2411711785392983,
      "grad_norm": 5.32913064956665,
      "learning_rate": 5e-06,
      "logits/chosen": 0.153803288936615,
      "logits/rejected": 0.062082141637802124,
      "logps/chosen": -231.48837280273438,
      "logps/rejected": -316.6298828125,
      "loss": 0.5109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.481601357460022,
      "rewards/margins": 1.0919071435928345,
      "rewards/rejected": -1.5735085010528564,
      "step": 1843
    },
    {
      "epoch": 0.24130203647665005,
      "grad_norm": 6.038543224334717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30964741110801697,
      "logits/rejected": 0.4331986904144287,
      "logps/chosen": -338.4323425292969,
      "logps/rejected": -467.58746337890625,
      "loss": 0.5342,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20150911808013916,
      "rewards/margins": 0.5489274859428406,
      "rewards/rejected": -0.7504366040229797,
      "step": 1844
    },
    {
      "epoch": 0.2414328944140018,
      "grad_norm": 6.349628925323486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3692896366119385,
      "logits/rejected": 0.2172771394252777,
      "logps/chosen": -463.6035461425781,
      "logps/rejected": -507.921630859375,
      "loss": 0.5227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05704687535762787,
      "rewards/margins": 1.0126951932907104,
      "rewards/rejected": -1.069742202758789,
      "step": 1845
    },
    {
      "epoch": 0.24156375235135355,
      "grad_norm": 7.667622089385986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06021799147129059,
      "logits/rejected": -0.12557834386825562,
      "logps/chosen": -409.328857421875,
      "logps/rejected": -371.7083740234375,
      "loss": 0.8235,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.160210520029068,
      "rewards/margins": 0.30493155121803284,
      "rewards/rejected": -0.46514204144477844,
      "step": 1846
    },
    {
      "epoch": 0.2416946102887053,
      "grad_norm": 8.137969970703125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2592376470565796,
      "logits/rejected": 0.07906071096658707,
      "logps/chosen": -366.4932861328125,
      "logps/rejected": -305.96112060546875,
      "loss": 0.6895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4786617159843445,
      "rewards/margins": 0.3098852038383484,
      "rewards/rejected": -0.7885469794273376,
      "step": 1847
    },
    {
      "epoch": 0.24182546822605708,
      "grad_norm": 7.6733269691467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1827140897512436,
      "logits/rejected": -0.03553329408168793,
      "logps/chosen": -508.29876708984375,
      "logps/rejected": -549.0702514648438,
      "loss": 0.6315,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09883520752191544,
      "rewards/margins": 0.5306838750839233,
      "rewards/rejected": -0.6295191049575806,
      "step": 1848
    },
    {
      "epoch": 0.24195632616340884,
      "grad_norm": 9.074804306030273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13830916583538055,
      "logits/rejected": -0.16672539710998535,
      "logps/chosen": -472.8330383300781,
      "logps/rejected": -409.8013610839844,
      "loss": 0.734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5689411163330078,
      "rewards/margins": 0.21445190906524658,
      "rewards/rejected": -0.7833930253982544,
      "step": 1849
    },
    {
      "epoch": 0.2420871841007606,
      "grad_norm": 9.99512004852295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33174288272857666,
      "logits/rejected": 0.11534550040960312,
      "logps/chosen": -406.1026306152344,
      "logps/rejected": -383.3326416015625,
      "loss": 0.6468,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21023809909820557,
      "rewards/margins": 0.4477495551109314,
      "rewards/rejected": -0.657987654209137,
      "step": 1850
    },
    {
      "epoch": 0.24221804203811237,
      "grad_norm": 5.22883415222168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20358572900295258,
      "logits/rejected": 0.161625474691391,
      "logps/chosen": -584.6318359375,
      "logps/rejected": -572.9710083007812,
      "loss": 0.3516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03791464865207672,
      "rewards/margins": 1.1112812757492065,
      "rewards/rejected": -1.0733665227890015,
      "step": 1851
    },
    {
      "epoch": 0.24234889997546413,
      "grad_norm": 6.547737121582031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006065730005502701,
      "logits/rejected": 0.0876736044883728,
      "logps/chosen": -451.8747253417969,
      "logps/rejected": -383.1438293457031,
      "loss": 0.511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3327835202217102,
      "rewards/margins": 0.7811601758003235,
      "rewards/rejected": -1.1139436960220337,
      "step": 1852
    },
    {
      "epoch": 0.2424797579128159,
      "grad_norm": 6.392156600952148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19816498458385468,
      "logits/rejected": 0.06468353420495987,
      "logps/chosen": -453.5268249511719,
      "logps/rejected": -348.39093017578125,
      "loss": 0.5586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5533835887908936,
      "rewards/margins": 0.8055367469787598,
      "rewards/rejected": -0.25215303897857666,
      "step": 1853
    },
    {
      "epoch": 0.24261061585016766,
      "grad_norm": 5.726062774658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23992836475372314,
      "logits/rejected": 0.21993586421012878,
      "logps/chosen": -325.9279479980469,
      "logps/rejected": -427.6522521972656,
      "loss": 0.3623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30682966113090515,
      "rewards/margins": 1.1956114768981934,
      "rewards/rejected": -0.8887817859649658,
      "step": 1854
    },
    {
      "epoch": 0.24274147378751942,
      "grad_norm": 7.717740535736084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33854371309280396,
      "logits/rejected": -0.07663904130458832,
      "logps/chosen": -577.077880859375,
      "logps/rejected": -417.89337158203125,
      "loss": 0.7638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30191248655319214,
      "rewards/margins": 0.4991483688354492,
      "rewards/rejected": -0.8010608553886414,
      "step": 1855
    },
    {
      "epoch": 0.2428723317248712,
      "grad_norm": 7.718334674835205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3617303967475891,
      "logits/rejected": 0.14991459250450134,
      "logps/chosen": -380.39361572265625,
      "logps/rejected": -354.01739501953125,
      "loss": 0.5961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017021754756569862,
      "rewards/margins": 0.8335555791854858,
      "rewards/rejected": -0.8165338635444641,
      "step": 1856
    },
    {
      "epoch": 0.24300318966222295,
      "grad_norm": 5.490756511688232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3590936064720154,
      "logits/rejected": 0.11033165454864502,
      "logps/chosen": -490.6702575683594,
      "logps/rejected": -523.9548950195312,
      "loss": 0.367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.36179429292678833,
      "rewards/margins": 1.3337973356246948,
      "rewards/rejected": -0.9720029830932617,
      "step": 1857
    },
    {
      "epoch": 0.24313404759957472,
      "grad_norm": 5.1221089363098145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09660826623439789,
      "logits/rejected": -0.019158707931637764,
      "logps/chosen": -453.40234375,
      "logps/rejected": -344.8770751953125,
      "loss": 0.4503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17933541536331177,
      "rewards/margins": 0.8316507339477539,
      "rewards/rejected": -0.6523153185844421,
      "step": 1858
    },
    {
      "epoch": 0.24326490553692648,
      "grad_norm": 6.11196756362915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0974220335483551,
      "logits/rejected": -0.0662039965391159,
      "logps/chosen": -335.0115051269531,
      "logps/rejected": -295.6015625,
      "loss": 0.4976,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04784928262233734,
      "rewards/margins": 0.7953362464904785,
      "rewards/rejected": -0.8431854248046875,
      "step": 1859
    },
    {
      "epoch": 0.24339576347427824,
      "grad_norm": 8.59886646270752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1070096492767334,
      "logits/rejected": -0.053210947662591934,
      "logps/chosen": -439.72784423828125,
      "logps/rejected": -359.44561767578125,
      "loss": 0.7671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7482590079307556,
      "rewards/margins": 0.04594952613115311,
      "rewards/rejected": -0.7942085266113281,
      "step": 1860
    },
    {
      "epoch": 0.24352662141163,
      "grad_norm": 19.21978759765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06296482682228088,
      "logits/rejected": 0.04864239692687988,
      "logps/chosen": -814.49560546875,
      "logps/rejected": -557.0654907226562,
      "loss": 1.3355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1394778490066528,
      "rewards/margins": -0.5708999633789062,
      "rewards/rejected": -0.568577766418457,
      "step": 1861
    },
    {
      "epoch": 0.24365747934898177,
      "grad_norm": 7.198662757873535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2568094730377197,
      "logits/rejected": 0.1200655996799469,
      "logps/chosen": -607.5545043945312,
      "logps/rejected": -498.0002136230469,
      "loss": 0.5717,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2874199151992798,
      "rewards/margins": 0.39772433042526245,
      "rewards/rejected": -0.6851441860198975,
      "step": 1862
    },
    {
      "epoch": 0.24378833728633353,
      "grad_norm": 7.555882453918457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03698362782597542,
      "logits/rejected": 0.05406872555613518,
      "logps/chosen": -540.4921264648438,
      "logps/rejected": -487.5988464355469,
      "loss": 0.5554,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3245750367641449,
      "rewards/margins": 0.5923025608062744,
      "rewards/rejected": -0.9168775677680969,
      "step": 1863
    },
    {
      "epoch": 0.2439191952236853,
      "grad_norm": 7.585772514343262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06143069267272949,
      "logits/rejected": 0.1863173395395279,
      "logps/chosen": -278.13909912109375,
      "logps/rejected": -316.3822021484375,
      "loss": 0.7242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4926601052284241,
      "rewards/margins": 0.03992480784654617,
      "rewards/rejected": -0.5325849652290344,
      "step": 1864
    },
    {
      "epoch": 0.24405005316103706,
      "grad_norm": 7.392582416534424,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18493705987930298,
      "logits/rejected": -0.17774900794029236,
      "logps/chosen": -377.6411437988281,
      "logps/rejected": -541.794921875,
      "loss": 0.5411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2959146499633789,
      "rewards/margins": 0.6169917583465576,
      "rewards/rejected": -0.9129064679145813,
      "step": 1865
    },
    {
      "epoch": 0.24418091109838883,
      "grad_norm": 5.295485496520996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032189734280109406,
      "logits/rejected": -0.009247131645679474,
      "logps/chosen": -455.1824645996094,
      "logps/rejected": -273.17755126953125,
      "loss": 0.4281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28904402256011963,
      "rewards/margins": 1.0930830240249634,
      "rewards/rejected": -1.3821271657943726,
      "step": 1866
    },
    {
      "epoch": 0.24431176903574056,
      "grad_norm": 8.277679443359375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10835611820220947,
      "logits/rejected": -0.4364267587661743,
      "logps/chosen": -461.0081787109375,
      "logps/rejected": -387.7856140136719,
      "loss": 0.8216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43538743257522583,
      "rewards/margins": 0.3216167092323303,
      "rewards/rejected": -0.7570041418075562,
      "step": 1867
    },
    {
      "epoch": 0.24444262697309233,
      "grad_norm": 11.266273498535156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1949186772108078,
      "logits/rejected": 0.13480041921138763,
      "logps/chosen": -431.84942626953125,
      "logps/rejected": -636.30517578125,
      "loss": 1.0165,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6332107782363892,
      "rewards/margins": -0.10789874196052551,
      "rewards/rejected": -0.525312066078186,
      "step": 1868
    },
    {
      "epoch": 0.2445734849104441,
      "grad_norm": 8.972417831420898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.001984730362892151,
      "logits/rejected": 0.1710321605205536,
      "logps/chosen": -478.1554260253906,
      "logps/rejected": -469.0896911621094,
      "loss": 0.5604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6099058389663696,
      "rewards/margins": 1.4586708545684814,
      "rewards/rejected": -2.0685768127441406,
      "step": 1869
    },
    {
      "epoch": 0.24470434284779585,
      "grad_norm": 4.7338666915893555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14009208977222443,
      "logits/rejected": 0.2836756706237793,
      "logps/chosen": -527.7943725585938,
      "logps/rejected": -474.3564147949219,
      "loss": 0.4221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24248073995113373,
      "rewards/margins": 0.9793201684951782,
      "rewards/rejected": -1.2218008041381836,
      "step": 1870
    },
    {
      "epoch": 0.24483520078514762,
      "grad_norm": 6.4874091148376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020811211317777634,
      "logits/rejected": 0.23183667659759521,
      "logps/chosen": -341.99237060546875,
      "logps/rejected": -415.9200439453125,
      "loss": 0.6427,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1961076706647873,
      "rewards/margins": 0.7437524795532227,
      "rewards/rejected": -0.9398601651191711,
      "step": 1871
    },
    {
      "epoch": 0.24496605872249938,
      "grad_norm": 7.920316219329834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34751778841018677,
      "logits/rejected": 0.24506714940071106,
      "logps/chosen": -392.6294250488281,
      "logps/rejected": -377.70379638671875,
      "loss": 0.5034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17879194021224976,
      "rewards/margins": 0.9414852857589722,
      "rewards/rejected": -1.1202772855758667,
      "step": 1872
    },
    {
      "epoch": 0.24509691665985114,
      "grad_norm": 8.280141830444336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08114206045866013,
      "logits/rejected": 0.08287325501441956,
      "logps/chosen": -329.33331298828125,
      "logps/rejected": -397.9114990234375,
      "loss": 0.6952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4596642255783081,
      "rewards/margins": 0.832822859287262,
      "rewards/rejected": -1.2924871444702148,
      "step": 1873
    },
    {
      "epoch": 0.2452277745972029,
      "grad_norm": 5.106646537780762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13773906230926514,
      "logits/rejected": -0.12936778366565704,
      "logps/chosen": -488.4341125488281,
      "logps/rejected": -419.5417175292969,
      "loss": 0.3444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0759273767471313,
      "rewards/margins": 1.2319711446762085,
      "rewards/rejected": -2.30789852142334,
      "step": 1874
    },
    {
      "epoch": 0.24535863253455467,
      "grad_norm": 4.48076057434082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020214498043060303,
      "logits/rejected": 0.23900200426578522,
      "logps/chosen": -595.84130859375,
      "logps/rejected": -589.3194580078125,
      "loss": 0.201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14435946941375732,
      "rewards/margins": 2.3840112686157227,
      "rewards/rejected": -2.5283706188201904,
      "step": 1875
    },
    {
      "epoch": 0.24548949047190644,
      "grad_norm": 4.278270721435547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12273173034191132,
      "logits/rejected": 0.11756785959005356,
      "logps/chosen": -372.8656005859375,
      "logps/rejected": -372.1083984375,
      "loss": 0.3291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.554692804813385,
      "rewards/margins": 1.3320461511611938,
      "rewards/rejected": -1.8867390155792236,
      "step": 1876
    },
    {
      "epoch": 0.2456203484092582,
      "grad_norm": 15.395339965820312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3351331353187561,
      "logits/rejected": 0.01392071507871151,
      "logps/chosen": -400.8529052734375,
      "logps/rejected": -384.69549560546875,
      "loss": 0.9574,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38174137473106384,
      "rewards/margins": 0.7568957805633545,
      "rewards/rejected": -1.1386371850967407,
      "step": 1877
    },
    {
      "epoch": 0.24575120634660996,
      "grad_norm": 8.558606147766113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03772091865539551,
      "logits/rejected": 0.02667590230703354,
      "logps/chosen": -465.01025390625,
      "logps/rejected": -408.2452697753906,
      "loss": 0.518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26713716983795166,
      "rewards/margins": 0.7335398197174072,
      "rewards/rejected": -0.46640264987945557,
      "step": 1878
    },
    {
      "epoch": 0.24588206428396173,
      "grad_norm": 5.441729545593262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21988609433174133,
      "logits/rejected": 0.15035077929496765,
      "logps/chosen": -387.4775085449219,
      "logps/rejected": -424.510986328125,
      "loss": 0.3428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.214198499917984,
      "rewards/margins": 1.3710073232650757,
      "rewards/rejected": -1.5852057933807373,
      "step": 1879
    },
    {
      "epoch": 0.2460129222213135,
      "grad_norm": 9.133268356323242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15078997611999512,
      "logits/rejected": 0.02548031508922577,
      "logps/chosen": -365.2180480957031,
      "logps/rejected": -379.28936767578125,
      "loss": 0.6429,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4302143156528473,
      "rewards/margins": 0.26845818758010864,
      "rewards/rejected": -0.6986725330352783,
      "step": 1880
    },
    {
      "epoch": 0.24614378015866525,
      "grad_norm": 6.023519515991211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30772268772125244,
      "logits/rejected": 0.04607081413269043,
      "logps/chosen": -582.0682373046875,
      "logps/rejected": -481.7619323730469,
      "loss": 0.3932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10543003678321838,
      "rewards/margins": 1.0117584466934204,
      "rewards/rejected": -0.9063283205032349,
      "step": 1881
    },
    {
      "epoch": 0.24627463809601702,
      "grad_norm": 6.164307117462158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3366032540798187,
      "logits/rejected": 0.12135586142539978,
      "logps/chosen": -583.007568359375,
      "logps/rejected": -602.0240478515625,
      "loss": 0.5216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3755123019218445,
      "rewards/margins": 0.8678271770477295,
      "rewards/rejected": -0.492314875125885,
      "step": 1882
    },
    {
      "epoch": 0.24640549603336878,
      "grad_norm": 6.586010932922363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1656205654144287,
      "logits/rejected": -0.07195477187633514,
      "logps/chosen": -452.32269287109375,
      "logps/rejected": -311.88189697265625,
      "loss": 0.5609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42079266905784607,
      "rewards/margins": 0.334188848733902,
      "rewards/rejected": -0.754981517791748,
      "step": 1883
    },
    {
      "epoch": 0.24653635397072055,
      "grad_norm": 11.105666160583496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1154511570930481,
      "logits/rejected": -0.01455320417881012,
      "logps/chosen": -339.9163513183594,
      "logps/rejected": -316.8810119628906,
      "loss": 0.8049,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14307801425457,
      "rewards/margins": 0.30772367119789124,
      "rewards/rejected": -0.45080170035362244,
      "step": 1884
    },
    {
      "epoch": 0.2466672119080723,
      "grad_norm": 6.8484697341918945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25386106967926025,
      "logits/rejected": 0.2855411171913147,
      "logps/chosen": -474.83489990234375,
      "logps/rejected": -427.60748291015625,
      "loss": 0.5387,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.627113938331604,
      "rewards/margins": 1.04829740524292,
      "rewards/rejected": -0.42118340730667114,
      "step": 1885
    },
    {
      "epoch": 0.24679806984542407,
      "grad_norm": 11.39820384979248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1284908950328827,
      "logits/rejected": -0.04507450386881828,
      "logps/chosen": -447.55462646484375,
      "logps/rejected": -381.0567321777344,
      "loss": 1.1409,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0797314643859863,
      "rewards/margins": -0.5021615028381348,
      "rewards/rejected": -0.5775697827339172,
      "step": 1886
    },
    {
      "epoch": 0.24692892778277584,
      "grad_norm": 5.659786701202393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1118866354227066,
      "logits/rejected": 0.03601337596774101,
      "logps/chosen": -346.8507995605469,
      "logps/rejected": -341.8564147949219,
      "loss": 0.5002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.390471488237381,
      "rewards/margins": 0.5587932467460632,
      "rewards/rejected": -0.16832181811332703,
      "step": 1887
    },
    {
      "epoch": 0.24705978572012757,
      "grad_norm": 8.115144729614258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1642511636018753,
      "logits/rejected": 0.20359376072883606,
      "logps/chosen": -587.4339599609375,
      "logps/rejected": -573.1485595703125,
      "loss": 0.4893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3329378068447113,
      "rewards/margins": 0.8409314751625061,
      "rewards/rejected": -1.173869252204895,
      "step": 1888
    },
    {
      "epoch": 0.24719064365747934,
      "grad_norm": 5.410217761993408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4054984748363495,
      "logits/rejected": 0.1444733440876007,
      "logps/chosen": -638.6482543945312,
      "logps/rejected": -432.2501525878906,
      "loss": 0.3985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8168708086013794,
      "rewards/margins": 1.0323071479797363,
      "rewards/rejected": -0.2154361754655838,
      "step": 1889
    },
    {
      "epoch": 0.2473215015948311,
      "grad_norm": 3.9863359928131104,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04204344376921654,
      "logits/rejected": 0.20070883631706238,
      "logps/chosen": -474.9532165527344,
      "logps/rejected": -280.3952331542969,
      "loss": 0.3441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5323696136474609,
      "rewards/margins": 1.3813714981079102,
      "rewards/rejected": -0.8490018844604492,
      "step": 1890
    },
    {
      "epoch": 0.24745235953218286,
      "grad_norm": 6.832949161529541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25708794593811035,
      "logits/rejected": 0.34011927247047424,
      "logps/chosen": -602.9238891601562,
      "logps/rejected": -581.5653076171875,
      "loss": 0.5566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1810975968837738,
      "rewards/margins": 0.5669053792953491,
      "rewards/rejected": -0.3858078420162201,
      "step": 1891
    },
    {
      "epoch": 0.24758321746953463,
      "grad_norm": 7.790012836456299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2161668986082077,
      "logits/rejected": 0.37850189208984375,
      "logps/chosen": -384.6242980957031,
      "logps/rejected": -404.6485900878906,
      "loss": 0.5427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14251193404197693,
      "rewards/margins": 0.6602628827095032,
      "rewards/rejected": -0.5177509188652039,
      "step": 1892
    },
    {
      "epoch": 0.2477140754068864,
      "grad_norm": 7.450516700744629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10473943501710892,
      "logits/rejected": 0.11839410662651062,
      "logps/chosen": -549.2122802734375,
      "logps/rejected": -400.67919921875,
      "loss": 0.5237,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04794539511203766,
      "rewards/margins": 0.6633731722831726,
      "rewards/rejected": -0.7113186120986938,
      "step": 1893
    },
    {
      "epoch": 0.24784493334423816,
      "grad_norm": 7.682660102844238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16801117360591888,
      "logits/rejected": 0.12288358807563782,
      "logps/chosen": -441.1346435546875,
      "logps/rejected": -473.3437194824219,
      "loss": 0.5443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49375152587890625,
      "rewards/margins": 0.5404601693153381,
      "rewards/rejected": -1.0342117547988892,
      "step": 1894
    },
    {
      "epoch": 0.24797579128158992,
      "grad_norm": 7.128307819366455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1990835964679718,
      "logits/rejected": -0.05965226888656616,
      "logps/chosen": -510.12091064453125,
      "logps/rejected": -509.64544677734375,
      "loss": 0.3641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0043947286903858185,
      "rewards/margins": 1.5992640256881714,
      "rewards/rejected": -1.6036587953567505,
      "step": 1895
    },
    {
      "epoch": 0.24810664921894168,
      "grad_norm": 8.605487823486328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25963616371154785,
      "logits/rejected": 0.2393800914287567,
      "logps/chosen": -316.5801086425781,
      "logps/rejected": -424.8489990234375,
      "loss": 0.8964,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.64360511302948,
      "rewards/margins": -0.031863126903772354,
      "rewards/rejected": -0.6117420792579651,
      "step": 1896
    },
    {
      "epoch": 0.24823750715629345,
      "grad_norm": 6.222708225250244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3757058382034302,
      "logits/rejected": 0.40375879406929016,
      "logps/chosen": -443.9151611328125,
      "logps/rejected": -397.1222229003906,
      "loss": 0.5066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3590158224105835,
      "rewards/margins": 0.7553596496582031,
      "rewards/rejected": -0.39634382724761963,
      "step": 1897
    },
    {
      "epoch": 0.2483683650936452,
      "grad_norm": 6.54467248916626,
      "learning_rate": 5e-06,
      "logits/chosen": 0.337795615196228,
      "logits/rejected": 0.3739340901374817,
      "logps/chosen": -528.5114135742188,
      "logps/rejected": -449.9466552734375,
      "loss": 0.4149,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03609466925263405,
      "rewards/margins": 0.7703432440757751,
      "rewards/rejected": -0.8064379096031189,
      "step": 1898
    },
    {
      "epoch": 0.24849922303099697,
      "grad_norm": 7.826968669891357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2553405165672302,
      "logits/rejected": 0.21683776378631592,
      "logps/chosen": -330.9642639160156,
      "logps/rejected": -543.9775390625,
      "loss": 0.506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26680201292037964,
      "rewards/margins": 0.7074823975563049,
      "rewards/rejected": -0.9742844104766846,
      "step": 1899
    },
    {
      "epoch": 0.24863008096834874,
      "grad_norm": 5.822197437286377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05267488956451416,
      "logits/rejected": 0.0159734059125185,
      "logps/chosen": -351.0419616699219,
      "logps/rejected": -422.86651611328125,
      "loss": 0.4603,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4585472345352173,
      "rewards/margins": 0.649248480796814,
      "rewards/rejected": -1.1077955961227417,
      "step": 1900
    },
    {
      "epoch": 0.2487609389057005,
      "grad_norm": 7.018641948699951,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07991302758455276,
      "logits/rejected": -0.0987347662448883,
      "logps/chosen": -447.0157470703125,
      "logps/rejected": -390.02471923828125,
      "loss": 0.6121,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.39051973819732666,
      "rewards/margins": 0.38817620277404785,
      "rewards/rejected": 0.0023435577750205994,
      "step": 1901
    },
    {
      "epoch": 0.24889179684305227,
      "grad_norm": 5.923829078674316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23256221413612366,
      "logits/rejected": 0.2769232392311096,
      "logps/chosen": -577.5150756835938,
      "logps/rejected": -379.7691650390625,
      "loss": 0.5034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10628640651702881,
      "rewards/margins": 1.189007043838501,
      "rewards/rejected": -1.0827206373214722,
      "step": 1902
    },
    {
      "epoch": 0.24902265478040403,
      "grad_norm": 4.131098747253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33043429255485535,
      "logits/rejected": 0.3235737979412079,
      "logps/chosen": -529.553955078125,
      "logps/rejected": -482.91015625,
      "loss": 0.2641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08376865088939667,
      "rewards/margins": 1.5255776643753052,
      "rewards/rejected": -1.6093463897705078,
      "step": 1903
    },
    {
      "epoch": 0.2491535127177558,
      "grad_norm": 9.232624053955078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09976158291101456,
      "logits/rejected": 0.45890507102012634,
      "logps/chosen": -425.75067138671875,
      "logps/rejected": -441.60699462890625,
      "loss": 0.7957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18770523369312286,
      "rewards/margins": 0.39745137095451355,
      "rewards/rejected": -0.5851566195487976,
      "step": 1904
    },
    {
      "epoch": 0.24928437065510756,
      "grad_norm": 7.315028667449951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009499900043010712,
      "logits/rejected": 0.08975964784622192,
      "logps/chosen": -551.2575073242188,
      "logps/rejected": -433.98309326171875,
      "loss": 0.4888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7477715611457825,
      "rewards/margins": 0.7385740280151367,
      "rewards/rejected": -1.4863457679748535,
      "step": 1905
    },
    {
      "epoch": 0.24941522859245932,
      "grad_norm": 7.375485420227051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17326140403747559,
      "logits/rejected": 0.04355229064822197,
      "logps/chosen": -453.3370361328125,
      "logps/rejected": -435.35284423828125,
      "loss": 0.4866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.26715052127838135,
      "rewards/margins": 0.804888129234314,
      "rewards/rejected": -0.5377376675605774,
      "step": 1906
    },
    {
      "epoch": 0.24954608652981108,
      "grad_norm": 7.367903232574463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2983352541923523,
      "logits/rejected": 0.08569811284542084,
      "logps/chosen": -405.0480041503906,
      "logps/rejected": -498.5589294433594,
      "loss": 0.5321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2208196520805359,
      "rewards/margins": 0.6066626310348511,
      "rewards/rejected": -0.8274822235107422,
      "step": 1907
    },
    {
      "epoch": 0.24967694446716285,
      "grad_norm": 7.3364410400390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23333384096622467,
      "logits/rejected": 0.3162989020347595,
      "logps/chosen": -657.532958984375,
      "logps/rejected": -469.1929016113281,
      "loss": 0.6872,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.6184452772140503,
      "rewards/margins": 0.22449341416358948,
      "rewards/rejected": 0.39395180344581604,
      "step": 1908
    },
    {
      "epoch": 0.2498078024045146,
      "grad_norm": 7.407931327819824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2954677641391754,
      "logits/rejected": 0.3704575002193451,
      "logps/chosen": -424.4606628417969,
      "logps/rejected": -404.42633056640625,
      "loss": 0.7328,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15460146963596344,
      "rewards/margins": 0.219166561961174,
      "rewards/rejected": -0.06456509232521057,
      "step": 1909
    },
    {
      "epoch": 0.24993866034186635,
      "grad_norm": 5.111820697784424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0692259892821312,
      "logits/rejected": 0.016374405473470688,
      "logps/chosen": -763.11474609375,
      "logps/rejected": -508.6087646484375,
      "loss": 0.3343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6964866518974304,
      "rewards/margins": 1.340026617050171,
      "rewards/rejected": -0.6435399651527405,
      "step": 1910
    },
    {
      "epoch": 0.25006951827921814,
      "grad_norm": 7.042191982269287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06187565624713898,
      "logits/rejected": 0.015081614255905151,
      "logps/chosen": -470.71783447265625,
      "logps/rejected": -458.00726318359375,
      "loss": 0.3274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003304317593574524,
      "rewards/margins": 1.3616443872451782,
      "rewards/rejected": -1.3649487495422363,
      "step": 1911
    },
    {
      "epoch": 0.2502003762165699,
      "grad_norm": 9.287501335144043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2173471450805664,
      "logits/rejected": 0.28994128108024597,
      "logps/chosen": -442.5657958984375,
      "logps/rejected": -474.9014892578125,
      "loss": 0.795,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5385721325874329,
      "rewards/margins": 0.01745755970478058,
      "rewards/rejected": -0.5560296773910522,
      "step": 1912
    },
    {
      "epoch": 0.25033123415392167,
      "grad_norm": 7.281121253967285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08330041915178299,
      "logits/rejected": 0.1222953349351883,
      "logps/chosen": -188.6987762451172,
      "logps/rejected": -318.5722961425781,
      "loss": 0.5754,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012105170637369156,
      "rewards/margins": 0.7952863574028015,
      "rewards/rejected": -0.8073914647102356,
      "step": 1913
    },
    {
      "epoch": 0.2504620920912734,
      "grad_norm": 5.434911251068115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13041403889656067,
      "logits/rejected": -0.050801221281290054,
      "logps/chosen": -496.8221435546875,
      "logps/rejected": -468.0839538574219,
      "loss": 0.4319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21706236898899078,
      "rewards/margins": 1.4618644714355469,
      "rewards/rejected": -1.2448021173477173,
      "step": 1914
    },
    {
      "epoch": 0.2505929500286252,
      "grad_norm": 9.491715431213379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16026757657527924,
      "logits/rejected": 0.19919420778751373,
      "logps/chosen": -490.0459899902344,
      "logps/rejected": -386.82000732421875,
      "loss": 0.7659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.299681693315506,
      "rewards/margins": 0.13858908414840698,
      "rewards/rejected": -0.4382707476615906,
      "step": 1915
    },
    {
      "epoch": 0.25072380796597693,
      "grad_norm": 10.101786613464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02291185036301613,
      "logits/rejected": 0.04960322380065918,
      "logps/chosen": -548.59716796875,
      "logps/rejected": -442.0684814453125,
      "loss": 0.9099,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14510920643806458,
      "rewards/margins": -0.20477870106697083,
      "rewards/rejected": 0.3498879075050354,
      "step": 1916
    },
    {
      "epoch": 0.2508546659033287,
      "grad_norm": 10.918673515319824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13923336565494537,
      "logits/rejected": -0.01222756877541542,
      "logps/chosen": -526.0694580078125,
      "logps/rejected": -388.1868896484375,
      "loss": 0.6581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.080671787261963,
      "rewards/margins": 0.3764146566390991,
      "rewards/rejected": -1.4570865631103516,
      "step": 1917
    },
    {
      "epoch": 0.25098552384068046,
      "grad_norm": 7.532867908477783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27281612157821655,
      "logits/rejected": 0.17199769616127014,
      "logps/chosen": -418.15252685546875,
      "logps/rejected": -436.39013671875,
      "loss": 0.6277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16474716365337372,
      "rewards/margins": 0.4308961033821106,
      "rewards/rejected": -0.5956432819366455,
      "step": 1918
    },
    {
      "epoch": 0.25111638177803225,
      "grad_norm": 8.603013038635254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2126217633485794,
      "logits/rejected": 0.15275639295578003,
      "logps/chosen": -325.4764709472656,
      "logps/rejected": -323.87713623046875,
      "loss": 0.7636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5103240609169006,
      "rewards/margins": 0.14023153483867645,
      "rewards/rejected": -0.6505555510520935,
      "step": 1919
    },
    {
      "epoch": 0.251247239715384,
      "grad_norm": 9.718472480773926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2551287114620209,
      "logits/rejected": 0.25475892424583435,
      "logps/chosen": -390.3644714355469,
      "logps/rejected": -339.54559326171875,
      "loss": 0.713,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5802295804023743,
      "rewards/margins": 0.8450084924697876,
      "rewards/rejected": -1.4252381324768066,
      "step": 1920
    },
    {
      "epoch": 0.2513780976527357,
      "grad_norm": 6.590003490447998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06756700575351715,
      "logits/rejected": -0.01956411451101303,
      "logps/chosen": -419.676513671875,
      "logps/rejected": -339.0054016113281,
      "loss": 0.4198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18176786601543427,
      "rewards/margins": 1.522381067276001,
      "rewards/rejected": -1.7041490077972412,
      "step": 1921
    },
    {
      "epoch": 0.2515089555900875,
      "grad_norm": 7.125607490539551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10437275469303131,
      "logits/rejected": -0.05862198770046234,
      "logps/chosen": -424.7548522949219,
      "logps/rejected": -331.20965576171875,
      "loss": 0.4429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07073019444942474,
      "rewards/margins": 1.1096785068511963,
      "rewards/rejected": -1.0389482975006104,
      "step": 1922
    },
    {
      "epoch": 0.25163981352743925,
      "grad_norm": 7.192620277404785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2781476080417633,
      "logits/rejected": 0.19632920622825623,
      "logps/chosen": -585.1990356445312,
      "logps/rejected": -474.76055908203125,
      "loss": 0.6407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8317902088165283,
      "rewards/margins": 0.714006245136261,
      "rewards/rejected": 0.11778394877910614,
      "step": 1923
    },
    {
      "epoch": 0.25177067146479104,
      "grad_norm": 5.376522541046143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20653200149536133,
      "logits/rejected": 0.31552523374557495,
      "logps/chosen": -374.9427185058594,
      "logps/rejected": -354.21246337890625,
      "loss": 0.4297,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1936851441860199,
      "rewards/margins": 1.0790475606918335,
      "rewards/rejected": -0.885362446308136,
      "step": 1924
    },
    {
      "epoch": 0.2519015294021428,
      "grad_norm": 9.346216201782227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010236524045467377,
      "logits/rejected": 0.11592400074005127,
      "logps/chosen": -515.5094604492188,
      "logps/rejected": -641.5335693359375,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2406117022037506,
      "rewards/margins": 0.6973270177841187,
      "rewards/rejected": -0.9379386901855469,
      "step": 1925
    },
    {
      "epoch": 0.25203238733949457,
      "grad_norm": 8.23636245727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25846028327941895,
      "logits/rejected": 0.3792679011821747,
      "logps/chosen": -544.7794799804688,
      "logps/rejected": -584.6630859375,
      "loss": 0.6184,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7821640372276306,
      "rewards/margins": 0.39942967891693115,
      "rewards/rejected": -1.181593656539917,
      "step": 1926
    },
    {
      "epoch": 0.2521632452768463,
      "grad_norm": 5.157953262329102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25861766934394836,
      "logits/rejected": 0.052778713405132294,
      "logps/chosen": -463.3856201171875,
      "logps/rejected": -453.8655090332031,
      "loss": 0.3368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37967777252197266,
      "rewards/margins": 1.1629326343536377,
      "rewards/rejected": -0.7832547426223755,
      "step": 1927
    },
    {
      "epoch": 0.2522941032141981,
      "grad_norm": 5.032327651977539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14870601892471313,
      "logits/rejected": -0.013028380461037159,
      "logps/chosen": -519.4867553710938,
      "logps/rejected": -796.28662109375,
      "loss": 0.3845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21608027815818787,
      "rewards/margins": 1.7367970943450928,
      "rewards/rejected": -1.520716905593872,
      "step": 1928
    },
    {
      "epoch": 0.25242496115154983,
      "grad_norm": 4.994264602661133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2931167781352997,
      "logits/rejected": 0.2723516821861267,
      "logps/chosen": -503.823974609375,
      "logps/rejected": -445.5560607910156,
      "loss": 0.509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6144891977310181,
      "rewards/margins": 0.6783713102340698,
      "rewards/rejected": -0.06388205289840698,
      "step": 1929
    },
    {
      "epoch": 0.2525558190889016,
      "grad_norm": 6.41961669921875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00035196542739868164,
      "logits/rejected": -0.09420448541641235,
      "logps/chosen": -539.0,
      "logps/rejected": -528.5169677734375,
      "loss": 0.3628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4187345504760742,
      "rewards/margins": 0.8962914943695068,
      "rewards/rejected": -1.3150261640548706,
      "step": 1930
    },
    {
      "epoch": 0.25268667702625336,
      "grad_norm": 3.8834540843963623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2078300416469574,
      "logits/rejected": -0.22750982642173767,
      "logps/chosen": -515.4150390625,
      "logps/rejected": -484.68597412109375,
      "loss": 0.3242,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04966774582862854,
      "rewards/margins": 1.2575511932373047,
      "rewards/rejected": -1.2078834772109985,
      "step": 1931
    },
    {
      "epoch": 0.25281753496360515,
      "grad_norm": 5.536272048950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4892871081829071,
      "logits/rejected": 0.5423194169998169,
      "logps/chosen": -282.5969543457031,
      "logps/rejected": -326.54693603515625,
      "loss": 0.5064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45073145627975464,
      "rewards/margins": 0.65623539686203,
      "rewards/rejected": -1.1069667339324951,
      "step": 1932
    },
    {
      "epoch": 0.2529483929009569,
      "grad_norm": 6.444413185119629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09356459975242615,
      "logits/rejected": 0.186725914478302,
      "logps/chosen": -551.9743041992188,
      "logps/rejected": -673.4761962890625,
      "loss": 0.4605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05253464728593826,
      "rewards/margins": 0.8307002782821655,
      "rewards/rejected": -0.7781656384468079,
      "step": 1933
    },
    {
      "epoch": 0.2530792508383087,
      "grad_norm": 7.084273338317871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053127825260162354,
      "logits/rejected": 0.05042999982833862,
      "logps/chosen": -721.087646484375,
      "logps/rejected": -370.1834411621094,
      "loss": 0.4667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.029837042093276978,
      "rewards/margins": 0.7981386184692383,
      "rewards/rejected": -0.8279756903648376,
      "step": 1934
    },
    {
      "epoch": 0.2532101087756604,
      "grad_norm": 7.315052509307861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12427634745836258,
      "logits/rejected": 0.08704245090484619,
      "logps/chosen": -360.8142395019531,
      "logps/rejected": -370.8498229980469,
      "loss": 0.8072,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03634569048881531,
      "rewards/margins": -0.15408191084861755,
      "rewards/rejected": 0.11773624271154404,
      "step": 1935
    },
    {
      "epoch": 0.2533409667130122,
      "grad_norm": 8.950925827026367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29359981417655945,
      "logits/rejected": 0.32080313563346863,
      "logps/chosen": -572.1014404296875,
      "logps/rejected": -658.746826171875,
      "loss": 0.8175,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.44328880310058594,
      "rewards/margins": 0.08098982274532318,
      "rewards/rejected": -0.5242786407470703,
      "step": 1936
    },
    {
      "epoch": 0.25347182465036394,
      "grad_norm": 7.108871936798096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15352763235569,
      "logits/rejected": 0.07510111480951309,
      "logps/chosen": -528.3215942382812,
      "logps/rejected": -500.55560302734375,
      "loss": 0.4867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1390228122472763,
      "rewards/margins": 0.7712442278862,
      "rewards/rejected": -0.6322214007377625,
      "step": 1937
    },
    {
      "epoch": 0.25360268258771573,
      "grad_norm": 6.4176836013793945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12549951672554016,
      "logits/rejected": -0.13373683393001556,
      "logps/chosen": -465.15838623046875,
      "logps/rejected": -530.2693481445312,
      "loss": 0.4965,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11670820415019989,
      "rewards/margins": 0.7104389071464539,
      "rewards/rejected": -0.8271470665931702,
      "step": 1938
    },
    {
      "epoch": 0.25373354052506747,
      "grad_norm": 3.260303497314453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11522352695465088,
      "logits/rejected": -0.06424610316753387,
      "logps/chosen": -606.44775390625,
      "logps/rejected": -443.96783447265625,
      "loss": 0.2703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2992105484008789,
      "rewards/margins": 1.5565171241760254,
      "rewards/rejected": -1.257306456565857,
      "step": 1939
    },
    {
      "epoch": 0.25386439846241926,
      "grad_norm": 5.694022178649902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20707522332668304,
      "logits/rejected": 0.2993364930152893,
      "logps/chosen": -887.4744262695312,
      "logps/rejected": -703.3357543945312,
      "loss": 0.3949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.756464958190918,
      "rewards/margins": 1.2891383171081543,
      "rewards/rejected": -0.5326732397079468,
      "step": 1940
    },
    {
      "epoch": 0.253995256399771,
      "grad_norm": 9.444899559020996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20539778470993042,
      "logits/rejected": 0.09725731611251831,
      "logps/chosen": -565.8521118164062,
      "logps/rejected": -472.93017578125,
      "loss": 0.5692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3420867621898651,
      "rewards/margins": 0.8371454477310181,
      "rewards/rejected": -1.1792322397232056,
      "step": 1941
    },
    {
      "epoch": 0.2541261143371228,
      "grad_norm": 8.807626724243164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03767254948616028,
      "logits/rejected": -0.00394841656088829,
      "logps/chosen": -451.81689453125,
      "logps/rejected": -384.7209167480469,
      "loss": 0.7222,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031148940324783325,
      "rewards/margins": 0.5506858825683594,
      "rewards/rejected": -0.5818347930908203,
      "step": 1942
    },
    {
      "epoch": 0.2542569722744745,
      "grad_norm": 7.156095504760742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05934334546327591,
      "logits/rejected": -0.05682923644781113,
      "logps/chosen": -294.5582275390625,
      "logps/rejected": -384.6278076171875,
      "loss": 0.5677,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0967789888381958,
      "rewards/margins": 0.577366828918457,
      "rewards/rejected": -1.6741458177566528,
      "step": 1943
    },
    {
      "epoch": 0.25438783021182626,
      "grad_norm": 8.143036842346191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12014241516590118,
      "logits/rejected": 0.10137274861335754,
      "logps/chosen": -439.6309509277344,
      "logps/rejected": -438.4414978027344,
      "loss": 0.6286,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08959904313087463,
      "rewards/margins": 0.3389839231967926,
      "rewards/rejected": -0.4285829961299896,
      "step": 1944
    },
    {
      "epoch": 0.25451868814917805,
      "grad_norm": 5.142815113067627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.050975993275642395,
      "logits/rejected": 0.003331143409013748,
      "logps/chosen": -410.99993896484375,
      "logps/rejected": -337.06866455078125,
      "loss": 0.4373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3837585747241974,
      "rewards/margins": 0.889461100101471,
      "rewards/rejected": -0.5057026147842407,
      "step": 1945
    },
    {
      "epoch": 0.2546495460865298,
      "grad_norm": 10.423094749450684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17798787355422974,
      "logits/rejected": -0.032209161669015884,
      "logps/chosen": -639.9158325195312,
      "logps/rejected": -550.8758544921875,
      "loss": 0.6178,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07732052356004715,
      "rewards/margins": 0.40286409854888916,
      "rewards/rejected": -0.325543612241745,
      "step": 1946
    },
    {
      "epoch": 0.2547804040238816,
      "grad_norm": 4.152776718139648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22402073442935944,
      "logits/rejected": 0.1492479145526886,
      "logps/chosen": -469.68914794921875,
      "logps/rejected": -466.5748291015625,
      "loss": 0.3074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08696328103542328,
      "rewards/margins": 1.5300959348678589,
      "rewards/rejected": -1.6170591115951538,
      "step": 1947
    },
    {
      "epoch": 0.2549112619612333,
      "grad_norm": 6.774503707885742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2598828375339508,
      "logits/rejected": 0.14569219946861267,
      "logps/chosen": -471.1307067871094,
      "logps/rejected": -456.9672546386719,
      "loss": 0.4423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15885275602340698,
      "rewards/margins": 1.2397222518920898,
      "rewards/rejected": -1.080869436264038,
      "step": 1948
    },
    {
      "epoch": 0.2550421198985851,
      "grad_norm": 4.475090026855469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3306906521320343,
      "logits/rejected": 0.21535640954971313,
      "logps/chosen": -678.3369750976562,
      "logps/rejected": -758.3560180664062,
      "loss": 0.2692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25653076171875,
      "rewards/margins": 1.6470630168914795,
      "rewards/rejected": -1.3905322551727295,
      "step": 1949
    },
    {
      "epoch": 0.25517297783593684,
      "grad_norm": 6.7177581787109375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1541214883327484,
      "logits/rejected": -0.06403796374797821,
      "logps/chosen": -486.20208740234375,
      "logps/rejected": -302.3994445800781,
      "loss": 0.5508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.911106526851654,
      "rewards/margins": 0.573622465133667,
      "rewards/rejected": -1.4847289323806763,
      "step": 1950
    },
    {
      "epoch": 0.25530383577328863,
      "grad_norm": 4.823801040649414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04973233863711357,
      "logits/rejected": -0.05589543282985687,
      "logps/chosen": -472.16485595703125,
      "logps/rejected": -385.816650390625,
      "loss": 0.343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2076890915632248,
      "rewards/margins": 1.6786212921142578,
      "rewards/rejected": -1.4709322452545166,
      "step": 1951
    },
    {
      "epoch": 0.25543469371064037,
      "grad_norm": 6.484559059143066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19937539100646973,
      "logits/rejected": -0.04889288544654846,
      "logps/chosen": -472.56072998046875,
      "logps/rejected": -510.1966552734375,
      "loss": 0.6612,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33260083198547363,
      "rewards/margins": 0.6463254690170288,
      "rewards/rejected": -0.9789263606071472,
      "step": 1952
    },
    {
      "epoch": 0.25556555164799216,
      "grad_norm": 5.73172664642334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006620585918426514,
      "logits/rejected": -0.12604814767837524,
      "logps/chosen": -759.3494873046875,
      "logps/rejected": -457.580322265625,
      "loss": 0.4446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5813747644424438,
      "rewards/margins": 0.8155340552330017,
      "rewards/rejected": -0.23415929079055786,
      "step": 1953
    },
    {
      "epoch": 0.2556964095853439,
      "grad_norm": 8.84721851348877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22081229090690613,
      "logits/rejected": -0.3328494727611542,
      "logps/chosen": -635.7420043945312,
      "logps/rejected": -594.0473022460938,
      "loss": 0.4882,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29982471466064453,
      "rewards/margins": 1.1147186756134033,
      "rewards/rejected": -1.4145435094833374,
      "step": 1954
    },
    {
      "epoch": 0.2558272675226957,
      "grad_norm": 7.605095386505127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2805916965007782,
      "logits/rejected": 0.2047043740749359,
      "logps/chosen": -388.5771179199219,
      "logps/rejected": -310.3060302734375,
      "loss": 0.5607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6596311330795288,
      "rewards/margins": 0.5320285558700562,
      "rewards/rejected": -1.191659688949585,
      "step": 1955
    },
    {
      "epoch": 0.2559581254600474,
      "grad_norm": 8.267027854919434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.307722806930542,
      "logits/rejected": 0.18119896948337555,
      "logps/chosen": -424.45562744140625,
      "logps/rejected": -391.66668701171875,
      "loss": 0.7009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27655187249183655,
      "rewards/margins": 0.6528248190879822,
      "rewards/rejected": -0.9293766617774963,
      "step": 1956
    },
    {
      "epoch": 0.2560889833973992,
      "grad_norm": 10.902874946594238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17933103442192078,
      "logits/rejected": 0.33186107873916626,
      "logps/chosen": -332.7101745605469,
      "logps/rejected": -346.2424011230469,
      "loss": 0.6423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3234897553920746,
      "rewards/margins": 0.37363630533218384,
      "rewards/rejected": -0.697126030921936,
      "step": 1957
    },
    {
      "epoch": 0.25621984133475095,
      "grad_norm": 10.869600296020508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09894958138465881,
      "logits/rejected": -0.1652214527130127,
      "logps/chosen": -559.8492431640625,
      "logps/rejected": -436.8681640625,
      "loss": 0.8713,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2154846042394638,
      "rewards/margins": 0.37132528424263,
      "rewards/rejected": -0.5868099331855774,
      "step": 1958
    },
    {
      "epoch": 0.25635069927210274,
      "grad_norm": 6.223357677459717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2526112198829651,
      "logits/rejected": -0.008020155131816864,
      "logps/chosen": -327.7449645996094,
      "logps/rejected": -336.28436279296875,
      "loss": 0.6918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08980941772460938,
      "rewards/margins": 0.44267770648002625,
      "rewards/rejected": -0.35286828875541687,
      "step": 1959
    },
    {
      "epoch": 0.2564815572094545,
      "grad_norm": 5.662090301513672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047358691692352295,
      "logits/rejected": 0.11523944139480591,
      "logps/chosen": -233.19728088378906,
      "logps/rejected": -260.49835205078125,
      "loss": 0.5759,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0724334716796875,
      "rewards/margins": 0.5310640335083008,
      "rewards/rejected": -0.4586305618286133,
      "step": 1960
    },
    {
      "epoch": 0.25661241514680627,
      "grad_norm": 7.65020751953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08766378462314606,
      "logits/rejected": 0.04800470173358917,
      "logps/chosen": -542.3312377929688,
      "logps/rejected": -429.48687744140625,
      "loss": 0.6631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.410159707069397,
      "rewards/margins": 0.43093129992485046,
      "rewards/rejected": -0.841090977191925,
      "step": 1961
    },
    {
      "epoch": 0.256743273084158,
      "grad_norm": 7.592011451721191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08419433236122131,
      "logits/rejected": -0.001295585185289383,
      "logps/chosen": -499.83099365234375,
      "logps/rejected": -325.6852722167969,
      "loss": 0.5858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011806666851043701,
      "rewards/margins": 0.8781287670135498,
      "rewards/rejected": -0.8899354934692383,
      "step": 1962
    },
    {
      "epoch": 0.2568741310215098,
      "grad_norm": 7.989944934844971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09645314514636993,
      "logits/rejected": -0.1316373348236084,
      "logps/chosen": -571.1678466796875,
      "logps/rejected": -674.4287719726562,
      "loss": 0.6304,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6709907054901123,
      "rewards/margins": 0.5457130670547485,
      "rewards/rejected": -1.2167037725448608,
      "step": 1963
    },
    {
      "epoch": 0.25700498895886154,
      "grad_norm": 5.329221248626709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008317983709275723,
      "logits/rejected": 0.03458210825920105,
      "logps/chosen": -641.0889282226562,
      "logps/rejected": -474.9163818359375,
      "loss": 0.3098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14680099487304688,
      "rewards/margins": 1.2710477113723755,
      "rewards/rejected": -1.4178487062454224,
      "step": 1964
    },
    {
      "epoch": 0.25713584689621327,
      "grad_norm": 5.637215614318848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3950069546699524,
      "logits/rejected": 0.4349345266819,
      "logps/chosen": -490.4466552734375,
      "logps/rejected": -476.2020263671875,
      "loss": 0.4524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5585903525352478,
      "rewards/margins": 1.049790859222412,
      "rewards/rejected": -1.6083812713623047,
      "step": 1965
    },
    {
      "epoch": 0.25726670483356506,
      "grad_norm": 14.090519905090332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21479225158691406,
      "logits/rejected": -0.27033382654190063,
      "logps/chosen": -518.8084716796875,
      "logps/rejected": -460.30609130859375,
      "loss": 0.626,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0086978673934937,
      "rewards/margins": 0.4217035174369812,
      "rewards/rejected": -1.4304014444351196,
      "step": 1966
    },
    {
      "epoch": 0.2573975627709168,
      "grad_norm": 4.894988536834717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10771482437849045,
      "logits/rejected": -0.10106191039085388,
      "logps/chosen": -373.7518310546875,
      "logps/rejected": -590.160888671875,
      "loss": 0.376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.058581553399562836,
      "rewards/margins": 1.4065052270889282,
      "rewards/rejected": -1.4650866985321045,
      "step": 1967
    },
    {
      "epoch": 0.2575284207082686,
      "grad_norm": 5.86978006362915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12385952472686768,
      "logits/rejected": 0.19144591689109802,
      "logps/chosen": -421.62603759765625,
      "logps/rejected": -458.4862365722656,
      "loss": 0.5022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6420555114746094,
      "rewards/margins": 0.6579734683036804,
      "rewards/rejected": -1.300028920173645,
      "step": 1968
    },
    {
      "epoch": 0.2576592786456203,
      "grad_norm": 6.7603044509887695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18219934403896332,
      "logits/rejected": 0.27410078048706055,
      "logps/chosen": -488.34759521484375,
      "logps/rejected": -568.56884765625,
      "loss": 0.4881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0641479417681694,
      "rewards/margins": 0.9702843427658081,
      "rewards/rejected": -0.9061362743377686,
      "step": 1969
    },
    {
      "epoch": 0.2577901365829721,
      "grad_norm": 6.501230716705322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11141133308410645,
      "logits/rejected": -0.19385989010334015,
      "logps/chosen": -431.2061767578125,
      "logps/rejected": -659.7462158203125,
      "loss": 0.5798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9872963428497314,
      "rewards/margins": 0.995938777923584,
      "rewards/rejected": -1.9832348823547363,
      "step": 1970
    },
    {
      "epoch": 0.25792099452032385,
      "grad_norm": 7.7396745681762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20897012948989868,
      "logits/rejected": 0.5080400109291077,
      "logps/chosen": -404.2508850097656,
      "logps/rejected": -502.17291259765625,
      "loss": 0.6851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19402921199798584,
      "rewards/margins": 0.5541359186172485,
      "rewards/rejected": -0.7481651306152344,
      "step": 1971
    },
    {
      "epoch": 0.25805185245767565,
      "grad_norm": 6.087673664093018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10434792190790176,
      "logits/rejected": -0.14524498581886292,
      "logps/chosen": -488.5419616699219,
      "logps/rejected": -359.382080078125,
      "loss": 0.5521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13549023866653442,
      "rewards/margins": 0.5197778940200806,
      "rewards/rejected": -0.38428762555122375,
      "step": 1972
    },
    {
      "epoch": 0.2581827103950274,
      "grad_norm": 5.5444016456604,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24171380698680878,
      "logits/rejected": 0.0348096527159214,
      "logps/chosen": -511.4976806640625,
      "logps/rejected": -407.3433837890625,
      "loss": 0.4305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03093355894088745,
      "rewards/margins": 0.9373161196708679,
      "rewards/rejected": -0.9682496786117554,
      "step": 1973
    },
    {
      "epoch": 0.2583135683323792,
      "grad_norm": 8.002017974853516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4342091679573059,
      "logits/rejected": 0.23731334507465363,
      "logps/chosen": -483.4139099121094,
      "logps/rejected": -516.987548828125,
      "loss": 0.5414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5645191073417664,
      "rewards/margins": 0.8243885040283203,
      "rewards/rejected": -1.3889076709747314,
      "step": 1974
    },
    {
      "epoch": 0.2584444262697309,
      "grad_norm": 5.114572525024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07385309040546417,
      "logits/rejected": 0.11151442676782608,
      "logps/chosen": -355.4234924316406,
      "logps/rejected": -394.475341796875,
      "loss": 0.3889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1340869963169098,
      "rewards/margins": 1.024741291999817,
      "rewards/rejected": -1.1588283777236938,
      "step": 1975
    },
    {
      "epoch": 0.2585752842070827,
      "grad_norm": 9.77030086517334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24237173795700073,
      "logits/rejected": 0.08911173790693283,
      "logps/chosen": -568.013916015625,
      "logps/rejected": -501.06317138671875,
      "loss": 0.5688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2896079421043396,
      "rewards/margins": 0.6899479627609253,
      "rewards/rejected": -0.9795560240745544,
      "step": 1976
    },
    {
      "epoch": 0.25870614214443444,
      "grad_norm": 9.28588581085205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2072644829750061,
      "logits/rejected": 0.24633938074111938,
      "logps/chosen": -534.1574096679688,
      "logps/rejected": -493.15277099609375,
      "loss": 0.7556,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16041868925094604,
      "rewards/margins": 0.09423981606960297,
      "rewards/rejected": -0.2546585202217102,
      "step": 1977
    },
    {
      "epoch": 0.25883700008178623,
      "grad_norm": 9.6798095703125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1111007034778595,
      "logits/rejected": 0.06184755265712738,
      "logps/chosen": -360.4488220214844,
      "logps/rejected": -433.3238525390625,
      "loss": 1.1887,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8512060046195984,
      "rewards/margins": -0.19546715915203094,
      "rewards/rejected": -0.655738890171051,
      "step": 1978
    },
    {
      "epoch": 0.25896785801913796,
      "grad_norm": 4.084299564361572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3268902599811554,
      "logits/rejected": 0.34527474641799927,
      "logps/chosen": -473.0269470214844,
      "logps/rejected": -372.0800476074219,
      "loss": 0.4435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.010921865701675415,
      "rewards/margins": 0.9262848496437073,
      "rewards/rejected": -0.9372066855430603,
      "step": 1979
    },
    {
      "epoch": 0.25909871595648976,
      "grad_norm": 7.062231540679932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39505064487457275,
      "logits/rejected": 0.22258049249649048,
      "logps/chosen": -400.1439514160156,
      "logps/rejected": -411.66815185546875,
      "loss": 0.4731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48443320393562317,
      "rewards/margins": 0.873653769493103,
      "rewards/rejected": -1.3580869436264038,
      "step": 1980
    },
    {
      "epoch": 0.2592295738938415,
      "grad_norm": 7.031373023986816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19854648411273956,
      "logits/rejected": 0.31520751118659973,
      "logps/chosen": -342.3753662109375,
      "logps/rejected": -505.1413269042969,
      "loss": 0.7093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3121130168437958,
      "rewards/margins": 0.426701158285141,
      "rewards/rejected": -0.7388141751289368,
      "step": 1981
    },
    {
      "epoch": 0.2593604318311933,
      "grad_norm": 24.588106155395508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02147795259952545,
      "logits/rejected": -0.029145389795303345,
      "logps/chosen": -401.3684387207031,
      "logps/rejected": -530.9027099609375,
      "loss": 0.9289,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9575161337852478,
      "rewards/margins": 0.10462167859077454,
      "rewards/rejected": -1.0621378421783447,
      "step": 1982
    },
    {
      "epoch": 0.259491289768545,
      "grad_norm": 6.135119438171387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029963456094264984,
      "logits/rejected": -0.13712862133979797,
      "logps/chosen": -513.8220825195312,
      "logps/rejected": -411.11669921875,
      "loss": 0.4163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0232192724943161,
      "rewards/margins": 1.1199718713760376,
      "rewards/rejected": -1.096752643585205,
      "step": 1983
    },
    {
      "epoch": 0.2596221477058968,
      "grad_norm": 5.832348346710205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5356400609016418,
      "logits/rejected": 0.5402442216873169,
      "logps/chosen": -464.3707275390625,
      "logps/rejected": -442.8603515625,
      "loss": 0.4928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1240127086639404,
      "rewards/margins": 0.6085703372955322,
      "rewards/rejected": -1.7325831651687622,
      "step": 1984
    },
    {
      "epoch": 0.25975300564324855,
      "grad_norm": 8.463191986083984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0829244926571846,
      "logits/rejected": 0.06002526730298996,
      "logps/chosen": -508.3926086425781,
      "logps/rejected": -429.01593017578125,
      "loss": 0.6777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1302328109741211,
      "rewards/margins": 0.32599204778671265,
      "rewards/rejected": -0.19575923681259155,
      "step": 1985
    },
    {
      "epoch": 0.25988386358060034,
      "grad_norm": 8.539238929748535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1651405394077301,
      "logits/rejected": 0.38554492592811584,
      "logps/chosen": -338.30035400390625,
      "logps/rejected": -372.7751159667969,
      "loss": 0.8284,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6496655941009521,
      "rewards/margins": 0.07934512197971344,
      "rewards/rejected": -0.7290107607841492,
      "step": 1986
    },
    {
      "epoch": 0.2600147215179521,
      "grad_norm": 6.157417297363281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09465790539979935,
      "logits/rejected": 0.14043308794498444,
      "logps/chosen": -294.0818786621094,
      "logps/rejected": -455.9978332519531,
      "loss": 0.4619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29771825671195984,
      "rewards/margins": 0.7610927820205688,
      "rewards/rejected": -1.0588109493255615,
      "step": 1987
    },
    {
      "epoch": 0.2601455794553038,
      "grad_norm": 6.6385016441345215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15974898636341095,
      "logits/rejected": 0.08022567629814148,
      "logps/chosen": -305.9112854003906,
      "logps/rejected": -313.28125,
      "loss": 0.7696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4621201753616333,
      "rewards/margins": 0.041610296815633774,
      "rewards/rejected": -1.5037304162979126,
      "step": 1988
    },
    {
      "epoch": 0.2602764373926556,
      "grad_norm": 4.19035530090332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3083003759384155,
      "logits/rejected": 0.24849475920200348,
      "logps/chosen": -442.5142822265625,
      "logps/rejected": -516.4063720703125,
      "loss": 0.3536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.183013916015625,
      "rewards/margins": 1.0564587116241455,
      "rewards/rejected": -0.873444676399231,
      "step": 1989
    },
    {
      "epoch": 0.26040729533000734,
      "grad_norm": 6.2679338455200195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010940957814455032,
      "logits/rejected": 0.015343096107244492,
      "logps/chosen": -274.13763427734375,
      "logps/rejected": -286.9327392578125,
      "loss": 0.506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18760834634304047,
      "rewards/margins": 0.8690227270126343,
      "rewards/rejected": -0.6814143657684326,
      "step": 1990
    },
    {
      "epoch": 0.26053815326735913,
      "grad_norm": 5.345463275909424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012432349845767021,
      "logits/rejected": -0.24584220349788666,
      "logps/chosen": -321.64605712890625,
      "logps/rejected": -323.54534912109375,
      "loss": 0.3157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47888490557670593,
      "rewards/margins": 1.0907690525054932,
      "rewards/rejected": -1.569653868675232,
      "step": 1991
    },
    {
      "epoch": 0.26066901120471087,
      "grad_norm": 7.278847694396973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1227034404873848,
      "logits/rejected": -0.004457153379917145,
      "logps/chosen": -515.815673828125,
      "logps/rejected": -608.634033203125,
      "loss": 0.5188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.42408275604248047,
      "rewards/margins": 0.9242390394210815,
      "rewards/rejected": -0.5001562237739563,
      "step": 1992
    },
    {
      "epoch": 0.26079986914206266,
      "grad_norm": 6.7098236083984375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007218163460493088,
      "logits/rejected": -0.019658586010336876,
      "logps/chosen": -446.6780090332031,
      "logps/rejected": -499.703369140625,
      "loss": 0.4827,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07663267850875854,
      "rewards/margins": 1.0447802543640137,
      "rewards/rejected": -1.121412992477417,
      "step": 1993
    },
    {
      "epoch": 0.2609307270794144,
      "grad_norm": 5.061089992523193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09495609998703003,
      "logits/rejected": 0.03877027705311775,
      "logps/chosen": -411.9654846191406,
      "logps/rejected": -304.1069030761719,
      "loss": 0.3995,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2696630358695984,
      "rewards/margins": 0.9134389758110046,
      "rewards/rejected": -1.183102011680603,
      "step": 1994
    },
    {
      "epoch": 0.2610615850167662,
      "grad_norm": 6.685149669647217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1963406503200531,
      "logits/rejected": 0.025678982958197594,
      "logps/chosen": -323.6439514160156,
      "logps/rejected": -340.3149719238281,
      "loss": 0.5791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.054470419883728,
      "rewards/margins": 0.35530030727386475,
      "rewards/rejected": -1.4097707271575928,
      "step": 1995
    },
    {
      "epoch": 0.2611924429541179,
      "grad_norm": 7.560620307922363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12079429626464844,
      "logits/rejected": -0.10205037146806717,
      "logps/chosen": -337.3313903808594,
      "logps/rejected": -283.4551696777344,
      "loss": 0.7707,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9773740768432617,
      "rewards/margins": 0.006733305752277374,
      "rewards/rejected": -0.9841074347496033,
      "step": 1996
    },
    {
      "epoch": 0.2613233008914697,
      "grad_norm": 12.915299415588379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19679808616638184,
      "logits/rejected": -0.10545489937067032,
      "logps/chosen": -580.4933471679688,
      "logps/rejected": -483.0838317871094,
      "loss": 0.6215,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.322571337223053,
      "rewards/margins": 0.7735939025878906,
      "rewards/rejected": -1.0961652994155884,
      "step": 1997
    },
    {
      "epoch": 0.26145415882882145,
      "grad_norm": 6.5260467529296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027780525386333466,
      "logits/rejected": -0.013559962622821331,
      "logps/chosen": -228.88174438476562,
      "logps/rejected": -224.09710693359375,
      "loss": 0.7406,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5808876156806946,
      "rewards/margins": 0.18409116566181183,
      "rewards/rejected": -0.76497882604599,
      "step": 1998
    },
    {
      "epoch": 0.26158501676617324,
      "grad_norm": 6.638553142547607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28504717350006104,
      "logits/rejected": 0.0202789306640625,
      "logps/chosen": -352.8995056152344,
      "logps/rejected": -354.1177978515625,
      "loss": 0.4065,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5347729325294495,
      "rewards/margins": 0.8485467433929443,
      "rewards/rejected": -1.383319616317749,
      "step": 1999
    },
    {
      "epoch": 0.261715874703525,
      "grad_norm": 8.920734405517578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0621124766767025,
      "logits/rejected": -0.2557238042354584,
      "logps/chosen": -584.8968505859375,
      "logps/rejected": -1228.33544921875,
      "loss": 0.6294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35577505826950073,
      "rewards/margins": 0.2266109436750412,
      "rewards/rejected": -0.5823860168457031,
      "step": 2000
    },
    {
      "epoch": 0.26184673264087677,
      "grad_norm": 5.621856689453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03224750608205795,
      "logits/rejected": 0.0626482218503952,
      "logps/chosen": -274.75640869140625,
      "logps/rejected": -406.61541748046875,
      "loss": 0.4257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5771816372871399,
      "rewards/margins": 0.7097694873809814,
      "rewards/rejected": -1.2869510650634766,
      "step": 2001
    },
    {
      "epoch": 0.2619775905782285,
      "grad_norm": 11.02145767211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07535126805305481,
      "logits/rejected": 0.08646803349256516,
      "logps/chosen": -485.3399963378906,
      "logps/rejected": -525.045654296875,
      "loss": 0.9271,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.939277708530426,
      "rewards/margins": 0.06412729620933533,
      "rewards/rejected": -1.003404974937439,
      "step": 2002
    },
    {
      "epoch": 0.2621084485155803,
      "grad_norm": 7.219240665435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02159850299358368,
      "logits/rejected": -0.0673711895942688,
      "logps/chosen": -674.6705322265625,
      "logps/rejected": -472.8096923828125,
      "loss": 0.4142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3367668092250824,
      "rewards/margins": 1.1178981065750122,
      "rewards/rejected": -1.4546650648117065,
      "step": 2003
    },
    {
      "epoch": 0.26223930645293203,
      "grad_norm": 5.606327533721924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38244205713272095,
      "logits/rejected": 0.4231458008289337,
      "logps/chosen": -647.4255981445312,
      "logps/rejected": -674.3078002929688,
      "loss": 0.3697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21215955913066864,
      "rewards/margins": 1.3325251340866089,
      "rewards/rejected": -1.544684648513794,
      "step": 2004
    },
    {
      "epoch": 0.2623701643902838,
      "grad_norm": 6.499288558959961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06648674607276917,
      "logits/rejected": -0.08485518395900726,
      "logps/chosen": -481.54742431640625,
      "logps/rejected": -379.8992919921875,
      "loss": 0.4955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47956010699272156,
      "rewards/margins": 0.6156690120697021,
      "rewards/rejected": -1.095229148864746,
      "step": 2005
    },
    {
      "epoch": 0.26250102232763556,
      "grad_norm": 10.555997848510742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2509380280971527,
      "logits/rejected": 0.31726565957069397,
      "logps/chosen": -520.951904296875,
      "logps/rejected": -445.7942199707031,
      "loss": 1.0604,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.967074453830719,
      "rewards/margins": -0.18676941096782684,
      "rewards/rejected": -0.7803050875663757,
      "step": 2006
    },
    {
      "epoch": 0.26263188026498735,
      "grad_norm": 7.118230819702148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10803256183862686,
      "logits/rejected": -0.26153022050857544,
      "logps/chosen": -417.8286437988281,
      "logps/rejected": -527.7029418945312,
      "loss": 0.4139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7905445098876953,
      "rewards/margins": 1.085054636001587,
      "rewards/rejected": -1.8755991458892822,
      "step": 2007
    },
    {
      "epoch": 0.2627627382023391,
      "grad_norm": 4.894651889801025,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01699695736169815,
      "logits/rejected": -0.038446761667728424,
      "logps/chosen": -419.36358642578125,
      "logps/rejected": -307.36810302734375,
      "loss": 0.4234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7169147729873657,
      "rewards/margins": 0.9187362194061279,
      "rewards/rejected": -0.20182132720947266,
      "step": 2008
    },
    {
      "epoch": 0.2628935961396908,
      "grad_norm": 5.156802654266357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00893663614988327,
      "logits/rejected": -0.044425539672374725,
      "logps/chosen": -566.1815795898438,
      "logps/rejected": -597.1513671875,
      "loss": 0.2896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07511693239212036,
      "rewards/margins": 1.785466194152832,
      "rewards/rejected": -1.710349202156067,
      "step": 2009
    },
    {
      "epoch": 0.2630244540770426,
      "grad_norm": 4.884042263031006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03034459426999092,
      "logits/rejected": 0.07574392855167389,
      "logps/chosen": -443.92144775390625,
      "logps/rejected": -481.0894775390625,
      "loss": 0.3422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41081660985946655,
      "rewards/margins": 0.9876677989959717,
      "rewards/rejected": -1.3984843492507935,
      "step": 2010
    },
    {
      "epoch": 0.26315531201439435,
      "grad_norm": 3.6930320262908936,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01724442094564438,
      "logits/rejected": 0.10886949300765991,
      "logps/chosen": -338.0203552246094,
      "logps/rejected": -347.1212463378906,
      "loss": 0.2997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5159265995025635,
      "rewards/margins": 1.5142970085144043,
      "rewards/rejected": -2.0302233695983887,
      "step": 2011
    },
    {
      "epoch": 0.26328616995174614,
      "grad_norm": 8.639557838439941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1353743076324463,
      "logits/rejected": 0.009464602917432785,
      "logps/chosen": -458.5218811035156,
      "logps/rejected": -480.7489013671875,
      "loss": 0.6147,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7758989930152893,
      "rewards/margins": 0.8890798687934875,
      "rewards/rejected": -1.6649787425994873,
      "step": 2012
    },
    {
      "epoch": 0.2634170278890979,
      "grad_norm": 13.341950416564941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1203184649348259,
      "logits/rejected": 0.18543870747089386,
      "logps/chosen": -389.9046936035156,
      "logps/rejected": -359.58258056640625,
      "loss": 0.9792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7845898270606995,
      "rewards/margins": 0.33092403411865234,
      "rewards/rejected": -1.115513801574707,
      "step": 2013
    },
    {
      "epoch": 0.26354788582644967,
      "grad_norm": 5.605286121368408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04706042632460594,
      "logits/rejected": -0.03912125900387764,
      "logps/chosen": -634.8851928710938,
      "logps/rejected": -641.1981811523438,
      "loss": 0.3291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06245804578065872,
      "rewards/margins": 1.4621063470840454,
      "rewards/rejected": -1.3996483087539673,
      "step": 2014
    },
    {
      "epoch": 0.2636787437638014,
      "grad_norm": 7.6600799560546875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1975737363100052,
      "logits/rejected": -0.013422884047031403,
      "logps/chosen": -391.6176452636719,
      "logps/rejected": -488.0142822265625,
      "loss": 0.5746,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41125261783599854,
      "rewards/margins": 0.6919077038764954,
      "rewards/rejected": -1.1031602621078491,
      "step": 2015
    },
    {
      "epoch": 0.2638096017011532,
      "grad_norm": 5.864252090454102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06557334959506989,
      "logits/rejected": 0.1544613540172577,
      "logps/chosen": -319.07891845703125,
      "logps/rejected": -352.03619384765625,
      "loss": 0.5344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.620866060256958,
      "rewards/margins": 0.7241827249526978,
      "rewards/rejected": -1.3450489044189453,
      "step": 2016
    },
    {
      "epoch": 0.26394045963850493,
      "grad_norm": 7.267115116119385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021406767889857292,
      "logits/rejected": 0.29830703139305115,
      "logps/chosen": -466.5484619140625,
      "logps/rejected": -522.2490234375,
      "loss": 0.4336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24440997838974,
      "rewards/margins": 0.9024425745010376,
      "rewards/rejected": -1.1468526124954224,
      "step": 2017
    },
    {
      "epoch": 0.2640713175758567,
      "grad_norm": 4.5844573974609375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23667758703231812,
      "logits/rejected": 0.1270095258951187,
      "logps/chosen": -421.40240478515625,
      "logps/rejected": -363.853271484375,
      "loss": 0.3842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18176400661468506,
      "rewards/margins": 1.006211757659912,
      "rewards/rejected": -0.8244478106498718,
      "step": 2018
    },
    {
      "epoch": 0.26420217551320846,
      "grad_norm": 9.266006469726562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24114251136779785,
      "logits/rejected": 0.4102919399738312,
      "logps/chosen": -589.9630126953125,
      "logps/rejected": -628.5637817382812,
      "loss": 0.7766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025478534400463104,
      "rewards/margins": 0.5227072238922119,
      "rewards/rejected": -0.49722862243652344,
      "step": 2019
    },
    {
      "epoch": 0.26433303345056025,
      "grad_norm": 6.862087726593018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07967652380466461,
      "logits/rejected": -0.11819162964820862,
      "logps/chosen": -323.4258117675781,
      "logps/rejected": -260.1341552734375,
      "loss": 0.6694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0780123472213745,
      "rewards/margins": 0.32860928773880005,
      "rewards/rejected": -1.4066215753555298,
      "step": 2020
    },
    {
      "epoch": 0.264463891387912,
      "grad_norm": 6.390749454498291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2525627315044403,
      "logits/rejected": 0.16729624569416046,
      "logps/chosen": -399.1847229003906,
      "logps/rejected": -542.5308227539062,
      "loss": 0.3958,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29218539595603943,
      "rewards/margins": 1.379815697669983,
      "rewards/rejected": -1.6720011234283447,
      "step": 2021
    },
    {
      "epoch": 0.2645947493252638,
      "grad_norm": 7.683421611785889,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07070476561784744,
      "logits/rejected": 0.007547922432422638,
      "logps/chosen": -391.6668395996094,
      "logps/rejected": -412.7356872558594,
      "loss": 0.6338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.852915346622467,
      "rewards/margins": 0.3891761898994446,
      "rewards/rejected": -1.2420916557312012,
      "step": 2022
    },
    {
      "epoch": 0.2647256072626155,
      "grad_norm": 9.005526542663574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01823103055357933,
      "logits/rejected": -0.21319308876991272,
      "logps/chosen": -346.2461853027344,
      "logps/rejected": -418.1512145996094,
      "loss": 0.7959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4970074892044067,
      "rewards/margins": -0.07182224094867706,
      "rewards/rejected": -1.425185203552246,
      "step": 2023
    },
    {
      "epoch": 0.2648564651999673,
      "grad_norm": 8.145224571228027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04491709917783737,
      "logits/rejected": -0.04863589257001877,
      "logps/chosen": -430.108154296875,
      "logps/rejected": -400.63885498046875,
      "loss": 0.6591,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5991817712783813,
      "rewards/margins": 0.552090048789978,
      "rewards/rejected": -1.1512718200683594,
      "step": 2024
    },
    {
      "epoch": 0.26498732313731904,
      "grad_norm": 4.815898418426514,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2549237012863159,
      "logits/rejected": 0.3263569176197052,
      "logps/chosen": -447.39508056640625,
      "logps/rejected": -363.8177185058594,
      "loss": 0.3909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2679902911186218,
      "rewards/margins": 1.1170450448989868,
      "rewards/rejected": -1.3850352764129639,
      "step": 2025
    },
    {
      "epoch": 0.26511818107467083,
      "grad_norm": 6.344241142272949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3581433594226837,
      "logits/rejected": 0.16737687587738037,
      "logps/chosen": -436.35693359375,
      "logps/rejected": -394.0807189941406,
      "loss": 0.4722,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29276716709136963,
      "rewards/margins": 0.5953003168106079,
      "rewards/rejected": -0.8880674242973328,
      "step": 2026
    },
    {
      "epoch": 0.26524903901202257,
      "grad_norm": 6.957345485687256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15161478519439697,
      "logits/rejected": 0.05827593058347702,
      "logps/chosen": -438.51922607421875,
      "logps/rejected": -394.38836669921875,
      "loss": 0.6181,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3003432750701904,
      "rewards/margins": 0.3193761706352234,
      "rewards/rejected": -1.6197195053100586,
      "step": 2027
    },
    {
      "epoch": 0.26537989694937436,
      "grad_norm": 4.816586494445801,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10334038734436035,
      "logits/rejected": 0.06094210594892502,
      "logps/chosen": -273.8387145996094,
      "logps/rejected": -255.81390380859375,
      "loss": 0.3441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06229383498430252,
      "rewards/margins": 1.129571795463562,
      "rewards/rejected": -1.0672779083251953,
      "step": 2028
    },
    {
      "epoch": 0.2655107548867261,
      "grad_norm": 6.8094162940979,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1502915471792221,
      "logits/rejected": -0.1906428337097168,
      "logps/chosen": -481.58660888671875,
      "logps/rejected": -562.1410522460938,
      "loss": 0.5525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6095497012138367,
      "rewards/margins": 0.8430582880973816,
      "rewards/rejected": -0.23350851237773895,
      "step": 2029
    },
    {
      "epoch": 0.26564161282407783,
      "grad_norm": 7.787425994873047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2700121998786926,
      "logits/rejected": -0.20183654129505157,
      "logps/chosen": -426.49554443359375,
      "logps/rejected": -492.9546813964844,
      "loss": 0.547,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2217271625995636,
      "rewards/margins": 0.93354731798172,
      "rewards/rejected": -1.155274510383606,
      "step": 2030
    },
    {
      "epoch": 0.2657724707614296,
      "grad_norm": 8.473465919494629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03575494885444641,
      "logits/rejected": 0.13381679356098175,
      "logps/chosen": -526.1806640625,
      "logps/rejected": -603.1278686523438,
      "loss": 0.6166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20082281529903412,
      "rewards/margins": 0.9671327471733093,
      "rewards/rejected": -1.1679556369781494,
      "step": 2031
    },
    {
      "epoch": 0.26590332869878136,
      "grad_norm": 8.403772354125977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0723663717508316,
      "logits/rejected": -0.4000703990459442,
      "logps/chosen": -455.4497985839844,
      "logps/rejected": -414.3828125,
      "loss": 0.616,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6937994956970215,
      "rewards/margins": 0.5769658088684082,
      "rewards/rejected": -1.2707653045654297,
      "step": 2032
    },
    {
      "epoch": 0.26603418663613315,
      "grad_norm": 5.993890285491943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07269144058227539,
      "logits/rejected": -0.3166792392730713,
      "logps/chosen": -354.53582763671875,
      "logps/rejected": -382.59893798828125,
      "loss": 0.4995,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8394169211387634,
      "rewards/margins": 0.715968906879425,
      "rewards/rejected": -1.5553858280181885,
      "step": 2033
    },
    {
      "epoch": 0.2661650445734849,
      "grad_norm": 6.780456066131592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031994111835956573,
      "logits/rejected": -0.21506381034851074,
      "logps/chosen": -419.3729248046875,
      "logps/rejected": -379.6835021972656,
      "loss": 0.513,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6980770230293274,
      "rewards/margins": 0.4947057366371155,
      "rewards/rejected": -1.1927828788757324,
      "step": 2034
    },
    {
      "epoch": 0.2662959025108367,
      "grad_norm": 9.586784362792969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1020408570766449,
      "logits/rejected": 0.029413457959890366,
      "logps/chosen": -458.283447265625,
      "logps/rejected": -552.7732543945312,
      "loss": 0.7686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6040603518486023,
      "rewards/margins": 0.4163101613521576,
      "rewards/rejected": -1.0203704833984375,
      "step": 2035
    },
    {
      "epoch": 0.2664267604481884,
      "grad_norm": 8.26739501953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19740235805511475,
      "logits/rejected": 0.13441523909568787,
      "logps/chosen": -568.689208984375,
      "logps/rejected": -476.40203857421875,
      "loss": 0.5635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0033380314707756042,
      "rewards/margins": 0.5729554891586304,
      "rewards/rejected": -0.569617509841919,
      "step": 2036
    },
    {
      "epoch": 0.2665576183855402,
      "grad_norm": 10.416053771972656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17135663330554962,
      "logits/rejected": -0.1260647177696228,
      "logps/chosen": -650.260986328125,
      "logps/rejected": -607.78662109375,
      "loss": 0.8865,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.755871057510376,
      "rewards/margins": -0.2416452318429947,
      "rewards/rejected": -0.5142257213592529,
      "step": 2037
    },
    {
      "epoch": 0.26668847632289194,
      "grad_norm": 9.675125122070312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08031288534402847,
      "logits/rejected": 0.09816549718379974,
      "logps/chosen": -409.21734619140625,
      "logps/rejected": -417.96759033203125,
      "loss": 0.7301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8789716362953186,
      "rewards/margins": 0.24873872101306915,
      "rewards/rejected": -1.1277103424072266,
      "step": 2038
    },
    {
      "epoch": 0.26681933426024373,
      "grad_norm": 4.347903251647949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5361272096633911,
      "logits/rejected": 0.2848352789878845,
      "logps/chosen": -489.2333984375,
      "logps/rejected": -480.21478271484375,
      "loss": 0.3568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10815068334341049,
      "rewards/margins": 1.3252403736114502,
      "rewards/rejected": -1.2170898914337158,
      "step": 2039
    },
    {
      "epoch": 0.26695019219759547,
      "grad_norm": 8.215042114257812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22726376354694366,
      "logits/rejected": 0.12086190283298492,
      "logps/chosen": -406.6899719238281,
      "logps/rejected": -395.97576904296875,
      "loss": 0.8042,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8692642450332642,
      "rewards/margins": -0.10474279522895813,
      "rewards/rejected": -0.7645214200019836,
      "step": 2040
    },
    {
      "epoch": 0.26708105013494726,
      "grad_norm": 7.000244617462158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13880416750907898,
      "logits/rejected": 0.21962469816207886,
      "logps/chosen": -456.747314453125,
      "logps/rejected": -559.1339721679688,
      "loss": 0.529,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05090082436800003,
      "rewards/margins": 0.7762933373451233,
      "rewards/rejected": -0.7253925800323486,
      "step": 2041
    },
    {
      "epoch": 0.267211908072299,
      "grad_norm": 8.76816177368164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09381736814975739,
      "logits/rejected": 0.11331970989704132,
      "logps/chosen": -370.0209655761719,
      "logps/rejected": -389.3757019042969,
      "loss": 0.5559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7853134274482727,
      "rewards/margins": 0.706584095954895,
      "rewards/rejected": -1.4918975830078125,
      "step": 2042
    },
    {
      "epoch": 0.2673427660096508,
      "grad_norm": 5.760162830352783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05838508903980255,
      "logits/rejected": 0.1012854129076004,
      "logps/chosen": -441.0281982421875,
      "logps/rejected": -450.4783935546875,
      "loss": 0.3935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6867443323135376,
      "rewards/margins": 1.1447795629501343,
      "rewards/rejected": -1.8315240144729614,
      "step": 2043
    },
    {
      "epoch": 0.2674736239470025,
      "grad_norm": 5.945555686950684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04164349287748337,
      "logits/rejected": -0.1011730507016182,
      "logps/chosen": -489.16204833984375,
      "logps/rejected": -356.79364013671875,
      "loss": 0.5625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36170274019241333,
      "rewards/margins": 0.5331319570541382,
      "rewards/rejected": -0.8948346972465515,
      "step": 2044
    },
    {
      "epoch": 0.2676044818843543,
      "grad_norm": 8.054945945739746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4565611481666565,
      "logits/rejected": 0.21399639546871185,
      "logps/chosen": -428.3262939453125,
      "logps/rejected": -371.3987731933594,
      "loss": 0.7097,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8264853954315186,
      "rewards/margins": 0.3234812021255493,
      "rewards/rejected": -1.1499665975570679,
      "step": 2045
    },
    {
      "epoch": 0.26773533982170605,
      "grad_norm": 4.321698188781738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3221280872821808,
      "logits/rejected": 0.3398670554161072,
      "logps/chosen": -441.63702392578125,
      "logps/rejected": -345.2763977050781,
      "loss": 0.2499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5564693212509155,
      "rewards/margins": 1.754183053970337,
      "rewards/rejected": -2.310652256011963,
      "step": 2046
    },
    {
      "epoch": 0.26786619775905784,
      "grad_norm": 8.871377944946289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03190216049551964,
      "logits/rejected": -0.08590546250343323,
      "logps/chosen": -638.9028930664062,
      "logps/rejected": -563.9287109375,
      "loss": 0.5745,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4635437726974487,
      "rewards/margins": 0.5770226716995239,
      "rewards/rejected": -2.0405664443969727,
      "step": 2047
    },
    {
      "epoch": 0.2679970556964096,
      "grad_norm": 5.124325752258301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16570007801055908,
      "logits/rejected": -0.0021889731287956238,
      "logps/chosen": -590.8556518554688,
      "logps/rejected": -545.5712280273438,
      "loss": 0.3701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7389376163482666,
      "rewards/margins": 1.1878471374511719,
      "rewards/rejected": -0.44890934228897095,
      "step": 2048
    },
    {
      "epoch": 0.26812791363376137,
      "grad_norm": 8.496253967285156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1744667887687683,
      "logits/rejected": -0.006517350673675537,
      "logps/chosen": -284.9681091308594,
      "logps/rejected": -441.1964111328125,
      "loss": 0.4942,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.513822615146637,
      "rewards/margins": 1.2717516422271729,
      "rewards/rejected": -1.7855743169784546,
      "step": 2049
    },
    {
      "epoch": 0.2682587715711131,
      "grad_norm": 7.9286417961120605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04776281863451004,
      "logits/rejected": -0.16935265064239502,
      "logps/chosen": -469.748046875,
      "logps/rejected": -536.7242431640625,
      "loss": 0.6213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.783258855342865,
      "rewards/margins": 0.7975475788116455,
      "rewards/rejected": -1.5808064937591553,
      "step": 2050
    },
    {
      "epoch": 0.2683896295084649,
      "grad_norm": 7.2266693115234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06475598365068436,
      "logits/rejected": -0.06923171877861023,
      "logps/chosen": -344.3473815917969,
      "logps/rejected": -323.25811767578125,
      "loss": 0.6321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5128570795059204,
      "rewards/margins": 0.596805214881897,
      "rewards/rejected": -1.1096621751785278,
      "step": 2051
    },
    {
      "epoch": 0.26852048744581664,
      "grad_norm": 5.133222579956055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11954033374786377,
      "logits/rejected": 0.058588698506355286,
      "logps/chosen": -662.869140625,
      "logps/rejected": -572.7002563476562,
      "loss": 0.2606,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.28722134232521057,
      "rewards/margins": 1.4515208005905151,
      "rewards/rejected": -1.1642993688583374,
      "step": 2052
    },
    {
      "epoch": 0.26865134538316837,
      "grad_norm": 4.95025634765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10778941959142685,
      "logits/rejected": 0.1114816963672638,
      "logps/chosen": -384.2919921875,
      "logps/rejected": -417.6521911621094,
      "loss": 0.3653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38756170868873596,
      "rewards/margins": 0.9894555807113647,
      "rewards/rejected": -0.6018937826156616,
      "step": 2053
    },
    {
      "epoch": 0.26878220332052016,
      "grad_norm": 6.697962760925293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37409383058547974,
      "logits/rejected": 0.13500922918319702,
      "logps/chosen": -478.62457275390625,
      "logps/rejected": -413.9476623535156,
      "loss": 0.4978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06681443750858307,
      "rewards/margins": 0.665798544883728,
      "rewards/rejected": -0.7326130270957947,
      "step": 2054
    },
    {
      "epoch": 0.2689130612578719,
      "grad_norm": 6.990237712860107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04063326120376587,
      "logits/rejected": -0.19766931235790253,
      "logps/chosen": -565.7146606445312,
      "logps/rejected": -417.11004638671875,
      "loss": 0.4652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2124364823102951,
      "rewards/margins": 1.0479826927185059,
      "rewards/rejected": -1.2604191303253174,
      "step": 2055
    },
    {
      "epoch": 0.2690439191952237,
      "grad_norm": 5.884073257446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2372123897075653,
      "logits/rejected": -0.5136776566505432,
      "logps/chosen": -376.7598876953125,
      "logps/rejected": -504.63812255859375,
      "loss": 0.4783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.332944393157959,
      "rewards/margins": 0.7879190444946289,
      "rewards/rejected": -2.120863437652588,
      "step": 2056
    },
    {
      "epoch": 0.2691747771325754,
      "grad_norm": 9.462481498718262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.050894033163785934,
      "logits/rejected": -0.08691585808992386,
      "logps/chosen": -363.78704833984375,
      "logps/rejected": -374.0752258300781,
      "loss": 0.6377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0064774751663208,
      "rewards/margins": 0.34130823612213135,
      "rewards/rejected": -1.3477858304977417,
      "step": 2057
    },
    {
      "epoch": 0.2693056350699272,
      "grad_norm": 3.5719900131225586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02308025024831295,
      "logits/rejected": -0.18883448839187622,
      "logps/chosen": -660.9344482421875,
      "logps/rejected": -841.3379516601562,
      "loss": 0.2023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15570490062236786,
      "rewards/margins": 2.0626096725463867,
      "rewards/rejected": -2.2183144092559814,
      "step": 2058
    },
    {
      "epoch": 0.26943649300727895,
      "grad_norm": 7.3395256996154785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3120020627975464,
      "logits/rejected": 0.5268852710723877,
      "logps/chosen": -468.60736083984375,
      "logps/rejected": -401.95068359375,
      "loss": 0.5813,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18830186128616333,
      "rewards/margins": 0.48738133907318115,
      "rewards/rejected": -0.6756832599639893,
      "step": 2059
    },
    {
      "epoch": 0.26956735094463075,
      "grad_norm": 5.267756938934326,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14445851743221283,
      "logits/rejected": -0.07749265432357788,
      "logps/chosen": -444.791748046875,
      "logps/rejected": -422.4842224121094,
      "loss": 0.5577,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2625235617160797,
      "rewards/margins": 0.918861448764801,
      "rewards/rejected": -0.6563379764556885,
      "step": 2060
    },
    {
      "epoch": 0.2696982088819825,
      "grad_norm": 5.627439975738525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15429389476776123,
      "logits/rejected": 0.12279102951288223,
      "logps/chosen": -451.5353698730469,
      "logps/rejected": -487.0379638671875,
      "loss": 0.3371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12989741563796997,
      "rewards/margins": 1.3406821489334106,
      "rewards/rejected": -1.2107847929000854,
      "step": 2061
    },
    {
      "epoch": 0.2698290668193343,
      "grad_norm": 5.9343791007995605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020648900419473648,
      "logits/rejected": 0.17175854742527008,
      "logps/chosen": -600.6893920898438,
      "logps/rejected": -354.713134765625,
      "loss": 0.4106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1158088892698288,
      "rewards/margins": 1.2703578472137451,
      "rewards/rejected": -1.3861668109893799,
      "step": 2062
    },
    {
      "epoch": 0.269959924756686,
      "grad_norm": 5.748564720153809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04566100239753723,
      "logits/rejected": -0.15447942912578583,
      "logps/chosen": -534.2213745117188,
      "logps/rejected": -617.201904296875,
      "loss": 0.4263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2074587047100067,
      "rewards/margins": 0.8137151598930359,
      "rewards/rejected": -0.6062564849853516,
      "step": 2063
    },
    {
      "epoch": 0.2700907826940378,
      "grad_norm": 7.281800746917725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016904359683394432,
      "logits/rejected": -0.3638361692428589,
      "logps/chosen": -575.6339721679688,
      "logps/rejected": -684.3126220703125,
      "loss": 0.6125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28043133020401,
      "rewards/margins": 1.0482430458068848,
      "rewards/rejected": -1.32867431640625,
      "step": 2064
    },
    {
      "epoch": 0.27022164063138954,
      "grad_norm": 4.610479831695557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03156580775976181,
      "logits/rejected": -0.061312973499298096,
      "logps/chosen": -559.934326171875,
      "logps/rejected": -556.4100341796875,
      "loss": 0.2491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.021288707852363586,
      "rewards/margins": 1.7723000049591064,
      "rewards/rejected": -1.7935887575149536,
      "step": 2065
    },
    {
      "epoch": 0.27035249856874133,
      "grad_norm": 11.187607765197754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09152862429618835,
      "logits/rejected": 0.19635538756847382,
      "logps/chosen": -482.5758056640625,
      "logps/rejected": -472.7999267578125,
      "loss": 0.622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.098413109779358,
      "rewards/margins": 0.7633417844772339,
      "rewards/rejected": -1.8617550134658813,
      "step": 2066
    },
    {
      "epoch": 0.27048335650609306,
      "grad_norm": 9.392386436462402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07673667371273041,
      "logits/rejected": 0.012437060475349426,
      "logps/chosen": -375.72607421875,
      "logps/rejected": -337.3956604003906,
      "loss": 0.7199,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3519681692123413,
      "rewards/margins": 0.4622963070869446,
      "rewards/rejected": -0.8142645359039307,
      "step": 2067
    },
    {
      "epoch": 0.27061421444344486,
      "grad_norm": 7.768551349639893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42099469900131226,
      "logits/rejected": 0.5182868838310242,
      "logps/chosen": -400.7108459472656,
      "logps/rejected": -388.5618591308594,
      "loss": 0.7295,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0014698505401611,
      "rewards/margins": 0.013044558465480804,
      "rewards/rejected": -1.0145143270492554,
      "step": 2068
    },
    {
      "epoch": 0.2707450723807966,
      "grad_norm": 7.1421799659729,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2701537013053894,
      "logits/rejected": 0.021782472729682922,
      "logps/chosen": -456.0924377441406,
      "logps/rejected": -354.95892333984375,
      "loss": 0.5819,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27544325590133667,
      "rewards/margins": 0.572519838809967,
      "rewards/rejected": -0.8479630947113037,
      "step": 2069
    },
    {
      "epoch": 0.2708759303181484,
      "grad_norm": 20.62364387512207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19584707915782928,
      "logits/rejected": 0.18639659881591797,
      "logps/chosen": -335.7758483886719,
      "logps/rejected": -508.7239074707031,
      "loss": 0.7034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4136630892753601,
      "rewards/margins": 0.34401267766952515,
      "rewards/rejected": -0.7576757073402405,
      "step": 2070
    },
    {
      "epoch": 0.2710067882555001,
      "grad_norm": 5.585898399353027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040127333253622055,
      "logits/rejected": 0.3360389173030853,
      "logps/chosen": -565.3834228515625,
      "logps/rejected": -523.9171752929688,
      "loss": 0.42,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7316869497299194,
      "rewards/margins": 0.7997118234634399,
      "rewards/rejected": -1.5313987731933594,
      "step": 2071
    },
    {
      "epoch": 0.2711376461928519,
      "grad_norm": 6.647818565368652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08706909418106079,
      "logits/rejected": 0.02084258198738098,
      "logps/chosen": -378.6821594238281,
      "logps/rejected": -431.3252868652344,
      "loss": 0.4585,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.30996131896972656,
      "rewards/margins": 1.5302339792251587,
      "rewards/rejected": -1.2202726602554321,
      "step": 2072
    },
    {
      "epoch": 0.27126850413020365,
      "grad_norm": 10.799036979675293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05711083486676216,
      "logits/rejected": -0.42208173871040344,
      "logps/chosen": -443.0068359375,
      "logps/rejected": -587.0134887695312,
      "loss": 0.8567,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.25359705090522766,
      "rewards/margins": 0.95162034034729,
      "rewards/rejected": -0.6980232000350952,
      "step": 2073
    },
    {
      "epoch": 0.2713993620675554,
      "grad_norm": 7.42230224609375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.280276358127594,
      "logits/rejected": 0.18330174684524536,
      "logps/chosen": -524.4444580078125,
      "logps/rejected": -427.0657043457031,
      "loss": 0.4298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.011376939713954926,
      "rewards/margins": 1.0730962753295898,
      "rewards/rejected": -1.0844733715057373,
      "step": 2074
    },
    {
      "epoch": 0.2715302200049072,
      "grad_norm": 9.336089134216309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21807587146759033,
      "logits/rejected": 0.30144622921943665,
      "logps/chosen": -413.71197509765625,
      "logps/rejected": -421.74310302734375,
      "loss": 0.8022,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5805991888046265,
      "rewards/margins": 0.017417356371879578,
      "rewards/rejected": -0.5980165600776672,
      "step": 2075
    },
    {
      "epoch": 0.2716610779422589,
      "grad_norm": 9.741493225097656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11840333789587021,
      "logits/rejected": -0.07495297491550446,
      "logps/chosen": -371.6310119628906,
      "logps/rejected": -357.6766662597656,
      "loss": 0.6026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.023008942604065,
      "rewards/margins": 0.4189451336860657,
      "rewards/rejected": -1.4419540166854858,
      "step": 2076
    },
    {
      "epoch": 0.2717919358796107,
      "grad_norm": 4.969454765319824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04407699033617973,
      "logits/rejected": 0.1600300371646881,
      "logps/chosen": -329.59814453125,
      "logps/rejected": -359.0867919921875,
      "loss": 0.5068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.195766419172287,
      "rewards/margins": 0.642652153968811,
      "rewards/rejected": -0.8384184837341309,
      "step": 2077
    },
    {
      "epoch": 0.27192279381696244,
      "grad_norm": 4.131644248962402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22290703654289246,
      "logits/rejected": 0.0300234854221344,
      "logps/chosen": -441.1803283691406,
      "logps/rejected": -455.5101318359375,
      "loss": 0.2789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5728986263275146,
      "rewards/margins": 1.2717875242233276,
      "rewards/rejected": -1.8446860313415527,
      "step": 2078
    },
    {
      "epoch": 0.27205365175431423,
      "grad_norm": 9.449007987976074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08773671835660934,
      "logits/rejected": 0.004344319924712181,
      "logps/chosen": -646.4000854492188,
      "logps/rejected": -567.9146118164062,
      "loss": 0.9569,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08496396988630295,
      "rewards/margins": -0.15316542983055115,
      "rewards/rejected": 0.2381294220685959,
      "step": 2079
    },
    {
      "epoch": 0.27218450969166597,
      "grad_norm": 7.195733547210693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0409867987036705,
      "logits/rejected": 0.04520845785737038,
      "logps/chosen": -432.1876525878906,
      "logps/rejected": -462.08978271484375,
      "loss": 0.5668,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43541792035102844,
      "rewards/margins": 0.832888126373291,
      "rewards/rejected": -1.2683061361312866,
      "step": 2080
    },
    {
      "epoch": 0.27231536762901776,
      "grad_norm": 5.467305660247803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04412725940346718,
      "logits/rejected": -0.008028507232666016,
      "logps/chosen": -642.554443359375,
      "logps/rejected": -497.7866516113281,
      "loss": 0.3716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.148616224527359,
      "rewards/margins": 1.1136869192123413,
      "rewards/rejected": -0.9650707244873047,
      "step": 2081
    },
    {
      "epoch": 0.2724462255663695,
      "grad_norm": 9.830103874206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07936824858188629,
      "logits/rejected": 0.04679051786661148,
      "logps/chosen": -586.7276611328125,
      "logps/rejected": -415.2998046875,
      "loss": 0.6769,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.494737833738327,
      "rewards/margins": 0.9669784307479858,
      "rewards/rejected": -1.4617161750793457,
      "step": 2082
    },
    {
      "epoch": 0.2725770835037213,
      "grad_norm": 8.33523941040039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3609060049057007,
      "logits/rejected": 0.3132222592830658,
      "logps/chosen": -481.03277587890625,
      "logps/rejected": -447.07781982421875,
      "loss": 0.7376,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.29335689544677734,
      "rewards/margins": 0.37056946754455566,
      "rewards/rejected": -0.07721252739429474,
      "step": 2083
    },
    {
      "epoch": 0.272707941441073,
      "grad_norm": 6.051905632019043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03340322524309158,
      "logits/rejected": -0.1847495138645172,
      "logps/chosen": -362.8492736816406,
      "logps/rejected": -351.7707214355469,
      "loss": 0.4916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08469590544700623,
      "rewards/margins": 0.8376462459564209,
      "rewards/rejected": -0.7529502511024475,
      "step": 2084
    },
    {
      "epoch": 0.2728387993784248,
      "grad_norm": 7.232476234436035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13700951635837555,
      "logits/rejected": -0.04274669289588928,
      "logps/chosen": -278.13226318359375,
      "logps/rejected": -380.3270568847656,
      "loss": 0.6672,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31468886137008667,
      "rewards/margins": 1.2155632972717285,
      "rewards/rejected": -1.5302519798278809,
      "step": 2085
    },
    {
      "epoch": 0.27296965731577655,
      "grad_norm": 11.74495792388916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019267581403255463,
      "logits/rejected": -0.22683227062225342,
      "logps/chosen": -553.1475830078125,
      "logps/rejected": -580.646240234375,
      "loss": 0.9018,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.648154079914093,
      "rewards/margins": 0.166767880320549,
      "rewards/rejected": -0.8149219751358032,
      "step": 2086
    },
    {
      "epoch": 0.27310051525312834,
      "grad_norm": 8.026476860046387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04259738326072693,
      "logits/rejected": 0.23858074843883514,
      "logps/chosen": -507.88916015625,
      "logps/rejected": -408.59869384765625,
      "loss": 0.5572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04306413605809212,
      "rewards/margins": 0.8037250638008118,
      "rewards/rejected": -0.8467891216278076,
      "step": 2087
    },
    {
      "epoch": 0.2732313731904801,
      "grad_norm": 10.576692581176758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06371590495109558,
      "logits/rejected": 0.15300972759723663,
      "logps/chosen": -491.2608947753906,
      "logps/rejected": -552.1903686523438,
      "loss": 0.9562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6184313297271729,
      "rewards/margins": 0.13840427994728088,
      "rewards/rejected": -0.7568355798721313,
      "step": 2088
    },
    {
      "epoch": 0.27336223112783187,
      "grad_norm": 4.670691967010498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31544098258018494,
      "logits/rejected": 0.12871065735816956,
      "logps/chosen": -347.056396484375,
      "logps/rejected": -401.7114562988281,
      "loss": 0.3182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3091663718223572,
      "rewards/margins": 1.3565642833709717,
      "rewards/rejected": -1.0473979711532593,
      "step": 2089
    },
    {
      "epoch": 0.2734930890651836,
      "grad_norm": 7.487025737762451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06929569691419601,
      "logits/rejected": 0.05896037444472313,
      "logps/chosen": -413.89691162109375,
      "logps/rejected": -533.1337280273438,
      "loss": 0.5494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.335475355386734,
      "rewards/margins": 0.7903379201889038,
      "rewards/rejected": -0.4548625349998474,
      "step": 2090
    },
    {
      "epoch": 0.2736239470025354,
      "grad_norm": 10.548309326171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22566770017147064,
      "logits/rejected": -0.34204021096229553,
      "logps/chosen": -305.6102294921875,
      "logps/rejected": -321.438232421875,
      "loss": 0.6445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0687202215194702,
      "rewards/margins": 0.41836756467819214,
      "rewards/rejected": -1.4870878458023071,
      "step": 2091
    },
    {
      "epoch": 0.27375480493988713,
      "grad_norm": 4.596423625946045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3408154845237732,
      "logits/rejected": 0.3237335681915283,
      "logps/chosen": -299.02685546875,
      "logps/rejected": -316.63031005859375,
      "loss": 0.3452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0011110268533229828,
      "rewards/margins": 1.47321617603302,
      "rewards/rejected": -1.4743272066116333,
      "step": 2092
    },
    {
      "epoch": 0.2738856628772389,
      "grad_norm": 8.028828620910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004270285367965698,
      "logits/rejected": 0.1487860083580017,
      "logps/chosen": -502.97576904296875,
      "logps/rejected": -492.71197509765625,
      "loss": 0.5464,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5566864013671875,
      "rewards/margins": 0.6624493598937988,
      "rewards/rejected": -0.10576307773590088,
      "step": 2093
    },
    {
      "epoch": 0.27401652081459066,
      "grad_norm": 9.26724910736084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039292603731155396,
      "logits/rejected": 0.16521689295768738,
      "logps/chosen": -675.2427368164062,
      "logps/rejected": -503.93353271484375,
      "loss": 0.7216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08922939747571945,
      "rewards/margins": 0.1301458179950714,
      "rewards/rejected": -0.040916457772254944,
      "step": 2094
    },
    {
      "epoch": 0.27414737875194245,
      "grad_norm": 5.3964924812316895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10611133277416229,
      "logits/rejected": -0.09519236534833908,
      "logps/chosen": -689.28466796875,
      "logps/rejected": -468.9633483886719,
      "loss": 0.342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1651097983121872,
      "rewards/margins": 1.4373164176940918,
      "rewards/rejected": -1.2722066640853882,
      "step": 2095
    },
    {
      "epoch": 0.2742782366892942,
      "grad_norm": 6.864964485168457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11830822378396988,
      "logits/rejected": -0.05223169922828674,
      "logps/chosen": -437.7215270996094,
      "logps/rejected": -606.17822265625,
      "loss": 0.4912,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8200960159301758,
      "rewards/margins": 0.5728239417076111,
      "rewards/rejected": -1.3929200172424316,
      "step": 2096
    },
    {
      "epoch": 0.2744090946266459,
      "grad_norm": 9.121405601501465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1086333692073822,
      "logits/rejected": 0.18150587379932404,
      "logps/chosen": -564.451416015625,
      "logps/rejected": -416.8487854003906,
      "loss": 0.5613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8070844411849976,
      "rewards/margins": 0.49829429388046265,
      "rewards/rejected": -1.305378794670105,
      "step": 2097
    },
    {
      "epoch": 0.2745399525639977,
      "grad_norm": 8.034415245056152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2208152711391449,
      "logits/rejected": -0.05331198871135712,
      "logps/chosen": -652.005859375,
      "logps/rejected": -470.28173828125,
      "loss": 0.5321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1282985657453537,
      "rewards/margins": 0.7194740772247314,
      "rewards/rejected": -0.8477725982666016,
      "step": 2098
    },
    {
      "epoch": 0.27467081050134945,
      "grad_norm": 2.889838457107544,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009793095290660858,
      "logits/rejected": -0.16770398616790771,
      "logps/chosen": -329.7768859863281,
      "logps/rejected": -528.9970703125,
      "loss": 0.1696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022560298442840576,
      "rewards/margins": 2.5389404296875,
      "rewards/rejected": -2.5615007877349854,
      "step": 2099
    },
    {
      "epoch": 0.27480166843870124,
      "grad_norm": 7.626249313354492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035051051527261734,
      "logits/rejected": 0.1889525055885315,
      "logps/chosen": -410.763671875,
      "logps/rejected": -415.393798828125,
      "loss": 0.7678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1740347146987915,
      "rewards/margins": 0.3724796772003174,
      "rewards/rejected": -0.1984449326992035,
      "step": 2100
    },
    {
      "epoch": 0.274932526376053,
      "grad_norm": 9.30810832977295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32067620754241943,
      "logits/rejected": 0.302754670381546,
      "logps/chosen": -458.40032958984375,
      "logps/rejected": -520.5133056640625,
      "loss": 0.8524,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.30573463439941406,
      "rewards/margins": 0.3010875880718231,
      "rewards/rejected": -0.6068222522735596,
      "step": 2101
    },
    {
      "epoch": 0.27506338431340477,
      "grad_norm": 7.0989298820495605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.057680029422044754,
      "logits/rejected": 0.16967804729938507,
      "logps/chosen": -475.0292053222656,
      "logps/rejected": -309.6419982910156,
      "loss": 0.6088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1582588404417038,
      "rewards/margins": 0.7538833022117615,
      "rewards/rejected": -0.5956243872642517,
      "step": 2102
    },
    {
      "epoch": 0.2751942422507565,
      "grad_norm": 7.287698745727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44015607237815857,
      "logits/rejected": 0.3398217558860779,
      "logps/chosen": -416.77789306640625,
      "logps/rejected": -454.8938293457031,
      "loss": 0.7106,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21376168727874756,
      "rewards/margins": 0.3963787257671356,
      "rewards/rejected": -0.6101404428482056,
      "step": 2103
    },
    {
      "epoch": 0.2753251001881083,
      "grad_norm": 7.817655563354492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00890250876545906,
      "logits/rejected": -0.15305528044700623,
      "logps/chosen": -455.5350341796875,
      "logps/rejected": -413.90924072265625,
      "loss": 0.5677,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6851218938827515,
      "rewards/margins": 0.47846490144729614,
      "rewards/rejected": -1.1635868549346924,
      "step": 2104
    },
    {
      "epoch": 0.27545595812546003,
      "grad_norm": 4.836092948913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21609963476657867,
      "logits/rejected": -0.11200408637523651,
      "logps/chosen": -456.8590087890625,
      "logps/rejected": -328.6949462890625,
      "loss": 0.3617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4008539319038391,
      "rewards/margins": 1.333923101425171,
      "rewards/rejected": -1.7347769737243652,
      "step": 2105
    },
    {
      "epoch": 0.2755868160628118,
      "grad_norm": 8.124388694763184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06544191390275955,
      "logits/rejected": -0.026177560910582542,
      "logps/chosen": -462.6113586425781,
      "logps/rejected": -536.2691650390625,
      "loss": 0.7558,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.049277305603027344,
      "rewards/margins": 0.3083839416503906,
      "rewards/rejected": -0.2591066360473633,
      "step": 2106
    },
    {
      "epoch": 0.27571767400016356,
      "grad_norm": 7.24345064163208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12658466398715973,
      "logits/rejected": -0.007397410925477743,
      "logps/chosen": -466.9765930175781,
      "logps/rejected": -466.4267578125,
      "loss": 0.557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06702879071235657,
      "rewards/margins": 0.6910927295684814,
      "rewards/rejected": -0.6240638494491577,
      "step": 2107
    },
    {
      "epoch": 0.27584853193751535,
      "grad_norm": 7.97907018661499,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16098368167877197,
      "logits/rejected": 0.08041106164455414,
      "logps/chosen": -496.8727722167969,
      "logps/rejected": -472.78167724609375,
      "loss": 0.6416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13156644999980927,
      "rewards/margins": 0.7341348528862,
      "rewards/rejected": -0.6025684475898743,
      "step": 2108
    },
    {
      "epoch": 0.2759793898748671,
      "grad_norm": 8.834546089172363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3567814826965332,
      "logits/rejected": 0.1325821876525879,
      "logps/chosen": -365.7151184082031,
      "logps/rejected": -614.0654296875,
      "loss": 0.7695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21673992276191711,
      "rewards/margins": 0.8990961313247681,
      "rewards/rejected": -1.1158361434936523,
      "step": 2109
    },
    {
      "epoch": 0.2761102478122189,
      "grad_norm": 6.38788366317749,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12325378507375717,
      "logits/rejected": -0.050946272909641266,
      "logps/chosen": -416.8000793457031,
      "logps/rejected": -453.240966796875,
      "loss": 0.4194,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19904330372810364,
      "rewards/margins": 0.824082612991333,
      "rewards/rejected": -0.625039279460907,
      "step": 2110
    },
    {
      "epoch": 0.2762411057495706,
      "grad_norm": 13.115986824035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12009002268314362,
      "logits/rejected": -0.10565166175365448,
      "logps/chosen": -533.555419921875,
      "logps/rejected": -543.7282104492188,
      "loss": 0.7912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8112850189208984,
      "rewards/margins": 0.32816052436828613,
      "rewards/rejected": -1.1394455432891846,
      "step": 2111
    },
    {
      "epoch": 0.2763719636869224,
      "grad_norm": 11.088826179504395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43726611137390137,
      "logits/rejected": -0.05870704725384712,
      "logps/chosen": -646.9608764648438,
      "logps/rejected": -495.2883605957031,
      "loss": 0.768,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6133345365524292,
      "rewards/margins": 0.11409200727939606,
      "rewards/rejected": -0.7274264693260193,
      "step": 2112
    },
    {
      "epoch": 0.27650282162427414,
      "grad_norm": 7.692410945892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14018195867538452,
      "logits/rejected": -0.12493227422237396,
      "logps/chosen": -298.7447814941406,
      "logps/rejected": -253.642578125,
      "loss": 0.6341,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48771995306015015,
      "rewards/margins": 0.2444862425327301,
      "rewards/rejected": -0.7322061657905579,
      "step": 2113
    },
    {
      "epoch": 0.27663367956162593,
      "grad_norm": 5.998995304107666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025232069194316864,
      "logits/rejected": -0.022012777626514435,
      "logps/chosen": -372.1321105957031,
      "logps/rejected": -447.7947998046875,
      "loss": 0.4952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.191254660487175,
      "rewards/margins": 0.8760970830917358,
      "rewards/rejected": -1.0673516988754272,
      "step": 2114
    },
    {
      "epoch": 0.27676453749897767,
      "grad_norm": 5.769173622131348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006735317409038544,
      "logits/rejected": -0.044925786554813385,
      "logps/chosen": -718.4605712890625,
      "logps/rejected": -450.63555908203125,
      "loss": 0.3149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2261037826538086,
      "rewards/margins": 1.2995128631591797,
      "rewards/rejected": -1.073409080505371,
      "step": 2115
    },
    {
      "epoch": 0.27689539543632946,
      "grad_norm": 4.0550031661987305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18578167259693146,
      "logits/rejected": 0.20563815534114838,
      "logps/chosen": -505.16229248046875,
      "logps/rejected": -422.6346740722656,
      "loss": 0.3102,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.620823085308075,
      "rewards/margins": 1.3094860315322876,
      "rewards/rejected": -0.6886629462242126,
      "step": 2116
    },
    {
      "epoch": 0.2770262533736812,
      "grad_norm": 8.131102561950684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22796615958213806,
      "logits/rejected": 0.4036800265312195,
      "logps/chosen": -375.67388916015625,
      "logps/rejected": -347.3070068359375,
      "loss": 0.6702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20057964324951172,
      "rewards/margins": 0.21677722036838531,
      "rewards/rejected": -0.4173568785190582,
      "step": 2117
    },
    {
      "epoch": 0.27715711131103293,
      "grad_norm": 4.8811492919921875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35147911310195923,
      "logits/rejected": 0.15599825978279114,
      "logps/chosen": -329.68585205078125,
      "logps/rejected": -373.4685363769531,
      "loss": 0.4263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5225839614868164,
      "rewards/margins": 1.5559412240982056,
      "rewards/rejected": -1.0333571434020996,
      "step": 2118
    },
    {
      "epoch": 0.2772879692483847,
      "grad_norm": 12.749540328979492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04522717744112015,
      "logits/rejected": -0.09977719932794571,
      "logps/chosen": -488.48370361328125,
      "logps/rejected": -412.6763610839844,
      "loss": 0.6358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9845629930496216,
      "rewards/margins": 0.5144607424736023,
      "rewards/rejected": -1.4990237951278687,
      "step": 2119
    },
    {
      "epoch": 0.27741882718573646,
      "grad_norm": 10.066859245300293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2229180485010147,
      "logits/rejected": 0.029155045747756958,
      "logps/chosen": -274.93426513671875,
      "logps/rejected": -351.8925476074219,
      "loss": 0.7374,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33653944730758667,
      "rewards/margins": 0.2718892991542816,
      "rewards/rejected": -0.6084287166595459,
      "step": 2120
    },
    {
      "epoch": 0.27754968512308825,
      "grad_norm": 3.6431331634521484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.053146280348300934,
      "logits/rejected": 0.32059019804000854,
      "logps/chosen": -683.5517578125,
      "logps/rejected": -357.53411865234375,
      "loss": 0.2258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4245603382587433,
      "rewards/margins": 1.6490819454193115,
      "rewards/rejected": -1.224521517753601,
      "step": 2121
    },
    {
      "epoch": 0.27768054306044,
      "grad_norm": 7.272467613220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12655311822891235,
      "logits/rejected": -0.046354539692401886,
      "logps/chosen": -405.2072448730469,
      "logps/rejected": -359.16571044921875,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.2827582359313965,
      "rewards/margins": 0.29070407152175903,
      "rewards/rejected": -0.007945813238620758,
      "step": 2122
    },
    {
      "epoch": 0.2778114009977918,
      "grad_norm": 4.132681369781494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19556555151939392,
      "logits/rejected": 0.2074057161808014,
      "logps/chosen": -537.3832397460938,
      "logps/rejected": -420.4113464355469,
      "loss": 0.279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4372093379497528,
      "rewards/margins": 1.7040308713912964,
      "rewards/rejected": -1.2668216228485107,
      "step": 2123
    },
    {
      "epoch": 0.2779422589351435,
      "grad_norm": 5.608360290527344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2570258677005768,
      "logits/rejected": 0.07967539131641388,
      "logps/chosen": -461.7304992675781,
      "logps/rejected": -535.4218139648438,
      "loss": 0.4098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20685255527496338,
      "rewards/margins": 1.0314993858337402,
      "rewards/rejected": -1.238351821899414,
      "step": 2124
    },
    {
      "epoch": 0.2780731168724953,
      "grad_norm": 6.454054832458496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2470441311597824,
      "logits/rejected": -0.17637397348880768,
      "logps/chosen": -548.828369140625,
      "logps/rejected": -461.97991943359375,
      "loss": 0.3994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27136003971099854,
      "rewards/margins": 0.9982200860977173,
      "rewards/rejected": -0.7268601059913635,
      "step": 2125
    },
    {
      "epoch": 0.27820397480984704,
      "grad_norm": 9.614787101745605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04594124108552933,
      "logits/rejected": 0.12854641675949097,
      "logps/chosen": -426.44793701171875,
      "logps/rejected": -515.5816650390625,
      "loss": 0.7503,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6714880466461182,
      "rewards/margins": 0.21101993322372437,
      "rewards/rejected": -0.8825079202651978,
      "step": 2126
    },
    {
      "epoch": 0.27833483274719883,
      "grad_norm": 9.77443790435791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14749471843242645,
      "logits/rejected": -0.09193587303161621,
      "logps/chosen": -346.2976379394531,
      "logps/rejected": -284.4948425292969,
      "loss": 0.6481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22450900077819824,
      "rewards/margins": 0.18744534254074097,
      "rewards/rejected": -0.4119543135166168,
      "step": 2127
    },
    {
      "epoch": 0.27846569068455057,
      "grad_norm": 5.694413661956787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3237648606300354,
      "logits/rejected": 0.17686167359352112,
      "logps/chosen": -358.9745788574219,
      "logps/rejected": -271.33587646484375,
      "loss": 0.6732,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.30659887194633484,
      "rewards/margins": 0.2827438414096832,
      "rewards/rejected": 0.023855017498135567,
      "step": 2128
    },
    {
      "epoch": 0.27859654862190236,
      "grad_norm": 6.020824909210205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5734706521034241,
      "logits/rejected": 0.6573119163513184,
      "logps/chosen": -327.71875,
      "logps/rejected": -423.21319580078125,
      "loss": 0.5521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14378030598163605,
      "rewards/margins": 0.6723372936248779,
      "rewards/rejected": -0.5285570025444031,
      "step": 2129
    },
    {
      "epoch": 0.2787274065592541,
      "grad_norm": 11.74480152130127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23869280517101288,
      "logits/rejected": 0.054900333285331726,
      "logps/chosen": -526.9073486328125,
      "logps/rejected": -453.5931091308594,
      "loss": 0.7972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6706449389457703,
      "rewards/margins": 0.33343276381492615,
      "rewards/rejected": -1.0040777921676636,
      "step": 2130
    },
    {
      "epoch": 0.2788582644966059,
      "grad_norm": 6.549435138702393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007265491411089897,
      "logits/rejected": -0.34496617317199707,
      "logps/chosen": -486.2568359375,
      "logps/rejected": -588.315673828125,
      "loss": 0.3199,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07304495573043823,
      "rewards/margins": 1.3798658847808838,
      "rewards/rejected": -1.3068208694458008,
      "step": 2131
    },
    {
      "epoch": 0.2789891224339576,
      "grad_norm": 5.47865104675293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25970739126205444,
      "logits/rejected": 0.060870349407196045,
      "logps/chosen": -533.568115234375,
      "logps/rejected": -411.80108642578125,
      "loss": 0.4851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3015638589859009,
      "rewards/margins": 0.7789763808250427,
      "rewards/rejected": -1.0805402994155884,
      "step": 2132
    },
    {
      "epoch": 0.2791199803713094,
      "grad_norm": 5.287130355834961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028749341145157814,
      "logits/rejected": 0.07659146934747696,
      "logps/chosen": -449.08892822265625,
      "logps/rejected": -514.949951171875,
      "loss": 0.3291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.439399391412735,
      "rewards/margins": 1.1925246715545654,
      "rewards/rejected": -0.7531251907348633,
      "step": 2133
    },
    {
      "epoch": 0.27925083830866115,
      "grad_norm": 11.834769248962402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13477513194084167,
      "logits/rejected": 0.10639141499996185,
      "logps/chosen": -504.0267639160156,
      "logps/rejected": -476.45574951171875,
      "loss": 1.0243,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.019766077399253845,
      "rewards/margins": 0.3565629720687866,
      "rewards/rejected": -0.37632906436920166,
      "step": 2134
    },
    {
      "epoch": 0.27938169624601294,
      "grad_norm": 7.438539981842041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28963229060173035,
      "logits/rejected": 0.4411902129650116,
      "logps/chosen": -706.944580078125,
      "logps/rejected": -742.9301147460938,
      "loss": 0.5912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5747180581092834,
      "rewards/margins": 0.6469102501869202,
      "rewards/rejected": -0.07219219207763672,
      "step": 2135
    },
    {
      "epoch": 0.2795125541833647,
      "grad_norm": 8.611083984375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32801902294158936,
      "logits/rejected": 0.15860019624233246,
      "logps/chosen": -512.9981689453125,
      "logps/rejected": -455.9880065917969,
      "loss": 0.5143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09773443639278412,
      "rewards/margins": 0.5269039869308472,
      "rewards/rejected": -0.6246383786201477,
      "step": 2136
    },
    {
      "epoch": 0.2796434121207165,
      "grad_norm": 8.899737358093262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.48341304063796997,
      "logits/rejected": 0.42796629667282104,
      "logps/chosen": -433.26153564453125,
      "logps/rejected": -321.79986572265625,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29916876554489136,
      "rewards/margins": 0.25872862339019775,
      "rewards/rejected": -0.5578973889350891,
      "step": 2137
    },
    {
      "epoch": 0.2797742700580682,
      "grad_norm": 6.665699005126953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031409211456775665,
      "logits/rejected": 0.013726356439292431,
      "logps/chosen": -527.6633911132812,
      "logps/rejected": -465.2248229980469,
      "loss": 0.5151,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3542032241821289,
      "rewards/margins": 0.686246931552887,
      "rewards/rejected": -0.33204367756843567,
      "step": 2138
    },
    {
      "epoch": 0.27990512799541994,
      "grad_norm": 6.097224712371826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08865567296743393,
      "logits/rejected": 0.23514766991138458,
      "logps/chosen": -397.8380126953125,
      "logps/rejected": -415.4323425292969,
      "loss": 0.5681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.034578535705804825,
      "rewards/margins": 0.8798070549964905,
      "rewards/rejected": -0.8452286124229431,
      "step": 2139
    },
    {
      "epoch": 0.28003598593277174,
      "grad_norm": 3.9421682357788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25813499093055725,
      "logits/rejected": -0.1792386770248413,
      "logps/chosen": -318.3624572753906,
      "logps/rejected": -294.59014892578125,
      "loss": 0.35,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31412842869758606,
      "rewards/margins": 1.1055347919464111,
      "rewards/rejected": -1.4196633100509644,
      "step": 2140
    },
    {
      "epoch": 0.28016684387012347,
      "grad_norm": 6.623532772064209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10391642153263092,
      "logits/rejected": -0.07476173341274261,
      "logps/chosen": -390.64776611328125,
      "logps/rejected": -405.32537841796875,
      "loss": 0.4943,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23319625854492188,
      "rewards/margins": 0.6552860140800476,
      "rewards/rejected": -0.8884823322296143,
      "step": 2141
    },
    {
      "epoch": 0.28029770180747526,
      "grad_norm": 8.451827049255371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11455053091049194,
      "logits/rejected": 0.16511523723602295,
      "logps/chosen": -491.8179931640625,
      "logps/rejected": -370.7891845703125,
      "loss": 0.5423,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.23013094067573547,
      "rewards/margins": 1.1762189865112305,
      "rewards/rejected": -0.9460880756378174,
      "step": 2142
    },
    {
      "epoch": 0.280428559744827,
      "grad_norm": 4.8549675941467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17387890815734863,
      "logits/rejected": 0.020793186500668526,
      "logps/chosen": -444.91461181640625,
      "logps/rejected": -493.5437316894531,
      "loss": 0.4121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3537660241127014,
      "rewards/margins": 0.8370688557624817,
      "rewards/rejected": -0.4833028316497803,
      "step": 2143
    },
    {
      "epoch": 0.2805594176821788,
      "grad_norm": 6.224650859832764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10742466896772385,
      "logits/rejected": 0.10277476906776428,
      "logps/chosen": -345.9219970703125,
      "logps/rejected": -541.0108032226562,
      "loss": 0.4444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.004258537665009499,
      "rewards/margins": 1.1813410520553589,
      "rewards/rejected": -1.1770825386047363,
      "step": 2144
    },
    {
      "epoch": 0.2806902756195305,
      "grad_norm": 8.774906158447266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029188334941864014,
      "logits/rejected": 0.10951908677816391,
      "logps/chosen": -493.4109191894531,
      "logps/rejected": -550.1649780273438,
      "loss": 0.6683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6706681251525879,
      "rewards/margins": 0.15614698827266693,
      "rewards/rejected": -0.8268150687217712,
      "step": 2145
    },
    {
      "epoch": 0.2808211335568823,
      "grad_norm": 5.635683536529541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06020399183034897,
      "logits/rejected": 0.24374377727508545,
      "logps/chosen": -394.5439453125,
      "logps/rejected": -335.9939880371094,
      "loss": 0.4357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24196550250053406,
      "rewards/margins": 1.0666786432266235,
      "rewards/rejected": -0.8247133493423462,
      "step": 2146
    },
    {
      "epoch": 0.28095199149423405,
      "grad_norm": 6.552149772644043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0017965398728847504,
      "logits/rejected": -0.37245917320251465,
      "logps/chosen": -453.4804992675781,
      "logps/rejected": -535.720947265625,
      "loss": 0.6122,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4979875683784485,
      "rewards/margins": 0.9268530011177063,
      "rewards/rejected": -1.4248404502868652,
      "step": 2147
    },
    {
      "epoch": 0.28108284943158585,
      "grad_norm": 5.3416314125061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009333565831184387,
      "logits/rejected": -0.04397227242588997,
      "logps/chosen": -607.3455200195312,
      "logps/rejected": -328.8519287109375,
      "loss": 0.4124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.039826780557632446,
      "rewards/margins": 1.0143718719482422,
      "rewards/rejected": -0.9745450615882874,
      "step": 2148
    },
    {
      "epoch": 0.2812137073689376,
      "grad_norm": 4.211431503295898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.058989495038986206,
      "logits/rejected": -0.028029024600982666,
      "logps/chosen": -468.83831787109375,
      "logps/rejected": -401.9125061035156,
      "loss": 0.3626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.42374494671821594,
      "rewards/margins": 1.0758419036865234,
      "rewards/rejected": -0.6520969271659851,
      "step": 2149
    },
    {
      "epoch": 0.2813445653062894,
      "grad_norm": 4.876791954040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05740485340356827,
      "logits/rejected": 0.09509333968162537,
      "logps/chosen": -313.391845703125,
      "logps/rejected": -307.2459411621094,
      "loss": 0.3985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.45056381821632385,
      "rewards/margins": 0.8568598031997681,
      "rewards/rejected": -0.4062959551811218,
      "step": 2150
    },
    {
      "epoch": 0.2814754232436411,
      "grad_norm": 8.761930465698242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13931496441364288,
      "logits/rejected": 0.24108287692070007,
      "logps/chosen": -585.6519775390625,
      "logps/rejected": -561.97607421875,
      "loss": 0.6257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5471386313438416,
      "rewards/margins": 0.36992722749710083,
      "rewards/rejected": -0.9170657992362976,
      "step": 2151
    },
    {
      "epoch": 0.2816062811809929,
      "grad_norm": 7.809976100921631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10535740852355957,
      "logits/rejected": -0.13091599941253662,
      "logps/chosen": -480.77239990234375,
      "logps/rejected": -745.0811157226562,
      "loss": 0.6367,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.464555025100708,
      "rewards/margins": 0.518602728843689,
      "rewards/rejected": -0.983157753944397,
      "step": 2152
    },
    {
      "epoch": 0.28173713911834464,
      "grad_norm": 7.179868221282959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16540423035621643,
      "logits/rejected": 0.0975189134478569,
      "logps/chosen": -371.45941162109375,
      "logps/rejected": -431.31988525390625,
      "loss": 0.4208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.48575055599212646,
      "rewards/margins": 1.06597900390625,
      "rewards/rejected": -0.5802283883094788,
      "step": 2153
    },
    {
      "epoch": 0.28186799705569643,
      "grad_norm": 6.789588928222656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1828424036502838,
      "logits/rejected": 0.24970553815364838,
      "logps/chosen": -476.4557800292969,
      "logps/rejected": -564.376953125,
      "loss": 0.4385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3575996458530426,
      "rewards/margins": 0.9596138596534729,
      "rewards/rejected": -1.3172134160995483,
      "step": 2154
    },
    {
      "epoch": 0.28199885499304816,
      "grad_norm": 6.319387912750244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11012596637010574,
      "logits/rejected": 0.22928589582443237,
      "logps/chosen": -351.91534423828125,
      "logps/rejected": -488.2148132324219,
      "loss": 0.5759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.013282403349876404,
      "rewards/margins": 0.43549954891204834,
      "rewards/rejected": -0.44878196716308594,
      "step": 2155
    },
    {
      "epoch": 0.28212971293039996,
      "grad_norm": 7.7881550788879395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03237130492925644,
      "logits/rejected": -0.20763462781906128,
      "logps/chosen": -396.327392578125,
      "logps/rejected": -358.9805603027344,
      "loss": 0.5477,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.018476540222764015,
      "rewards/margins": 0.8210275173187256,
      "rewards/rejected": -0.839504063129425,
      "step": 2156
    },
    {
      "epoch": 0.2822605708677517,
      "grad_norm": 5.798449516296387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03445335477590561,
      "logits/rejected": 0.005307361483573914,
      "logps/chosen": -444.68511962890625,
      "logps/rejected": -446.27008056640625,
      "loss": 0.3651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13717041909694672,
      "rewards/margins": 1.164358377456665,
      "rewards/rejected": -1.3015286922454834,
      "step": 2157
    },
    {
      "epoch": 0.2823914288051035,
      "grad_norm": 7.049951076507568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12091441452503204,
      "logits/rejected": 0.11983388662338257,
      "logps/chosen": -421.83551025390625,
      "logps/rejected": -443.8745422363281,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49995458126068115,
      "rewards/margins": 0.10677376389503479,
      "rewards/rejected": -0.6067283153533936,
      "step": 2158
    },
    {
      "epoch": 0.2825222867424552,
      "grad_norm": 7.351524829864502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27253034710884094,
      "logits/rejected": 0.2741694748401642,
      "logps/chosen": -480.5337829589844,
      "logps/rejected": -500.09088134765625,
      "loss": 0.5476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.37082481384277344,
      "rewards/margins": 0.6749140620231628,
      "rewards/rejected": -0.30408933758735657,
      "step": 2159
    },
    {
      "epoch": 0.282653144679807,
      "grad_norm": 8.505453109741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09263522922992706,
      "logits/rejected": 0.026879623532295227,
      "logps/chosen": -420.4228515625,
      "logps/rejected": -270.8233642578125,
      "loss": 0.896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18547464907169342,
      "rewards/margins": 0.23227912187576294,
      "rewards/rejected": -0.04680454730987549,
      "step": 2160
    },
    {
      "epoch": 0.28278400261715875,
      "grad_norm": 7.026191234588623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09662728756666183,
      "logits/rejected": 0.15593339502811432,
      "logps/chosen": -333.7610168457031,
      "logps/rejected": -479.50555419921875,
      "loss": 0.4437,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32692062854766846,
      "rewards/margins": 0.9297872185707092,
      "rewards/rejected": -1.256707787513733,
      "step": 2161
    },
    {
      "epoch": 0.2829148605545105,
      "grad_norm": 6.091305732727051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23845170438289642,
      "logits/rejected": 0.16345000267028809,
      "logps/chosen": -444.4559020996094,
      "logps/rejected": -379.80364990234375,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.013460885733366013,
      "rewards/margins": 0.8705205917358398,
      "rewards/rejected": -0.883981466293335,
      "step": 2162
    },
    {
      "epoch": 0.2830457184918623,
      "grad_norm": 4.8981499671936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22959375381469727,
      "logits/rejected": 0.1797189712524414,
      "logps/chosen": -416.7231140136719,
      "logps/rejected": -431.59674072265625,
      "loss": 0.1782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3488386571407318,
      "rewards/margins": 1.8753254413604736,
      "rewards/rejected": -1.526486873626709,
      "step": 2163
    },
    {
      "epoch": 0.283176576429214,
      "grad_norm": 7.123744010925293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11463688313961029,
      "logits/rejected": 0.012800849974155426,
      "logps/chosen": -420.2901611328125,
      "logps/rejected": -373.4027099609375,
      "loss": 0.5458,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5967880487442017,
      "rewards/margins": 0.6714780926704407,
      "rewards/rejected": -1.2682660818099976,
      "step": 2164
    },
    {
      "epoch": 0.2833074343665658,
      "grad_norm": 6.252364635467529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04003656283020973,
      "logits/rejected": 0.11643383651971817,
      "logps/chosen": -432.3997802734375,
      "logps/rejected": -438.1658020019531,
      "loss": 0.652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38992711901664734,
      "rewards/margins": 0.5012565851211548,
      "rewards/rejected": -0.8911837339401245,
      "step": 2165
    },
    {
      "epoch": 0.28343829230391754,
      "grad_norm": 9.698347091674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00013372302055358887,
      "logits/rejected": 0.009849384427070618,
      "logps/chosen": -374.5048522949219,
      "logps/rejected": -404.785400390625,
      "loss": 0.6497,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7819047570228577,
      "rewards/margins": 0.5707778930664062,
      "rewards/rejected": -1.3526827096939087,
      "step": 2166
    },
    {
      "epoch": 0.28356915024126933,
      "grad_norm": 8.007856369018555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44141483306884766,
      "logits/rejected": 0.416659414768219,
      "logps/chosen": -383.535400390625,
      "logps/rejected": -427.91937255859375,
      "loss": 0.7526,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1336669772863388,
      "rewards/margins": 0.0524364598095417,
      "rewards/rejected": -0.1861034333705902,
      "step": 2167
    },
    {
      "epoch": 0.28370000817862107,
      "grad_norm": 7.652442932128906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0074896663427352905,
      "logits/rejected": 0.01974973827600479,
      "logps/chosen": -435.87750244140625,
      "logps/rejected": -553.0648803710938,
      "loss": 0.6946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2579021453857422,
      "rewards/margins": 0.2669921815395355,
      "rewards/rejected": -0.5248943567276001,
      "step": 2168
    },
    {
      "epoch": 0.28383086611597286,
      "grad_norm": 6.292201042175293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2010478377342224,
      "logits/rejected": 0.36622902750968933,
      "logps/chosen": -456.12890625,
      "logps/rejected": -589.1055908203125,
      "loss": 0.3994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3695419430732727,
      "rewards/margins": 1.0160225629806519,
      "rewards/rejected": -0.6464805006980896,
      "step": 2169
    },
    {
      "epoch": 0.2839617240533246,
      "grad_norm": 6.341591835021973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1569691300392151,
      "logits/rejected": 0.33639851212501526,
      "logps/chosen": -500.9787902832031,
      "logps/rejected": -478.3709411621094,
      "loss": 0.37,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23374013602733612,
      "rewards/margins": 1.2067043781280518,
      "rewards/rejected": -1.4404445886611938,
      "step": 2170
    },
    {
      "epoch": 0.2840925819906764,
      "grad_norm": 7.6827216148376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16149413585662842,
      "logits/rejected": 0.1226920560002327,
      "logps/chosen": -508.3451232910156,
      "logps/rejected": -458.8689880371094,
      "loss": 0.4191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37842121720314026,
      "rewards/margins": 0.9518577456474304,
      "rewards/rejected": -1.3302791118621826,
      "step": 2171
    },
    {
      "epoch": 0.2842234399280281,
      "grad_norm": 5.2806501388549805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33093535900115967,
      "logits/rejected": 0.026587210595607758,
      "logps/chosen": -550.0927734375,
      "logps/rejected": -376.29364013671875,
      "loss": 0.4098,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24091432988643646,
      "rewards/margins": 1.4193766117095947,
      "rewards/rejected": -1.178462266921997,
      "step": 2172
    },
    {
      "epoch": 0.2843542978653799,
      "grad_norm": 7.036606788635254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22663378715515137,
      "logits/rejected": 0.2346164584159851,
      "logps/chosen": -456.2366027832031,
      "logps/rejected": -468.7073669433594,
      "loss": 0.4565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15271800756454468,
      "rewards/margins": 1.3581489324569702,
      "rewards/rejected": -1.2054309844970703,
      "step": 2173
    },
    {
      "epoch": 0.28448515580273165,
      "grad_norm": 7.434305191040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2769585847854614,
      "logits/rejected": -0.11300679296255112,
      "logps/chosen": -473.6936950683594,
      "logps/rejected": -452.2334289550781,
      "loss": 0.4869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3181329071521759,
      "rewards/margins": 0.793507993221283,
      "rewards/rejected": -1.1116410493850708,
      "step": 2174
    },
    {
      "epoch": 0.28461601374008344,
      "grad_norm": 9.045683860778809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08758160471916199,
      "logits/rejected": -0.28914695978164673,
      "logps/chosen": -576.643310546875,
      "logps/rejected": -381.7720642089844,
      "loss": 0.6736,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4789735972881317,
      "rewards/margins": 0.43734490871429443,
      "rewards/rejected": -0.9163185358047485,
      "step": 2175
    },
    {
      "epoch": 0.2847468716774352,
      "grad_norm": 7.891586780548096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2720268964767456,
      "logits/rejected": 0.11657916009426117,
      "logps/chosen": -422.88677978515625,
      "logps/rejected": -498.68902587890625,
      "loss": 0.5846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5384147763252258,
      "rewards/margins": 0.9043492674827576,
      "rewards/rejected": -1.4427639245986938,
      "step": 2176
    },
    {
      "epoch": 0.28487772961478697,
      "grad_norm": 5.764184474945068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1638871431350708,
      "logits/rejected": 0.2587752938270569,
      "logps/chosen": -400.36199951171875,
      "logps/rejected": -340.131591796875,
      "loss": 0.5057,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1308521181344986,
      "rewards/margins": 0.9031540155410767,
      "rewards/rejected": -1.034006118774414,
      "step": 2177
    },
    {
      "epoch": 0.2850085875521387,
      "grad_norm": 8.415152549743652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19029782712459564,
      "logits/rejected": 0.34712105989456177,
      "logps/chosen": -367.130615234375,
      "logps/rejected": -451.70489501953125,
      "loss": 0.741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36719590425491333,
      "rewards/margins": 0.21123051643371582,
      "rewards/rejected": -0.5784264206886292,
      "step": 2178
    },
    {
      "epoch": 0.2851394454894905,
      "grad_norm": 7.489233016967773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11977716535329819,
      "logits/rejected": -0.1402607560157776,
      "logps/chosen": -498.0924987792969,
      "logps/rejected": -395.31134033203125,
      "loss": 0.4716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3595291078090668,
      "rewards/margins": 1.077672004699707,
      "rewards/rejected": -1.4372010231018066,
      "step": 2179
    },
    {
      "epoch": 0.28527030342684223,
      "grad_norm": 8.76475715637207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1080540269613266,
      "logits/rejected": -0.04240594804286957,
      "logps/chosen": -324.8695068359375,
      "logps/rejected": -396.49444580078125,
      "loss": 0.6566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8833349347114563,
      "rewards/margins": 0.4161091446876526,
      "rewards/rejected": -1.2994439601898193,
      "step": 2180
    },
    {
      "epoch": 0.285401161364194,
      "grad_norm": 5.0942206382751465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35911715030670166,
      "logits/rejected": 0.30130890011787415,
      "logps/chosen": -412.4042663574219,
      "logps/rejected": -401.0896911621094,
      "loss": 0.3637,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30289608240127563,
      "rewards/margins": 1.1121150255203247,
      "rewards/rejected": -1.4150112867355347,
      "step": 2181
    },
    {
      "epoch": 0.28553201930154576,
      "grad_norm": 6.445039749145508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4270137846469879,
      "logits/rejected": 0.2276332974433899,
      "logps/chosen": -520.0562133789062,
      "logps/rejected": -371.54241943359375,
      "loss": 0.4407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014847002923488617,
      "rewards/margins": 0.9977440237998962,
      "rewards/rejected": -0.9828970432281494,
      "step": 2182
    },
    {
      "epoch": 0.2856628772388975,
      "grad_norm": 4.535618782043457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2602924108505249,
      "logits/rejected": 0.10439196228981018,
      "logps/chosen": -410.9527587890625,
      "logps/rejected": -422.4874267578125,
      "loss": 0.2791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25034675002098083,
      "rewards/margins": 1.5674289464950562,
      "rewards/rejected": -1.817775845527649,
      "step": 2183
    },
    {
      "epoch": 0.2857937351762493,
      "grad_norm": 9.092500686645508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014066310599446297,
      "logits/rejected": 0.20257924497127533,
      "logps/chosen": -333.111083984375,
      "logps/rejected": -402.82220458984375,
      "loss": 1.1613,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6924245953559875,
      "rewards/margins": -0.2091585099697113,
      "rewards/rejected": -0.48326608538627625,
      "step": 2184
    },
    {
      "epoch": 0.285924593113601,
      "grad_norm": 3.8451993465423584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31304648518562317,
      "logits/rejected": 0.19884471595287323,
      "logps/chosen": -509.6387939453125,
      "logps/rejected": -374.6712646484375,
      "loss": 0.4019,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4976795017719269,
      "rewards/margins": 1.1579947471618652,
      "rewards/rejected": -0.660315215587616,
      "step": 2185
    },
    {
      "epoch": 0.2860554510509528,
      "grad_norm": 5.292064189910889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05741054564714432,
      "logits/rejected": -0.26221027970314026,
      "logps/chosen": -341.9193420410156,
      "logps/rejected": -373.9219970703125,
      "loss": 0.3943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5598185062408447,
      "rewards/margins": 1.1191505193710327,
      "rewards/rejected": -1.678968906402588,
      "step": 2186
    },
    {
      "epoch": 0.28618630898830455,
      "grad_norm": 3.969820261001587,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17499922215938568,
      "logits/rejected": -0.03173109143972397,
      "logps/chosen": -414.3942565917969,
      "logps/rejected": -426.9086608886719,
      "loss": 0.2971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18742084503173828,
      "rewards/margins": 1.4074469804763794,
      "rewards/rejected": -1.5948678255081177,
      "step": 2187
    },
    {
      "epoch": 0.28631716692565634,
      "grad_norm": 6.16651725769043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05517122149467468,
      "logits/rejected": 0.061779849231243134,
      "logps/chosen": -499.3934631347656,
      "logps/rejected": -393.75726318359375,
      "loss": 0.4702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.039450839161872864,
      "rewards/margins": 0.8622194528579712,
      "rewards/rejected": -0.9016703367233276,
      "step": 2188
    },
    {
      "epoch": 0.2864480248630081,
      "grad_norm": 7.379387378692627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03316347301006317,
      "logits/rejected": 0.1583622246980667,
      "logps/chosen": -538.1869506835938,
      "logps/rejected": -401.91766357421875,
      "loss": 0.6442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7558943033218384,
      "rewards/margins": 0.21032045781612396,
      "rewards/rejected": -0.9662147760391235,
      "step": 2189
    },
    {
      "epoch": 0.28657888280035987,
      "grad_norm": 4.116746425628662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1856154352426529,
      "logits/rejected": 0.09792055934667587,
      "logps/chosen": -405.4661865234375,
      "logps/rejected": -340.0787658691406,
      "loss": 0.2739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22184297442436218,
      "rewards/margins": 1.686730146408081,
      "rewards/rejected": -1.464887022972107,
      "step": 2190
    },
    {
      "epoch": 0.2867097407377116,
      "grad_norm": 6.536263942718506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11831571161746979,
      "logits/rejected": 0.05515166372060776,
      "logps/chosen": -287.551513671875,
      "logps/rejected": -375.3687438964844,
      "loss": 0.4315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7407343983650208,
      "rewards/margins": 1.1157256364822388,
      "rewards/rejected": -1.8564600944519043,
      "step": 2191
    },
    {
      "epoch": 0.2868405986750634,
      "grad_norm": 6.396949768066406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11663860827684402,
      "logits/rejected": -0.2751027047634125,
      "logps/chosen": -509.84619140625,
      "logps/rejected": -371.375732421875,
      "loss": 0.4635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007421672344207764,
      "rewards/margins": 0.649588406085968,
      "rewards/rejected": -0.6570100784301758,
      "step": 2192
    },
    {
      "epoch": 0.28697145661241513,
      "grad_norm": 7.914184093475342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002978786826133728,
      "logits/rejected": 0.01685263216495514,
      "logps/chosen": -351.0650634765625,
      "logps/rejected": -271.69158935546875,
      "loss": 0.5034,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.705531895160675,
      "rewards/margins": 0.8795655369758606,
      "rewards/rejected": -1.585097312927246,
      "step": 2193
    },
    {
      "epoch": 0.2871023145497669,
      "grad_norm": 5.336556434631348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23432748019695282,
      "logits/rejected": 0.2452821284532547,
      "logps/chosen": -520.6641845703125,
      "logps/rejected": -481.1087646484375,
      "loss": 0.4446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5373008847236633,
      "rewards/margins": 0.8286145925521851,
      "rewards/rejected": -0.2913137674331665,
      "step": 2194
    },
    {
      "epoch": 0.28723317248711866,
      "grad_norm": 13.198710441589355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09928968548774719,
      "logits/rejected": -0.2952340245246887,
      "logps/chosen": -557.328125,
      "logps/rejected": -462.4851379394531,
      "loss": 1.0166,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6111154556274414,
      "rewards/margins": 0.007836401462554932,
      "rewards/rejected": -0.6189517974853516,
      "step": 2195
    },
    {
      "epoch": 0.28736403042447045,
      "grad_norm": 6.59407377243042,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003967598080635071,
      "logits/rejected": -0.0229770690202713,
      "logps/chosen": -376.78839111328125,
      "logps/rejected": -402.4932861328125,
      "loss": 0.3681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23689113557338715,
      "rewards/margins": 1.0239839553833008,
      "rewards/rejected": -1.2608752250671387,
      "step": 2196
    },
    {
      "epoch": 0.2874948883618222,
      "grad_norm": 6.726519584655762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05615891516208649,
      "logits/rejected": 0.1583116054534912,
      "logps/chosen": -282.2152404785156,
      "logps/rejected": -414.62664794921875,
      "loss": 0.4838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7723409533500671,
      "rewards/margins": 0.6029127240180969,
      "rewards/rejected": -1.375253677368164,
      "step": 2197
    },
    {
      "epoch": 0.287625746299174,
      "grad_norm": 8.233908653259277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0216725692152977,
      "logits/rejected": 0.1556376963853836,
      "logps/chosen": -319.9973449707031,
      "logps/rejected": -468.4678039550781,
      "loss": 0.722,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19845160841941833,
      "rewards/margins": 0.17077142000198364,
      "rewards/rejected": -0.36922305822372437,
      "step": 2198
    },
    {
      "epoch": 0.2877566042365257,
      "grad_norm": 4.8767313957214355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15794196724891663,
      "logits/rejected": -0.1548725813627243,
      "logps/chosen": -378.06048583984375,
      "logps/rejected": -470.1496887207031,
      "loss": 0.3895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21820278465747833,
      "rewards/margins": 1.1394574642181396,
      "rewards/rejected": -0.9212547540664673,
      "step": 2199
    },
    {
      "epoch": 0.2878874621738775,
      "grad_norm": 9.31637954711914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11117863655090332,
      "logits/rejected": -0.05344328656792641,
      "logps/chosen": -496.3130798339844,
      "logps/rejected": -590.7943725585938,
      "loss": 0.6773,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.23409903049468994,
      "rewards/margins": 0.2960599660873413,
      "rewards/rejected": -0.530159056186676,
      "step": 2200
    },
    {
      "epoch": 0.28801832011122924,
      "grad_norm": 6.1707305908203125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17291800677776337,
      "logits/rejected": 0.09043137729167938,
      "logps/chosen": -380.43328857421875,
      "logps/rejected": -470.2289733886719,
      "loss": 0.4303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0917326956987381,
      "rewards/margins": 1.1215039491653442,
      "rewards/rejected": -1.0297712087631226,
      "step": 2201
    },
    {
      "epoch": 0.28814917804858103,
      "grad_norm": 6.7501654624938965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11308856308460236,
      "logits/rejected": 0.060900986194610596,
      "logps/chosen": -460.3108825683594,
      "logps/rejected": -374.0397644042969,
      "loss": 0.5884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3160894215106964,
      "rewards/margins": 0.7378557324409485,
      "rewards/rejected": -1.0539450645446777,
      "step": 2202
    },
    {
      "epoch": 0.28828003598593277,
      "grad_norm": 8.32979965209961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1603117138147354,
      "logits/rejected": -0.0848398506641388,
      "logps/chosen": -395.3989562988281,
      "logps/rejected": -432.5920715332031,
      "loss": 0.6611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8913005590438843,
      "rewards/margins": 0.4127543568611145,
      "rewards/rejected": -1.3040549755096436,
      "step": 2203
    },
    {
      "epoch": 0.28841089392328456,
      "grad_norm": 9.657944679260254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026724040508270264,
      "logits/rejected": 0.05966857075691223,
      "logps/chosen": -438.3959045410156,
      "logps/rejected": -413.5133972167969,
      "loss": 1.0204,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25765460729599,
      "rewards/margins": 0.19234788417816162,
      "rewards/rejected": -0.4500024914741516,
      "step": 2204
    },
    {
      "epoch": 0.2885417518606363,
      "grad_norm": 16.370685577392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12444014847278595,
      "logits/rejected": -0.15095314383506775,
      "logps/chosen": -657.7989501953125,
      "logps/rejected": -514.7294921875,
      "loss": 1.2378,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2386555671691895,
      "rewards/margins": 0.12413090467453003,
      "rewards/rejected": -1.3627864122390747,
      "step": 2205
    },
    {
      "epoch": 0.28867260979798803,
      "grad_norm": 5.406253814697266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05828423053026199,
      "logits/rejected": -0.19387264549732208,
      "logps/chosen": -583.6541137695312,
      "logps/rejected": -732.9383544921875,
      "loss": 0.3867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25884702801704407,
      "rewards/margins": 1.2764904499053955,
      "rewards/rejected": -1.5353374481201172,
      "step": 2206
    },
    {
      "epoch": 0.2888034677353398,
      "grad_norm": 7.434749603271484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2012217789888382,
      "logits/rejected": -0.14312878251075745,
      "logps/chosen": -349.21392822265625,
      "logps/rejected": -289.1911315917969,
      "loss": 0.645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4303244948387146,
      "rewards/margins": 0.323957234621048,
      "rewards/rejected": -0.7542818188667297,
      "step": 2207
    },
    {
      "epoch": 0.28893432567269156,
      "grad_norm": 7.983161926269531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15237711369991302,
      "logits/rejected": -0.10870352387428284,
      "logps/chosen": -338.1285705566406,
      "logps/rejected": -338.5718994140625,
      "loss": 0.5864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9863634705543518,
      "rewards/margins": 0.36306247115135193,
      "rewards/rejected": -1.3494259119033813,
      "step": 2208
    },
    {
      "epoch": 0.28906518361004335,
      "grad_norm": 5.960695266723633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23796258866786957,
      "logits/rejected": 0.1325984001159668,
      "logps/chosen": -553.0094604492188,
      "logps/rejected": -426.66314697265625,
      "loss": 0.3018,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.545941174030304,
      "rewards/margins": 1.4774950742721558,
      "rewards/rejected": -2.0234360694885254,
      "step": 2209
    },
    {
      "epoch": 0.2891960415473951,
      "grad_norm": 9.1006441116333,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24960564076900482,
      "logits/rejected": -0.22980129718780518,
      "logps/chosen": -506.528076171875,
      "logps/rejected": -374.3303527832031,
      "loss": 0.9664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1520915031433105,
      "rewards/margins": -0.05986250936985016,
      "rewards/rejected": -1.0922290086746216,
      "step": 2210
    },
    {
      "epoch": 0.2893268994847469,
      "grad_norm": 7.952341556549072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17456239461898804,
      "logits/rejected": -0.31714969873428345,
      "logps/chosen": -399.71466064453125,
      "logps/rejected": -440.20361328125,
      "loss": 0.4975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8572269678115845,
      "rewards/margins": 0.9171693325042725,
      "rewards/rejected": -1.774396300315857,
      "step": 2211
    },
    {
      "epoch": 0.2894577574220986,
      "grad_norm": 9.909565925598145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29837799072265625,
      "logits/rejected": -0.11449234187602997,
      "logps/chosen": -350.4083557128906,
      "logps/rejected": -314.84765625,
      "loss": 0.7957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6761107444763184,
      "rewards/margins": 0.091898612678051,
      "rewards/rejected": -0.7680094242095947,
      "step": 2212
    },
    {
      "epoch": 0.2895886153594504,
      "grad_norm": 6.1706109046936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1690315455198288,
      "logits/rejected": -0.01409342885017395,
      "logps/chosen": -515.1614379882812,
      "logps/rejected": -537.2359619140625,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3399587869644165,
      "rewards/margins": 0.6871688365936279,
      "rewards/rejected": -1.0271276235580444,
      "step": 2213
    },
    {
      "epoch": 0.28971947329680214,
      "grad_norm": 7.091423511505127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03530491143465042,
      "logits/rejected": -0.1762937754392624,
      "logps/chosen": -619.6622314453125,
      "logps/rejected": -414.1159973144531,
      "loss": 0.3416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01017361506819725,
      "rewards/margins": 1.7526741027832031,
      "rewards/rejected": -1.7628475427627563,
      "step": 2214
    },
    {
      "epoch": 0.28985033123415394,
      "grad_norm": 6.383253574371338,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005310762673616409,
      "logits/rejected": 0.0381113663315773,
      "logps/chosen": -488.6088562011719,
      "logps/rejected": -561.677734375,
      "loss": 0.4588,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34069234132766724,
      "rewards/margins": 0.9975923299789429,
      "rewards/rejected": -1.3382847309112549,
      "step": 2215
    },
    {
      "epoch": 0.28998118917150567,
      "grad_norm": 4.918972492218018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07311567664146423,
      "logits/rejected": -0.10276515036821365,
      "logps/chosen": -467.9642333984375,
      "logps/rejected": -436.2393798828125,
      "loss": 0.3259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6396230459213257,
      "rewards/margins": 1.1139452457427979,
      "rewards/rejected": -1.753568410873413,
      "step": 2216
    },
    {
      "epoch": 0.29011204710885746,
      "grad_norm": 11.202954292297363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0759715586900711,
      "logits/rejected": -0.1736394166946411,
      "logps/chosen": -439.6034240722656,
      "logps/rejected": -352.8001708984375,
      "loss": 0.6677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7268781661987305,
      "rewards/margins": 0.4665752351284027,
      "rewards/rejected": -1.193453311920166,
      "step": 2217
    },
    {
      "epoch": 0.2902429050462092,
      "grad_norm": 3.798187732696533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18972517549991608,
      "logits/rejected": -0.16407358646392822,
      "logps/chosen": -406.7964782714844,
      "logps/rejected": -411.36053466796875,
      "loss": 0.2371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47837433218955994,
      "rewards/margins": 2.067030906677246,
      "rewards/rejected": -1.5886566638946533,
      "step": 2218
    },
    {
      "epoch": 0.290373762983561,
      "grad_norm": 10.288925170898438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2610730528831482,
      "logits/rejected": -0.24854198098182678,
      "logps/chosen": -514.8521118164062,
      "logps/rejected": -368.74346923828125,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5093351602554321,
      "rewards/margins": 0.42940446734428406,
      "rewards/rejected": -0.938739538192749,
      "step": 2219
    },
    {
      "epoch": 0.2905046209209127,
      "grad_norm": 3.9778707027435303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13216263055801392,
      "logits/rejected": -0.02564729005098343,
      "logps/chosen": -269.2370300292969,
      "logps/rejected": -346.2915954589844,
      "loss": 0.314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13637703657150269,
      "rewards/margins": 1.4085825681686401,
      "rewards/rejected": -1.2722055912017822,
      "step": 2220
    },
    {
      "epoch": 0.2906354788582645,
      "grad_norm": 10.61077880859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12176486104726791,
      "logits/rejected": -0.06459442526102066,
      "logps/chosen": -476.9279479980469,
      "logps/rejected": -448.5723571777344,
      "loss": 0.9407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7913967370986938,
      "rewards/margins": -0.0627388209104538,
      "rewards/rejected": -0.7286579608917236,
      "step": 2221
    },
    {
      "epoch": 0.29076633679561625,
      "grad_norm": 7.143133163452148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10704721510410309,
      "logits/rejected": 0.232070192694664,
      "logps/chosen": -460.22015380859375,
      "logps/rejected": -367.1156921386719,
      "loss": 0.578,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014738164842128754,
      "rewards/margins": 0.8850638270378113,
      "rewards/rejected": -0.8703257441520691,
      "step": 2222
    },
    {
      "epoch": 0.29089719473296805,
      "grad_norm": 6.683010578155518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05424008518457413,
      "logits/rejected": -0.10655437409877777,
      "logps/chosen": -403.80535888671875,
      "logps/rejected": -428.5014953613281,
      "loss": 0.4013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11159513890743256,
      "rewards/margins": 1.5556364059448242,
      "rewards/rejected": -1.4440412521362305,
      "step": 2223
    },
    {
      "epoch": 0.2910280526703198,
      "grad_norm": 7.74416446685791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07452481240034103,
      "logits/rejected": 0.016964582726359367,
      "logps/chosen": -471.08740234375,
      "logps/rejected": -547.9736328125,
      "loss": 0.6338,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07822686433792114,
      "rewards/margins": 0.8023352026939392,
      "rewards/rejected": -0.7241083383560181,
      "step": 2224
    },
    {
      "epoch": 0.2911589106076716,
      "grad_norm": 9.484901428222656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2884812653064728,
      "logits/rejected": 0.17363247275352478,
      "logps/chosen": -439.88665771484375,
      "logps/rejected": -299.9471435546875,
      "loss": 0.6813,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39926186203956604,
      "rewards/margins": 0.32514020800590515,
      "rewards/rejected": -0.7244020700454712,
      "step": 2225
    },
    {
      "epoch": 0.2912897685450233,
      "grad_norm": 6.65259313583374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22887544333934784,
      "logits/rejected": -0.06408018618822098,
      "logps/chosen": -594.7940673828125,
      "logps/rejected": -352.3218994140625,
      "loss": 0.3963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13382530212402344,
      "rewards/margins": 0.9787594079971313,
      "rewards/rejected": -1.1125845909118652,
      "step": 2226
    },
    {
      "epoch": 0.29142062648237504,
      "grad_norm": 6.79798698425293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21986663341522217,
      "logits/rejected": 0.03511359915137291,
      "logps/chosen": -496.26519775390625,
      "logps/rejected": -376.58819580078125,
      "loss": 0.5485,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5429395437240601,
      "rewards/margins": 0.7000587582588196,
      "rewards/rejected": -1.2429983615875244,
      "step": 2227
    },
    {
      "epoch": 0.29155148441972684,
      "grad_norm": 6.762449264526367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39478129148483276,
      "logits/rejected": 0.10592266917228699,
      "logps/chosen": -594.7025756835938,
      "logps/rejected": -463.88299560546875,
      "loss": 0.4738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45385271310806274,
      "rewards/margins": 0.6827967762947083,
      "rewards/rejected": -0.22894403338432312,
      "step": 2228
    },
    {
      "epoch": 0.2916823423570786,
      "grad_norm": 11.616129875183105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004839159548282623,
      "logits/rejected": 0.07617993652820587,
      "logps/chosen": -530.523681640625,
      "logps/rejected": -445.93017578125,
      "loss": 0.6809,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0253748893737793,
      "rewards/margins": 0.35188019275665283,
      "rewards/rejected": -1.3772549629211426,
      "step": 2229
    },
    {
      "epoch": 0.29181320029443036,
      "grad_norm": 9.086713790893555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2979889512062073,
      "logits/rejected": 0.21783584356307983,
      "logps/chosen": -372.112060546875,
      "logps/rejected": -405.6168212890625,
      "loss": 0.8053,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.185139462351799,
      "rewards/margins": 0.0907709002494812,
      "rewards/rejected": 0.09436856210231781,
      "step": 2230
    },
    {
      "epoch": 0.2919440582317821,
      "grad_norm": 5.278213977813721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19247029721736908,
      "logits/rejected": 0.19878144562244415,
      "logps/chosen": -399.5706787109375,
      "logps/rejected": -431.25958251953125,
      "loss": 0.4551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5555450320243835,
      "rewards/margins": 0.8871074914932251,
      "rewards/rejected": -0.33156242966651917,
      "step": 2231
    },
    {
      "epoch": 0.2920749161691339,
      "grad_norm": 6.960874557495117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02653665840625763,
      "logits/rejected": 0.03568859025835991,
      "logps/chosen": -412.1590270996094,
      "logps/rejected": -448.8432312011719,
      "loss": 0.4701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.269156813621521,
      "rewards/margins": 0.8150309324264526,
      "rewards/rejected": -1.084187626838684,
      "step": 2232
    },
    {
      "epoch": 0.2922057741064856,
      "grad_norm": 5.660645008087158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0845559686422348,
      "logits/rejected": 0.1485382616519928,
      "logps/chosen": -361.61151123046875,
      "logps/rejected": -363.563232421875,
      "loss": 0.411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8796026706695557,
      "rewards/margins": 1.0038903951644897,
      "rewards/rejected": -1.8834930658340454,
      "step": 2233
    },
    {
      "epoch": 0.2923366320438374,
      "grad_norm": 6.72391414642334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15184299647808075,
      "logits/rejected": -0.26297301054000854,
      "logps/chosen": -433.14361572265625,
      "logps/rejected": -340.7162780761719,
      "loss": 0.4824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2901384234428406,
      "rewards/margins": 0.9784847497940063,
      "rewards/rejected": -0.6883463263511658,
      "step": 2234
    },
    {
      "epoch": 0.29246748998118915,
      "grad_norm": 4.7140984535217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1838465929031372,
      "logits/rejected": -0.03898908197879791,
      "logps/chosen": -354.88726806640625,
      "logps/rejected": -251.6411895751953,
      "loss": 0.4836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.434234619140625,
      "rewards/margins": 0.8037471771240234,
      "rewards/rejected": -1.2379817962646484,
      "step": 2235
    },
    {
      "epoch": 0.29259834791854095,
      "grad_norm": 7.601996898651123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22497017681598663,
      "logits/rejected": -0.07348393648862839,
      "logps/chosen": -387.58282470703125,
      "logps/rejected": -424.1435546875,
      "loss": 0.5767,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08211889863014221,
      "rewards/margins": 0.9227128028869629,
      "rewards/rejected": -0.8405938744544983,
      "step": 2236
    },
    {
      "epoch": 0.2927292058558927,
      "grad_norm": 6.913957118988037,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03462732210755348,
      "logits/rejected": 0.015607748180627823,
      "logps/chosen": -457.1960754394531,
      "logps/rejected": -488.5401611328125,
      "loss": 0.4737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2837022840976715,
      "rewards/margins": 0.792100727558136,
      "rewards/rejected": -1.0758030414581299,
      "step": 2237
    },
    {
      "epoch": 0.2928600637932445,
      "grad_norm": 10.603025436401367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09113021194934845,
      "logits/rejected": 0.05161839723587036,
      "logps/chosen": -498.1731872558594,
      "logps/rejected": -468.9918212890625,
      "loss": 0.9338,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13981474936008453,
      "rewards/margins": -0.17391662299633026,
      "rewards/rejected": 0.034101858735084534,
      "step": 2238
    },
    {
      "epoch": 0.2929909217305962,
      "grad_norm": 5.515004634857178,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035017069429159164,
      "logits/rejected": -0.06276898086071014,
      "logps/chosen": -292.6567077636719,
      "logps/rejected": -330.05535888671875,
      "loss": 0.5002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1486726701259613,
      "rewards/margins": 1.06914484500885,
      "rewards/rejected": -1.2178174257278442,
      "step": 2239
    },
    {
      "epoch": 0.293121779667948,
      "grad_norm": 4.918887138366699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31568339467048645,
      "logits/rejected": -0.14764878153800964,
      "logps/chosen": -340.9642333984375,
      "logps/rejected": -372.9765319824219,
      "loss": 0.4555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027276046574115753,
      "rewards/margins": 1.0604079961776733,
      "rewards/rejected": -1.087684154510498,
      "step": 2240
    },
    {
      "epoch": 0.29325263760529974,
      "grad_norm": 7.205976486206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28132471442222595,
      "logits/rejected": 0.10799916088581085,
      "logps/chosen": -462.26214599609375,
      "logps/rejected": -434.1302185058594,
      "loss": 0.5806,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00613364577293396,
      "rewards/margins": 0.39462414383888245,
      "rewards/rejected": -0.3884904980659485,
      "step": 2241
    },
    {
      "epoch": 0.29338349554265153,
      "grad_norm": 6.13863468170166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23336246609687805,
      "logits/rejected": 0.33517831563949585,
      "logps/chosen": -557.9865112304688,
      "logps/rejected": -544.1611938476562,
      "loss": 0.3432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09729444980621338,
      "rewards/margins": 1.4707154035568237,
      "rewards/rejected": -1.568009853363037,
      "step": 2242
    },
    {
      "epoch": 0.29351435348000326,
      "grad_norm": 8.17949390411377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10106729716062546,
      "logits/rejected": -0.0785006731748581,
      "logps/chosen": -306.02032470703125,
      "logps/rejected": -272.4794006347656,
      "loss": 0.7137,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8188438415527344,
      "rewards/margins": 0.017381496727466583,
      "rewards/rejected": -0.8362253308296204,
      "step": 2243
    },
    {
      "epoch": 0.29364521141735506,
      "grad_norm": 4.40870475769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09349691867828369,
      "logits/rejected": 0.08107782900333405,
      "logps/chosen": -294.289306640625,
      "logps/rejected": -275.107666015625,
      "loss": 0.4852,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.437438040971756,
      "rewards/margins": 0.5588569641113281,
      "rewards/rejected": -0.9962950348854065,
      "step": 2244
    },
    {
      "epoch": 0.2937760693547068,
      "grad_norm": 7.113984107971191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05949259549379349,
      "logits/rejected": 0.14896683394908905,
      "logps/chosen": -415.96160888671875,
      "logps/rejected": -406.5390625,
      "loss": 0.6485,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7951250076293945,
      "rewards/margins": 0.33209723234176636,
      "rewards/rejected": -1.1272222995758057,
      "step": 2245
    },
    {
      "epoch": 0.2939069272920586,
      "grad_norm": 6.237410545349121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053583525121212006,
      "logits/rejected": -0.08147180080413818,
      "logps/chosen": -500.21563720703125,
      "logps/rejected": -436.4042053222656,
      "loss": 0.4616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08878135681152344,
      "rewards/margins": 0.7771263122558594,
      "rewards/rejected": -0.6883449554443359,
      "step": 2246
    },
    {
      "epoch": 0.2940377852294103,
      "grad_norm": 10.088470458984375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10569042712450027,
      "logits/rejected": 0.11860060691833496,
      "logps/chosen": -456.7132568359375,
      "logps/rejected": -508.21337890625,
      "loss": 0.9151,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5711782574653625,
      "rewards/margins": 0.2898557186126709,
      "rewards/rejected": -0.8610340356826782,
      "step": 2247
    },
    {
      "epoch": 0.2941686431667621,
      "grad_norm": 7.548771381378174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07400193810462952,
      "logits/rejected": -0.039738643914461136,
      "logps/chosen": -411.16937255859375,
      "logps/rejected": -578.052490234375,
      "loss": 0.5723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12598493695259094,
      "rewards/margins": 0.5595299601554871,
      "rewards/rejected": -0.43354493379592896,
      "step": 2248
    },
    {
      "epoch": 0.29429950110411385,
      "grad_norm": 6.9607038497924805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4614107012748718,
      "logits/rejected": -0.22930535674095154,
      "logps/chosen": -425.5916748046875,
      "logps/rejected": -534.667236328125,
      "loss": 0.5229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32722002267837524,
      "rewards/margins": 1.1555436849594116,
      "rewards/rejected": -1.4827637672424316,
      "step": 2249
    },
    {
      "epoch": 0.2944303590414656,
      "grad_norm": 6.06654167175293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3648085594177246,
      "logits/rejected": 0.1983097791671753,
      "logps/chosen": -363.45355224609375,
      "logps/rejected": -401.358642578125,
      "loss": 0.4869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6267374157905579,
      "rewards/margins": 0.6745689511299133,
      "rewards/rejected": -1.3013063669204712,
      "step": 2250
    },
    {
      "epoch": 0.2945612169788174,
      "grad_norm": 6.547889232635498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1541597843170166,
      "logits/rejected": 0.014872871339321136,
      "logps/chosen": -532.721435546875,
      "logps/rejected": -546.6006469726562,
      "loss": 0.4602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44001486897468567,
      "rewards/margins": 0.8489696383476257,
      "rewards/rejected": -1.2889844179153442,
      "step": 2251
    },
    {
      "epoch": 0.2946920749161691,
      "grad_norm": 4.073923110961914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3966560959815979,
      "logits/rejected": 0.1449088305234909,
      "logps/chosen": -542.8139038085938,
      "logps/rejected": -441.1715393066406,
      "loss": 0.2656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4864940643310547,
      "rewards/margins": 1.9036192893981934,
      "rewards/rejected": -1.4171252250671387,
      "step": 2252
    },
    {
      "epoch": 0.2948229328535209,
      "grad_norm": 10.23096752166748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10507611185312271,
      "logits/rejected": 0.27171653509140015,
      "logps/chosen": -522.6826782226562,
      "logps/rejected": -568.7308959960938,
      "loss": 0.9444,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6885800957679749,
      "rewards/margins": -0.24930039048194885,
      "rewards/rejected": -0.4392797350883484,
      "step": 2253
    },
    {
      "epoch": 0.29495379079087264,
      "grad_norm": 8.43334674835205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0929272323846817,
      "logits/rejected": 0.14783959090709686,
      "logps/chosen": -567.99609375,
      "logps/rejected": -581.0448608398438,
      "loss": 0.499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24862441420555115,
      "rewards/margins": 1.0300740003585815,
      "rewards/rejected": -0.7814494967460632,
      "step": 2254
    },
    {
      "epoch": 0.29508464872822443,
      "grad_norm": 6.955740928649902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15445539355278015,
      "logits/rejected": 0.021183187142014503,
      "logps/chosen": -403.3700866699219,
      "logps/rejected": -522.6607055664062,
      "loss": 0.4969,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1636073887348175,
      "rewards/margins": 0.6674711108207703,
      "rewards/rejected": -0.5038636922836304,
      "step": 2255
    },
    {
      "epoch": 0.29521550666557617,
      "grad_norm": 10.147948265075684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24254629015922546,
      "logits/rejected": 0.31709349155426025,
      "logps/chosen": -543.884765625,
      "logps/rejected": -601.9262084960938,
      "loss": 0.7371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5371800065040588,
      "rewards/margins": 0.335044801235199,
      "rewards/rejected": -0.8722248077392578,
      "step": 2256
    },
    {
      "epoch": 0.29534636460292796,
      "grad_norm": 8.364324569702148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13364943861961365,
      "logits/rejected": 0.13626447319984436,
      "logps/chosen": -684.2274169921875,
      "logps/rejected": -646.3283081054688,
      "loss": 0.5544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0885368138551712,
      "rewards/margins": 0.8609548807144165,
      "rewards/rejected": -0.7724180221557617,
      "step": 2257
    },
    {
      "epoch": 0.2954772225402797,
      "grad_norm": 10.39920711517334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060123052448034286,
      "logits/rejected": 0.05761665478348732,
      "logps/chosen": -541.5992431640625,
      "logps/rejected": -329.3829040527344,
      "loss": 1.0174,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3207801580429077,
      "rewards/margins": -0.25350260734558105,
      "rewards/rejected": -1.0672775506973267,
      "step": 2258
    },
    {
      "epoch": 0.2956080804776315,
      "grad_norm": 7.387092590332031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06688389182090759,
      "logits/rejected": -0.05643889307975769,
      "logps/chosen": -480.913330078125,
      "logps/rejected": -427.068603515625,
      "loss": 0.5385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8186129331588745,
      "rewards/margins": 0.5094997882843018,
      "rewards/rejected": -1.3281126022338867,
      "step": 2259
    },
    {
      "epoch": 0.2957389384149832,
      "grad_norm": 5.140437602996826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4450431168079376,
      "logits/rejected": 0.4610058665275574,
      "logps/chosen": -676.7327880859375,
      "logps/rejected": -529.3829956054688,
      "loss": 0.352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45643746852874756,
      "rewards/margins": 1.1747267246246338,
      "rewards/rejected": -0.7182891368865967,
      "step": 2260
    },
    {
      "epoch": 0.295869796352335,
      "grad_norm": 4.342089653015137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17562393844127655,
      "logits/rejected": 0.3112756907939911,
      "logps/chosen": -448.95062255859375,
      "logps/rejected": -614.4108276367188,
      "loss": 0.3451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23977774381637573,
      "rewards/margins": 1.6529262065887451,
      "rewards/rejected": -1.413148283958435,
      "step": 2261
    },
    {
      "epoch": 0.29600065428968675,
      "grad_norm": 5.84218692779541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1846398115158081,
      "logits/rejected": 0.13478893041610718,
      "logps/chosen": -414.6101379394531,
      "logps/rejected": -433.5953369140625,
      "loss": 0.4504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19763509929180145,
      "rewards/margins": 1.2202378511428833,
      "rewards/rejected": -1.0226027965545654,
      "step": 2262
    },
    {
      "epoch": 0.29613151222703854,
      "grad_norm": 4.728266716003418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09486648440361023,
      "logits/rejected": -0.14463607966899872,
      "logps/chosen": -584.883544921875,
      "logps/rejected": -525.6052856445312,
      "loss": 0.3225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22026118636131287,
      "rewards/margins": 1.1506683826446533,
      "rewards/rejected": -0.9304071664810181,
      "step": 2263
    },
    {
      "epoch": 0.2962623701643903,
      "grad_norm": 8.887621879577637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23709118366241455,
      "logits/rejected": -0.019702551886439323,
      "logps/chosen": -445.2733459472656,
      "logps/rejected": -335.30926513671875,
      "loss": 0.8666,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.72859787940979,
      "rewards/margins": 0.054808422923088074,
      "rewards/rejected": -0.7834063172340393,
      "step": 2264
    },
    {
      "epoch": 0.29639322810174207,
      "grad_norm": 40.5976448059082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04861854016780853,
      "logits/rejected": -0.12042152136564255,
      "logps/chosen": -436.2603759765625,
      "logps/rejected": -426.9544372558594,
      "loss": 0.8215,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1551824808120728,
      "rewards/margins": 0.008713580667972565,
      "rewards/rejected": -1.1638959646224976,
      "step": 2265
    },
    {
      "epoch": 0.2965240860390938,
      "grad_norm": 5.986143589019775,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06417093425989151,
      "logits/rejected": 0.1396816372871399,
      "logps/chosen": -590.2628173828125,
      "logps/rejected": -481.31787109375,
      "loss": 0.4995,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06359003484249115,
      "rewards/margins": 0.5845283269882202,
      "rewards/rejected": -0.5209383368492126,
      "step": 2266
    },
    {
      "epoch": 0.2966549439764456,
      "grad_norm": 8.352303504943848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26405954360961914,
      "logits/rejected": -0.10713553428649902,
      "logps/chosen": -449.177734375,
      "logps/rejected": -465.0136413574219,
      "loss": 0.4413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5126476287841797,
      "rewards/margins": 1.3499548435211182,
      "rewards/rejected": -1.8626024723052979,
      "step": 2267
    },
    {
      "epoch": 0.29678580191379733,
      "grad_norm": 10.132203102111816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2034498006105423,
      "logits/rejected": -0.06390316039323807,
      "logps/chosen": -484.4276123046875,
      "logps/rejected": -397.010498046875,
      "loss": 0.7656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4636921286582947,
      "rewards/margins": 0.203831285238266,
      "rewards/rejected": -0.6675233840942383,
      "step": 2268
    },
    {
      "epoch": 0.2969166598511491,
      "grad_norm": 7.027990341186523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11329001188278198,
      "logits/rejected": 0.15884611010551453,
      "logps/chosen": -486.072265625,
      "logps/rejected": -462.30889892578125,
      "loss": 0.4523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07005655020475388,
      "rewards/margins": 1.183677315711975,
      "rewards/rejected": -1.2537338733673096,
      "step": 2269
    },
    {
      "epoch": 0.29704751778850086,
      "grad_norm": 7.947591781616211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15280775725841522,
      "logits/rejected": 0.04138092324137688,
      "logps/chosen": -369.8772888183594,
      "logps/rejected": -420.52349853515625,
      "loss": 0.5865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06707057356834412,
      "rewards/margins": 0.7163586020469666,
      "rewards/rejected": -0.7834291458129883,
      "step": 2270
    },
    {
      "epoch": 0.2971783757258526,
      "grad_norm": 8.246246337890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24126651883125305,
      "logits/rejected": 0.5151239037513733,
      "logps/chosen": -574.8272705078125,
      "logps/rejected": -704.5281982421875,
      "loss": 0.5605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1584976315498352,
      "rewards/margins": 0.8460788726806641,
      "rewards/rejected": -0.6875812411308289,
      "step": 2271
    },
    {
      "epoch": 0.2973092336632044,
      "grad_norm": 6.617557525634766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14333488047122955,
      "logits/rejected": 0.03888951614499092,
      "logps/chosen": -520.0299072265625,
      "logps/rejected": -436.9031066894531,
      "loss": 0.4224,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35539838671684265,
      "rewards/margins": 0.9922477006912231,
      "rewards/rejected": -0.6368494033813477,
      "step": 2272
    },
    {
      "epoch": 0.2974400916005561,
      "grad_norm": 7.896450996398926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2599647045135498,
      "logits/rejected": -0.03573384881019592,
      "logps/chosen": -652.6046142578125,
      "logps/rejected": -496.9161376953125,
      "loss": 0.5505,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4230871796607971,
      "rewards/margins": 0.5148942470550537,
      "rewards/rejected": -0.9379814863204956,
      "step": 2273
    },
    {
      "epoch": 0.2975709495379079,
      "grad_norm": 9.949034690856934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11662845313549042,
      "logits/rejected": 0.08296472579240799,
      "logps/chosen": -479.4234619140625,
      "logps/rejected": -376.16619873046875,
      "loss": 0.6773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9171314835548401,
      "rewards/margins": 0.5201536417007446,
      "rewards/rejected": -1.4372851848602295,
      "step": 2274
    },
    {
      "epoch": 0.29770180747525965,
      "grad_norm": 5.770741939544678,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10439996421337128,
      "logits/rejected": -0.2468053102493286,
      "logps/chosen": -495.33721923828125,
      "logps/rejected": -355.67742919921875,
      "loss": 0.5041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7288064956665039,
      "rewards/margins": 0.6667631268501282,
      "rewards/rejected": -1.3955695629119873,
      "step": 2275
    },
    {
      "epoch": 0.29783266541261144,
      "grad_norm": 7.96417760848999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20656311511993408,
      "logits/rejected": 0.14917080104351044,
      "logps/chosen": -456.67816162109375,
      "logps/rejected": -521.8486328125,
      "loss": 0.602,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.224543958902359,
      "rewards/margins": 0.6317211389541626,
      "rewards/rejected": -0.8562650680541992,
      "step": 2276
    },
    {
      "epoch": 0.2979635233499632,
      "grad_norm": 4.312448024749756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13641522824764252,
      "logits/rejected": -0.1963651180267334,
      "logps/chosen": -345.0847473144531,
      "logps/rejected": -353.68890380859375,
      "loss": 0.4325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32025623321533203,
      "rewards/margins": 0.8723526000976562,
      "rewards/rejected": -1.1926088333129883,
      "step": 2277
    },
    {
      "epoch": 0.29809438128731497,
      "grad_norm": 4.780266284942627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057610634714365005,
      "logits/rejected": -0.027472708374261856,
      "logps/chosen": -369.93951416015625,
      "logps/rejected": -476.21173095703125,
      "loss": 0.2939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4150732159614563,
      "rewards/margins": 1.741542100906372,
      "rewards/rejected": -1.3264687061309814,
      "step": 2278
    },
    {
      "epoch": 0.2982252392246667,
      "grad_norm": 6.159212589263916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19632264971733093,
      "logits/rejected": -0.015522666275501251,
      "logps/chosen": -467.3712158203125,
      "logps/rejected": -454.8441162109375,
      "loss": 0.4561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8596130609512329,
      "rewards/margins": 0.681317150592804,
      "rewards/rejected": -1.540930151939392,
      "step": 2279
    },
    {
      "epoch": 0.2983560971620185,
      "grad_norm": 7.469109058380127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12639449536800385,
      "logits/rejected": 0.0759817510843277,
      "logps/chosen": -523.2659301757812,
      "logps/rejected": -395.16778564453125,
      "loss": 0.4867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5469099283218384,
      "rewards/margins": 0.8765337467193604,
      "rewards/rejected": -1.4234436750411987,
      "step": 2280
    },
    {
      "epoch": 0.29848695509937023,
      "grad_norm": 6.2584428787231445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44973790645599365,
      "logits/rejected": 0.3348659574985504,
      "logps/chosen": -480.94549560546875,
      "logps/rejected": -323.8564758300781,
      "loss": 0.4278,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39354175329208374,
      "rewards/margins": 0.8914625644683838,
      "rewards/rejected": -0.49792081117630005,
      "step": 2281
    },
    {
      "epoch": 0.298617813036722,
      "grad_norm": 4.1505327224731445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3729214370250702,
      "logits/rejected": 0.07761714607477188,
      "logps/chosen": -569.5950317382812,
      "logps/rejected": -383.0777587890625,
      "loss": 0.3594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2077951729297638,
      "rewards/margins": 1.0983121395111084,
      "rewards/rejected": -0.890516996383667,
      "step": 2282
    },
    {
      "epoch": 0.29874867097407376,
      "grad_norm": 7.3942742347717285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20815886557102203,
      "logits/rejected": -0.4489808976650238,
      "logps/chosen": -614.9678955078125,
      "logps/rejected": -502.18603515625,
      "loss": 0.4396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2807396650314331,
      "rewards/margins": 1.0073752403259277,
      "rewards/rejected": -1.2881150245666504,
      "step": 2283
    },
    {
      "epoch": 0.29887952891142555,
      "grad_norm": 9.067710876464844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17443904280662537,
      "logits/rejected": 0.14803579449653625,
      "logps/chosen": -365.9857177734375,
      "logps/rejected": -405.6343688964844,
      "loss": 0.484,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9144281148910522,
      "rewards/margins": 0.9245067834854126,
      "rewards/rejected": -1.8389348983764648,
      "step": 2284
    },
    {
      "epoch": 0.2990103868487773,
      "grad_norm": 5.394237518310547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016021542251110077,
      "logits/rejected": 0.062064409255981445,
      "logps/chosen": -425.32562255859375,
      "logps/rejected": -558.2207641601562,
      "loss": 0.3913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06969986855983734,
      "rewards/margins": 0.9486945867538452,
      "rewards/rejected": -1.0183944702148438,
      "step": 2285
    },
    {
      "epoch": 0.2991412447861291,
      "grad_norm": 8.35072135925293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03890407457947731,
      "logits/rejected": -0.05645297095179558,
      "logps/chosen": -440.5453796386719,
      "logps/rejected": -453.197998046875,
      "loss": 0.4902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14208602905273438,
      "rewards/margins": 1.3885011672973633,
      "rewards/rejected": -1.5305871963500977,
      "step": 2286
    },
    {
      "epoch": 0.2992721027234808,
      "grad_norm": 7.60760498046875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2556159496307373,
      "logits/rejected": 0.10285519063472748,
      "logps/chosen": -558.4526977539062,
      "logps/rejected": -629.82763671875,
      "loss": 0.4962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3359642028808594,
      "rewards/margins": 1.2519025802612305,
      "rewards/rejected": -1.5878665447235107,
      "step": 2287
    },
    {
      "epoch": 0.2994029606608326,
      "grad_norm": 7.411624908447266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09124704450368881,
      "logits/rejected": -0.03549659997224808,
      "logps/chosen": -420.7018127441406,
      "logps/rejected": -399.3690185546875,
      "loss": 0.5296,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2209337055683136,
      "rewards/margins": 0.7165447473526001,
      "rewards/rejected": -0.49561095237731934,
      "step": 2288
    },
    {
      "epoch": 0.29953381859818434,
      "grad_norm": 5.660470962524414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15214776992797852,
      "logits/rejected": -0.08454786241054535,
      "logps/chosen": -514.3316650390625,
      "logps/rejected": -578.8782348632812,
      "loss": 0.3765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44725149869918823,
      "rewards/margins": 1.3468408584594727,
      "rewards/rejected": -0.8995893597602844,
      "step": 2289
    },
    {
      "epoch": 0.29966467653553613,
      "grad_norm": 5.6192216873168945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004752730950713158,
      "logits/rejected": 0.03861675783991814,
      "logps/chosen": -328.3742980957031,
      "logps/rejected": -634.473876953125,
      "loss": 0.4467,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2017974704504013,
      "rewards/margins": 1.007805585861206,
      "rewards/rejected": -1.2096030712127686,
      "step": 2290
    },
    {
      "epoch": 0.29979553447288787,
      "grad_norm": 3.6812663078308105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.455685555934906,
      "logits/rejected": 0.20416784286499023,
      "logps/chosen": -394.52288818359375,
      "logps/rejected": -463.6028137207031,
      "loss": 0.3126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2019481658935547,
      "rewards/margins": 1.3235927820205688,
      "rewards/rejected": -1.1216446161270142,
      "step": 2291
    },
    {
      "epoch": 0.2999263924102396,
      "grad_norm": 8.921338081359863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.042655181139707565,
      "logits/rejected": -0.02392818033695221,
      "logps/chosen": -463.4167175292969,
      "logps/rejected": -577.2791137695312,
      "loss": 0.6735,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18188688158988953,
      "rewards/margins": 0.4144099950790405,
      "rewards/rejected": -0.5962968468666077,
      "step": 2292
    },
    {
      "epoch": 0.3000572503475914,
      "grad_norm": 5.988609790802002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09098759293556213,
      "logits/rejected": -0.37055909633636475,
      "logps/chosen": -477.5049133300781,
      "logps/rejected": -342.2791442871094,
      "loss": 0.3835,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08820344507694244,
      "rewards/margins": 1.145917534828186,
      "rewards/rejected": -1.0577141046524048,
      "step": 2293
    },
    {
      "epoch": 0.30018810828494313,
      "grad_norm": 5.032156944274902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28735488653182983,
      "logits/rejected": 0.1580859124660492,
      "logps/chosen": -439.7331237792969,
      "logps/rejected": -377.64398193359375,
      "loss": 0.3479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47864875197410583,
      "rewards/margins": 1.1209570169448853,
      "rewards/rejected": -0.6423081755638123,
      "step": 2294
    },
    {
      "epoch": 0.3003189662222949,
      "grad_norm": 8.162175178527832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.067862868309021,
      "logits/rejected": -0.2669040858745575,
      "logps/chosen": -507.31500244140625,
      "logps/rejected": -460.8146057128906,
      "loss": 0.5321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4675741493701935,
      "rewards/margins": 0.5728483200073242,
      "rewards/rejected": -1.0404224395751953,
      "step": 2295
    },
    {
      "epoch": 0.30044982415964666,
      "grad_norm": 10.623957633972168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006698600947856903,
      "logits/rejected": 0.0307120643556118,
      "logps/chosen": -590.0880126953125,
      "logps/rejected": -515.225830078125,
      "loss": 0.5028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1435869336128235,
      "rewards/margins": 1.0932303667068481,
      "rewards/rejected": -1.2368173599243164,
      "step": 2296
    },
    {
      "epoch": 0.30058068209699845,
      "grad_norm": 6.747323036193848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3557261824607849,
      "logits/rejected": 0.3012745678424835,
      "logps/chosen": -558.280517578125,
      "logps/rejected": -645.9188842773438,
      "loss": 0.5024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2619835138320923,
      "rewards/margins": 0.6307339668273926,
      "rewards/rejected": -0.3687504529953003,
      "step": 2297
    },
    {
      "epoch": 0.3007115400343502,
      "grad_norm": 6.001184463500977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1805802881717682,
      "logits/rejected": -0.17764811217784882,
      "logps/chosen": -566.2557373046875,
      "logps/rejected": -394.3774108886719,
      "loss": 0.589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8927984833717346,
      "rewards/margins": 0.36117541790008545,
      "rewards/rejected": -1.2539737224578857,
      "step": 2298
    },
    {
      "epoch": 0.300842397971702,
      "grad_norm": 5.601047515869141,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29490190744400024,
      "logits/rejected": 0.2759230136871338,
      "logps/chosen": -448.1141357421875,
      "logps/rejected": -503.4764709472656,
      "loss": 0.3784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13948597013950348,
      "rewards/margins": 0.9991247057914734,
      "rewards/rejected": -1.138610601425171,
      "step": 2299
    },
    {
      "epoch": 0.3009732559090537,
      "grad_norm": 4.902700424194336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15958675742149353,
      "logits/rejected": 0.1010778546333313,
      "logps/chosen": -313.753173828125,
      "logps/rejected": -324.32177734375,
      "loss": 0.3777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014180175960063934,
      "rewards/margins": 1.0179572105407715,
      "rewards/rejected": -1.003777027130127,
      "step": 2300
    },
    {
      "epoch": 0.3011041138464055,
      "grad_norm": 5.021937847137451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025346234440803528,
      "logits/rejected": 0.11294382810592651,
      "logps/chosen": -322.7236022949219,
      "logps/rejected": -405.023193359375,
      "loss": 0.3755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32324010133743286,
      "rewards/margins": 1.1666723489761353,
      "rewards/rejected": -1.489912509918213,
      "step": 2301
    },
    {
      "epoch": 0.30123497178375724,
      "grad_norm": 9.544625282287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14471308887004852,
      "logits/rejected": -0.20364269614219666,
      "logps/chosen": -419.5434265136719,
      "logps/rejected": -356.5314025878906,
      "loss": 0.8875,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.7934772372245789,
      "rewards/margins": -0.11169257014989853,
      "rewards/rejected": -0.6817846298217773,
      "step": 2302
    },
    {
      "epoch": 0.30136582972110904,
      "grad_norm": 9.12055492401123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24583488702774048,
      "logits/rejected": 0.030015451833605766,
      "logps/chosen": -435.38177490234375,
      "logps/rejected": -556.3662719726562,
      "loss": 0.7388,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7721238136291504,
      "rewards/margins": 0.02658405900001526,
      "rewards/rejected": -1.7987078428268433,
      "step": 2303
    },
    {
      "epoch": 0.30149668765846077,
      "grad_norm": 4.353696346282959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10284819453954697,
      "logits/rejected": -0.016877125948667526,
      "logps/chosen": -356.67144775390625,
      "logps/rejected": -356.0924987792969,
      "loss": 0.242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26298999786376953,
      "rewards/margins": 1.9785817861557007,
      "rewards/rejected": -1.7155919075012207,
      "step": 2304
    },
    {
      "epoch": 0.30162754559581256,
      "grad_norm": 11.77007007598877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04578307271003723,
      "logits/rejected": -0.14644935727119446,
      "logps/chosen": -403.82672119140625,
      "logps/rejected": -364.03704833984375,
      "loss": 0.8194,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9838166236877441,
      "rewards/margins": -0.07709160447120667,
      "rewards/rejected": -0.9067251682281494,
      "step": 2305
    },
    {
      "epoch": 0.3017584035331643,
      "grad_norm": 8.449178695678711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09848169982433319,
      "logits/rejected": -0.20833507180213928,
      "logps/chosen": -303.9792175292969,
      "logps/rejected": -349.87457275390625,
      "loss": 0.8057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.507122814655304,
      "rewards/margins": 0.13515779376029968,
      "rewards/rejected": -0.6422805786132812,
      "step": 2306
    },
    {
      "epoch": 0.3018892614705161,
      "grad_norm": 9.589431762695312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24587833881378174,
      "logits/rejected": 0.21461066603660583,
      "logps/chosen": -438.86041259765625,
      "logps/rejected": -363.1910705566406,
      "loss": 0.8017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2145158052444458,
      "rewards/margins": 0.15392009913921356,
      "rewards/rejected": -0.3684358596801758,
      "step": 2307
    },
    {
      "epoch": 0.3020201194078678,
      "grad_norm": 8.640597343444824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5230610370635986,
      "logits/rejected": 0.5040369033813477,
      "logps/chosen": -361.4969787597656,
      "logps/rejected": -375.9910583496094,
      "loss": 0.7425,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4143604636192322,
      "rewards/margins": 0.014581859111785889,
      "rewards/rejected": -0.4289422631263733,
      "step": 2308
    },
    {
      "epoch": 0.3021509773452196,
      "grad_norm": 6.3761162757873535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11978486180305481,
      "logits/rejected": -0.028193127363920212,
      "logps/chosen": -471.520263671875,
      "logps/rejected": -357.3424072265625,
      "loss": 0.3872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.75505530834198,
      "rewards/margins": 1.0989214181900024,
      "rewards/rejected": -1.8539764881134033,
      "step": 2309
    },
    {
      "epoch": 0.30228183528257135,
      "grad_norm": 6.343998908996582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36262524127960205,
      "logits/rejected": 0.2157244235277176,
      "logps/chosen": -589.070556640625,
      "logps/rejected": -442.171142578125,
      "loss": 0.4997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31780701875686646,
      "rewards/margins": 0.5176645517349243,
      "rewards/rejected": -0.8354715704917908,
      "step": 2310
    },
    {
      "epoch": 0.30241269321992315,
      "grad_norm": 11.512462615966797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1232626885175705,
      "logits/rejected": -0.18589094281196594,
      "logps/chosen": -469.1473693847656,
      "logps/rejected": -298.2027282714844,
      "loss": 1.0262,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2765166759490967,
      "rewards/margins": -0.20853789150714874,
      "rewards/rejected": -1.067978858947754,
      "step": 2311
    },
    {
      "epoch": 0.3025435511572749,
      "grad_norm": 10.755812644958496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09843424707651138,
      "logits/rejected": -0.15918466448783875,
      "logps/chosen": -443.93212890625,
      "logps/rejected": -522.0568237304688,
      "loss": 0.88,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18800048530101776,
      "rewards/margins": 0.14610403776168823,
      "rewards/rejected": 0.04189640283584595,
      "step": 2312
    },
    {
      "epoch": 0.3026744090946267,
      "grad_norm": 6.855533599853516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07356207072734833,
      "logits/rejected": -0.031710803508758545,
      "logps/chosen": -480.38275146484375,
      "logps/rejected": -350.32586669921875,
      "loss": 0.5534,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17195051908493042,
      "rewards/margins": 1.0364408493041992,
      "rewards/rejected": -1.2083914279937744,
      "step": 2313
    },
    {
      "epoch": 0.3028052670319784,
      "grad_norm": 5.176833152770996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1691337525844574,
      "logits/rejected": 0.1739606261253357,
      "logps/chosen": -422.6868591308594,
      "logps/rejected": -493.0423278808594,
      "loss": 0.2676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8134398460388184,
      "rewards/margins": 1.6914973258972168,
      "rewards/rejected": -0.8780574798583984,
      "step": 2314
    },
    {
      "epoch": 0.30293612496933014,
      "grad_norm": 4.034987926483154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012902984395623207,
      "logits/rejected": 0.19740290939807892,
      "logps/chosen": -331.73388671875,
      "logps/rejected": -503.76678466796875,
      "loss": 0.3094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30363959074020386,
      "rewards/margins": 1.6786909103393555,
      "rewards/rejected": -1.3750513792037964,
      "step": 2315
    },
    {
      "epoch": 0.30306698290668194,
      "grad_norm": 4.375851631164551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0748683363199234,
      "logits/rejected": 0.041448187083005905,
      "logps/chosen": -296.3431091308594,
      "logps/rejected": -324.9432678222656,
      "loss": 0.3384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24106691777706146,
      "rewards/margins": 1.382925271987915,
      "rewards/rejected": -1.6239922046661377,
      "step": 2316
    },
    {
      "epoch": 0.3031978408440337,
      "grad_norm": 3.7882919311523438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26071658730506897,
      "logits/rejected": 0.23030933737754822,
      "logps/chosen": -362.9458923339844,
      "logps/rejected": -378.1902160644531,
      "loss": 0.3165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2624431550502777,
      "rewards/margins": 1.1730738878250122,
      "rewards/rejected": -1.4355170726776123,
      "step": 2317
    },
    {
      "epoch": 0.30332869878138546,
      "grad_norm": 5.635074138641357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11354604363441467,
      "logits/rejected": 0.28476470708847046,
      "logps/chosen": -454.3315124511719,
      "logps/rejected": -399.6828918457031,
      "loss": 0.4292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5411088466644287,
      "rewards/margins": 0.900307834148407,
      "rewards/rejected": -1.4414167404174805,
      "step": 2318
    },
    {
      "epoch": 0.3034595567187372,
      "grad_norm": 11.149612426757812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21276861429214478,
      "logits/rejected": 0.2363073080778122,
      "logps/chosen": -447.34033203125,
      "logps/rejected": -464.63409423828125,
      "loss": 0.8042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6838816404342651,
      "rewards/margins": 0.6370282769203186,
      "rewards/rejected": -1.3209099769592285,
      "step": 2319
    },
    {
      "epoch": 0.303590414656089,
      "grad_norm": 5.709831237792969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1503848284482956,
      "logits/rejected": 0.024766920134425163,
      "logps/chosen": -630.3325805664062,
      "logps/rejected": -528.2377319335938,
      "loss": 0.4511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06472304463386536,
      "rewards/margins": 1.0863152742385864,
      "rewards/rejected": -1.1510382890701294,
      "step": 2320
    },
    {
      "epoch": 0.3037212725934407,
      "grad_norm": 5.7743024826049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2518601715564728,
      "logits/rejected": -0.16221404075622559,
      "logps/chosen": -261.235595703125,
      "logps/rejected": -300.4490051269531,
      "loss": 0.7068,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5030097961425781,
      "rewards/margins": 0.34353941679000854,
      "rewards/rejected": -0.8465492725372314,
      "step": 2321
    },
    {
      "epoch": 0.3038521305307925,
      "grad_norm": 8.200011253356934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0973052829504013,
      "logits/rejected": 0.09841255843639374,
      "logps/chosen": -672.1161499023438,
      "logps/rejected": -552.8093872070312,
      "loss": 0.5378,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23028181493282318,
      "rewards/margins": 1.001771092414856,
      "rewards/rejected": -1.2320530414581299,
      "step": 2322
    },
    {
      "epoch": 0.30398298846814426,
      "grad_norm": 6.822482109069824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42066502571105957,
      "logits/rejected": 0.3113509714603424,
      "logps/chosen": -412.5636901855469,
      "logps/rejected": -337.5550842285156,
      "loss": 0.5457,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7025007009506226,
      "rewards/margins": 0.6538732051849365,
      "rewards/rejected": -1.356373906135559,
      "step": 2323
    },
    {
      "epoch": 0.30411384640549605,
      "grad_norm": 4.082125663757324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0021680332720279694,
      "logits/rejected": -0.2629885673522949,
      "logps/chosen": -451.8011474609375,
      "logps/rejected": -381.7479553222656,
      "loss": 0.3118,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03136458992958069,
      "rewards/margins": 1.3735144138336182,
      "rewards/rejected": -1.3421498537063599,
      "step": 2324
    },
    {
      "epoch": 0.3042447043428478,
      "grad_norm": 5.852408409118652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14551198482513428,
      "logits/rejected": 0.03909064084291458,
      "logps/chosen": -371.7867431640625,
      "logps/rejected": -402.90386962890625,
      "loss": 0.3688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.011368364095687866,
      "rewards/margins": 1.169298529624939,
      "rewards/rejected": -1.1806668043136597,
      "step": 2325
    },
    {
      "epoch": 0.3043755622801996,
      "grad_norm": 6.764354705810547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10700416564941406,
      "logits/rejected": 0.020899474620819092,
      "logps/chosen": -435.4293518066406,
      "logps/rejected": -375.5914306640625,
      "loss": 0.4494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07699661701917648,
      "rewards/margins": 0.7669599056243896,
      "rewards/rejected": -0.8439565300941467,
      "step": 2326
    },
    {
      "epoch": 0.3045064202175513,
      "grad_norm": 8.436247825622559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11892256140708923,
      "logits/rejected": 0.05965040624141693,
      "logps/chosen": -576.8800659179688,
      "logps/rejected": -442.65576171875,
      "loss": 0.7746,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07640283554792404,
      "rewards/margins": 0.5123302936553955,
      "rewards/rejected": -0.5887331366539001,
      "step": 2327
    },
    {
      "epoch": 0.3046372781549031,
      "grad_norm": 4.380954265594482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015103144571185112,
      "logits/rejected": -0.07970111072063446,
      "logps/chosen": -655.5941162109375,
      "logps/rejected": -509.800048828125,
      "loss": 0.2653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5946919918060303,
      "rewards/margins": 1.4440921545028687,
      "rewards/rejected": -0.8494001626968384,
      "step": 2328
    },
    {
      "epoch": 0.30476813609225484,
      "grad_norm": 4.077573299407959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20795708894729614,
      "logits/rejected": -0.041778482496738434,
      "logps/chosen": -287.7651062011719,
      "logps/rejected": -386.41888427734375,
      "loss": 0.2901,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04761171340942383,
      "rewards/margins": 1.648909330368042,
      "rewards/rejected": -1.6965209245681763,
      "step": 2329
    },
    {
      "epoch": 0.30489899402960663,
      "grad_norm": 7.54017972946167,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19347192347049713,
      "logits/rejected": 0.22493305802345276,
      "logps/chosen": -406.6044921875,
      "logps/rejected": -351.26434326171875,
      "loss": 0.5245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49907630681991577,
      "rewards/margins": 0.7770993709564209,
      "rewards/rejected": -1.2761757373809814,
      "step": 2330
    },
    {
      "epoch": 0.30502985196695837,
      "grad_norm": 8.982540130615234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2723051905632019,
      "logits/rejected": -0.17047683894634247,
      "logps/chosen": -533.1387329101562,
      "logps/rejected": -467.7163391113281,
      "loss": 0.626,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6818526983261108,
      "rewards/margins": 0.7840057611465454,
      "rewards/rejected": -1.4658584594726562,
      "step": 2331
    },
    {
      "epoch": 0.30516070990431016,
      "grad_norm": 9.093711853027344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004149295389652252,
      "logits/rejected": 0.08853308856487274,
      "logps/chosen": -427.505615234375,
      "logps/rejected": -572.9334716796875,
      "loss": 0.7069,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06553250551223755,
      "rewards/margins": 0.6547654867172241,
      "rewards/rejected": -0.5892330408096313,
      "step": 2332
    },
    {
      "epoch": 0.3052915678416619,
      "grad_norm": 7.836609363555908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19346484541893005,
      "logits/rejected": 0.20274239778518677,
      "logps/chosen": -555.972412109375,
      "logps/rejected": -753.8896484375,
      "loss": 0.5341,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3994930386543274,
      "rewards/margins": 1.2649016380310059,
      "rewards/rejected": -0.865408718585968,
      "step": 2333
    },
    {
      "epoch": 0.3054224257790137,
      "grad_norm": 6.482623100280762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1561763882637024,
      "logits/rejected": 0.07195660471916199,
      "logps/chosen": -479.7537536621094,
      "logps/rejected": -527.2855224609375,
      "loss": 0.3985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7097952365875244,
      "rewards/margins": 0.9231510162353516,
      "rewards/rejected": -0.21335580945014954,
      "step": 2334
    },
    {
      "epoch": 0.3055532837163654,
      "grad_norm": 9.080769538879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17000050842761993,
      "logits/rejected": 0.04786314070224762,
      "logps/chosen": -530.6539916992188,
      "logps/rejected": -470.81103515625,
      "loss": 0.7175,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6830357313156128,
      "rewards/margins": 0.24708667397499084,
      "rewards/rejected": -0.9301223754882812,
      "step": 2335
    },
    {
      "epoch": 0.30568414165371716,
      "grad_norm": 5.5955681800842285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10275749862194061,
      "logits/rejected": -0.20750880241394043,
      "logps/chosen": -581.829345703125,
      "logps/rejected": -503.9285888671875,
      "loss": 0.3477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05903627723455429,
      "rewards/margins": 1.1023616790771484,
      "rewards/rejected": -1.043325424194336,
      "step": 2336
    },
    {
      "epoch": 0.30581499959106895,
      "grad_norm": 8.092843055725098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3796765208244324,
      "logits/rejected": 0.2210586965084076,
      "logps/chosen": -426.9471435546875,
      "logps/rejected": -378.6990051269531,
      "loss": 0.7777,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12715721130371094,
      "rewards/margins": 0.20282290875911713,
      "rewards/rejected": -0.3299800753593445,
      "step": 2337
    },
    {
      "epoch": 0.3059458575284207,
      "grad_norm": 6.656763076782227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11452293395996094,
      "logits/rejected": 0.05133539065718651,
      "logps/chosen": -336.8315124511719,
      "logps/rejected": -289.06195068359375,
      "loss": 0.7417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43613201379776,
      "rewards/margins": 0.31030258536338806,
      "rewards/rejected": -0.7464346289634705,
      "step": 2338
    },
    {
      "epoch": 0.3060767154657725,
      "grad_norm": 5.5362548828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2536112666130066,
      "logits/rejected": 0.37435078620910645,
      "logps/chosen": -547.7227172851562,
      "logps/rejected": -628.1732177734375,
      "loss": 0.3854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1282060742378235,
      "rewards/margins": 0.979629635810852,
      "rewards/rejected": -0.8514236807823181,
      "step": 2339
    },
    {
      "epoch": 0.3062075734031242,
      "grad_norm": 5.651500701904297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04833567142486572,
      "logits/rejected": 0.05462898686528206,
      "logps/chosen": -531.8909912109375,
      "logps/rejected": -518.8019409179688,
      "loss": 0.344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4891282916069031,
      "rewards/margins": 1.767948031425476,
      "rewards/rejected": -1.2788196802139282,
      "step": 2340
    },
    {
      "epoch": 0.306338431340476,
      "grad_norm": 6.997448921203613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1743655651807785,
      "logits/rejected": 0.08358395844697952,
      "logps/chosen": -398.3966369628906,
      "logps/rejected": -418.6600646972656,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7887091636657715,
      "rewards/margins": 0.5048476457595825,
      "rewards/rejected": -1.293556809425354,
      "step": 2341
    },
    {
      "epoch": 0.30646928927782774,
      "grad_norm": 5.755527019500732,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42786717414855957,
      "logits/rejected": 0.3712691366672516,
      "logps/chosen": -745.8748168945312,
      "logps/rejected": -603.5574951171875,
      "loss": 0.3739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0744953155517578,
      "rewards/margins": 1.2277967929840088,
      "rewards/rejected": -0.1533014178276062,
      "step": 2342
    },
    {
      "epoch": 0.30660014721517953,
      "grad_norm": 8.301487922668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.111415795981884,
      "logits/rejected": 0.30187368392944336,
      "logps/chosen": -399.69940185546875,
      "logps/rejected": -482.64801025390625,
      "loss": 0.7852,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7520922422409058,
      "rewards/margins": 0.1749027669429779,
      "rewards/rejected": -0.9269949197769165,
      "step": 2343
    },
    {
      "epoch": 0.30673100515253127,
      "grad_norm": 9.050832748413086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06747998297214508,
      "logits/rejected": 0.2814731299877167,
      "logps/chosen": -341.00445556640625,
      "logps/rejected": -401.0688781738281,
      "loss": 0.6144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39982685446739197,
      "rewards/margins": 0.668209969997406,
      "rewards/rejected": -1.0680369138717651,
      "step": 2344
    },
    {
      "epoch": 0.30686186308988306,
      "grad_norm": 8.479275703430176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15711838006973267,
      "logits/rejected": 0.39543992280960083,
      "logps/chosen": -407.27099609375,
      "logps/rejected": -527.916748046875,
      "loss": 0.8016,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5317325592041016,
      "rewards/margins": 0.11646806448698044,
      "rewards/rejected": -0.6482006311416626,
      "step": 2345
    },
    {
      "epoch": 0.3069927210272348,
      "grad_norm": 7.321524620056152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23625141382217407,
      "logits/rejected": 0.3464530110359192,
      "logps/chosen": -332.5693359375,
      "logps/rejected": -445.1502685546875,
      "loss": 0.4494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2373945415019989,
      "rewards/margins": 0.9834701418876648,
      "rewards/rejected": -0.7460755705833435,
      "step": 2346
    },
    {
      "epoch": 0.3071235789645866,
      "grad_norm": 9.24537467956543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4190114736557007,
      "logits/rejected": 0.3213965892791748,
      "logps/chosen": -475.7557678222656,
      "logps/rejected": -488.6097717285156,
      "loss": 0.7327,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8720251321792603,
      "rewards/margins": 0.30248624086380005,
      "rewards/rejected": -1.1745113134384155,
      "step": 2347
    },
    {
      "epoch": 0.3072544369019383,
      "grad_norm": 6.048788070678711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3498770594596863,
      "logits/rejected": 0.2511196732521057,
      "logps/chosen": -573.894287109375,
      "logps/rejected": -423.2785949707031,
      "loss": 0.5572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03351632505655289,
      "rewards/margins": 0.5689447522163391,
      "rewards/rejected": -0.5354284644126892,
      "step": 2348
    },
    {
      "epoch": 0.3073852948392901,
      "grad_norm": 7.808878421783447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17650040984153748,
      "logits/rejected": 0.21856491267681122,
      "logps/chosen": -276.10736083984375,
      "logps/rejected": -413.6470031738281,
      "loss": 0.6994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16746273636817932,
      "rewards/margins": 0.797837495803833,
      "rewards/rejected": -0.9653002023696899,
      "step": 2349
    },
    {
      "epoch": 0.30751615277664185,
      "grad_norm": 8.895196914672852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08526988327503204,
      "logits/rejected": 0.05001049488782883,
      "logps/chosen": -308.33575439453125,
      "logps/rejected": -333.969970703125,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29898545145988464,
      "rewards/margins": 0.3690105676651001,
      "rewards/rejected": -0.6679960489273071,
      "step": 2350
    },
    {
      "epoch": 0.30764701071399364,
      "grad_norm": 7.246440887451172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2373013198375702,
      "logits/rejected": -0.061946459114551544,
      "logps/chosen": -385.595703125,
      "logps/rejected": -531.09228515625,
      "loss": 0.6078,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26514551043510437,
      "rewards/margins": 0.3553876578807831,
      "rewards/rejected": -0.6205331683158875,
      "step": 2351
    },
    {
      "epoch": 0.3077778686513454,
      "grad_norm": 7.799315929412842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40670931339263916,
      "logits/rejected": 0.021709464490413666,
      "logps/chosen": -386.20086669921875,
      "logps/rejected": -374.71807861328125,
      "loss": 0.7134,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21376383304595947,
      "rewards/margins": 0.8555172681808472,
      "rewards/rejected": -1.069280982017517,
      "step": 2352
    },
    {
      "epoch": 0.30790872658869717,
      "grad_norm": 5.475559711456299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27265530824661255,
      "logits/rejected": 0.10962985455989838,
      "logps/chosen": -477.65045166015625,
      "logps/rejected": -474.2196044921875,
      "loss": 0.2861,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37380069494247437,
      "rewards/margins": 1.3569592237472534,
      "rewards/rejected": -0.9831585884094238,
      "step": 2353
    },
    {
      "epoch": 0.3080395845260489,
      "grad_norm": 8.890995979309082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007433146238327026,
      "logits/rejected": 0.0434703454375267,
      "logps/chosen": -620.8538208007812,
      "logps/rejected": -521.4395141601562,
      "loss": 0.564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4120548963546753,
      "rewards/margins": 0.7107293009757996,
      "rewards/rejected": -1.1227842569351196,
      "step": 2354
    },
    {
      "epoch": 0.3081704424634007,
      "grad_norm": 9.145559310913086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19932745397090912,
      "logits/rejected": 0.16637443006038666,
      "logps/chosen": -561.628662109375,
      "logps/rejected": -536.4940795898438,
      "loss": 0.6492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13687247037887573,
      "rewards/margins": 0.6313387155532837,
      "rewards/rejected": -0.7682111859321594,
      "step": 2355
    },
    {
      "epoch": 0.30830130040075243,
      "grad_norm": 5.879158020019531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15898722410202026,
      "logits/rejected": 0.01668190211057663,
      "logps/chosen": -243.68385314941406,
      "logps/rejected": -305.06329345703125,
      "loss": 0.5697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1480066180229187,
      "rewards/margins": 0.364153116941452,
      "rewards/rejected": -0.5121597051620483,
      "step": 2356
    },
    {
      "epoch": 0.3084321583381042,
      "grad_norm": 7.989974021911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016490239650011063,
      "logits/rejected": -0.08442188054323196,
      "logps/chosen": -420.0081481933594,
      "logps/rejected": -397.5618896484375,
      "loss": 0.6259,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3843122720718384,
      "rewards/margins": 0.6067973971366882,
      "rewards/rejected": -0.9911096692085266,
      "step": 2357
    },
    {
      "epoch": 0.30856301627545596,
      "grad_norm": 7.514769554138184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35069531202316284,
      "logits/rejected": 0.30888041853904724,
      "logps/chosen": -488.8482666015625,
      "logps/rejected": -409.98321533203125,
      "loss": 0.5785,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.34918421506881714,
      "rewards/margins": 0.6843873858451843,
      "rewards/rejected": -0.33520323038101196,
      "step": 2358
    },
    {
      "epoch": 0.3086938742128077,
      "grad_norm": 4.7410712242126465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2846253216266632,
      "logits/rejected": 0.2643360197544098,
      "logps/chosen": -475.8373107910156,
      "logps/rejected": -442.15997314453125,
      "loss": 0.3438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14713439345359802,
      "rewards/margins": 1.2427353858947754,
      "rewards/rejected": -1.095600962638855,
      "step": 2359
    },
    {
      "epoch": 0.3088247321501595,
      "grad_norm": 7.291512489318848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05785057693719864,
      "logits/rejected": -0.007727605756372213,
      "logps/chosen": -583.1449584960938,
      "logps/rejected": -405.20245361328125,
      "loss": 0.5721,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12935811281204224,
      "rewards/margins": 0.5560399889945984,
      "rewards/rejected": -0.4266819357872009,
      "step": 2360
    },
    {
      "epoch": 0.3089555900875112,
      "grad_norm": 8.53481388092041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2864663004875183,
      "logits/rejected": 0.11714527755975723,
      "logps/chosen": -566.4991455078125,
      "logps/rejected": -542.7069702148438,
      "loss": 0.5663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19761620461940765,
      "rewards/margins": 0.7076114416122437,
      "rewards/rejected": -0.9052276611328125,
      "step": 2361
    },
    {
      "epoch": 0.309086448024863,
      "grad_norm": 6.611915588378906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17496372759342194,
      "logits/rejected": 0.14007513225078583,
      "logps/chosen": -512.3240966796875,
      "logps/rejected": -493.505126953125,
      "loss": 0.5556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6445189714431763,
      "rewards/margins": 0.8335501551628113,
      "rewards/rejected": -1.4780690670013428,
      "step": 2362
    },
    {
      "epoch": 0.30921730596221475,
      "grad_norm": 6.068628311157227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1725444793701172,
      "logits/rejected": 0.22736582159996033,
      "logps/chosen": -493.3106689453125,
      "logps/rejected": -521.8163452148438,
      "loss": 0.4416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6305648684501648,
      "rewards/margins": 0.9550001621246338,
      "rewards/rejected": -1.5855650901794434,
      "step": 2363
    },
    {
      "epoch": 0.30934816389956654,
      "grad_norm": 8.836695671081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17311514914035797,
      "logits/rejected": 0.1430065631866455,
      "logps/chosen": -580.0186157226562,
      "logps/rejected": -629.5492553710938,
      "loss": 0.6087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07391813397407532,
      "rewards/margins": 0.7379139065742493,
      "rewards/rejected": -0.6639957427978516,
      "step": 2364
    },
    {
      "epoch": 0.3094790218369183,
      "grad_norm": 7.099048137664795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26612401008605957,
      "logits/rejected": 0.33782175183296204,
      "logps/chosen": -371.5546875,
      "logps/rejected": -365.6993713378906,
      "loss": 0.611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3925269842147827,
      "rewards/margins": 0.3172605335712433,
      "rewards/rejected": -0.7097875475883484,
      "step": 2365
    },
    {
      "epoch": 0.30960987977427007,
      "grad_norm": 7.346336841583252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1538943350315094,
      "logits/rejected": -0.06011483073234558,
      "logps/chosen": -705.185546875,
      "logps/rejected": -522.574462890625,
      "loss": 0.5653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4320072531700134,
      "rewards/margins": 0.56943678855896,
      "rewards/rejected": -1.0014441013336182,
      "step": 2366
    },
    {
      "epoch": 0.3097407377116218,
      "grad_norm": 4.134562969207764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11028124392032623,
      "logits/rejected": 0.12348996102809906,
      "logps/chosen": -378.2821350097656,
      "logps/rejected": -516.5559692382812,
      "loss": 0.3373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.054325878620147705,
      "rewards/margins": 1.4662927389144897,
      "rewards/rejected": -1.5206185579299927,
      "step": 2367
    },
    {
      "epoch": 0.3098715956489736,
      "grad_norm": 6.690579891204834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.050537362694740295,
      "logits/rejected": 0.14604128897190094,
      "logps/chosen": -690.465576171875,
      "logps/rejected": -567.4681396484375,
      "loss": 0.3785,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4542129635810852,
      "rewards/margins": 1.0966777801513672,
      "rewards/rejected": -0.6424648761749268,
      "step": 2368
    },
    {
      "epoch": 0.31000245358632533,
      "grad_norm": 8.003277778625488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21529871225357056,
      "logits/rejected": 0.1502353549003601,
      "logps/chosen": -262.01824951171875,
      "logps/rejected": -347.443359375,
      "loss": 0.8132,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7306930422782898,
      "rewards/margins": 0.041916653513908386,
      "rewards/rejected": -0.7726097702980042,
      "step": 2369
    },
    {
      "epoch": 0.3101333115236771,
      "grad_norm": 6.804641246795654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09340523928403854,
      "logits/rejected": -0.06851351261138916,
      "logps/chosen": -409.316650390625,
      "logps/rejected": -507.8583984375,
      "loss": 0.4363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003925526514649391,
      "rewards/margins": 0.8722356557846069,
      "rewards/rejected": -0.8683102130889893,
      "step": 2370
    },
    {
      "epoch": 0.31026416946102886,
      "grad_norm": 8.468745231628418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09221626073122025,
      "logits/rejected": 0.0789569839835167,
      "logps/chosen": -517.057373046875,
      "logps/rejected": -439.470703125,
      "loss": 0.6153,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2813882827758789,
      "rewards/margins": 0.5427265167236328,
      "rewards/rejected": -0.8241147994995117,
      "step": 2371
    },
    {
      "epoch": 0.31039502739838065,
      "grad_norm": 6.6993937492370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06858581304550171,
      "logits/rejected": -0.11633364111185074,
      "logps/chosen": -478.8909912109375,
      "logps/rejected": -499.2403259277344,
      "loss": 0.4854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0167336463928223,
      "rewards/margins": 0.7118093967437744,
      "rewards/rejected": -1.7285428047180176,
      "step": 2372
    },
    {
      "epoch": 0.3105258853357324,
      "grad_norm": 6.312275409698486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011727489531040192,
      "logits/rejected": 0.18302734196186066,
      "logps/chosen": -427.4722595214844,
      "logps/rejected": -414.7486267089844,
      "loss": 0.6062,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.016015246510505676,
      "rewards/margins": 0.40034353733062744,
      "rewards/rejected": -0.4163587689399719,
      "step": 2373
    },
    {
      "epoch": 0.3106567432730842,
      "grad_norm": 8.37199592590332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38131749629974365,
      "logits/rejected": 0.25767046213150024,
      "logps/chosen": -568.450927734375,
      "logps/rejected": -457.2748107910156,
      "loss": 0.5864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1800035536289215,
      "rewards/margins": 0.5642945766448975,
      "rewards/rejected": -0.7442981600761414,
      "step": 2374
    },
    {
      "epoch": 0.3107876012104359,
      "grad_norm": 6.012915134429932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1754496991634369,
      "logits/rejected": -0.10889110714197159,
      "logps/chosen": -461.4924011230469,
      "logps/rejected": -470.926025390625,
      "loss": 0.3936,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2810562252998352,
      "rewards/margins": 1.0770652294158936,
      "rewards/rejected": -0.7960091233253479,
      "step": 2375
    },
    {
      "epoch": 0.3109184591477877,
      "grad_norm": 9.928637504577637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004074311349540949,
      "logits/rejected": -0.16073814034461975,
      "logps/chosen": -671.493408203125,
      "logps/rejected": -499.72259521484375,
      "loss": 0.6311,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10266170650720596,
      "rewards/margins": 1.2934911251068115,
      "rewards/rejected": -1.1908293962478638,
      "step": 2376
    },
    {
      "epoch": 0.31104931708513944,
      "grad_norm": 10.264080047607422,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08078271150588989,
      "logits/rejected": 0.1714312732219696,
      "logps/chosen": -493.5841369628906,
      "logps/rejected": -533.2470092773438,
      "loss": 0.9988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23825703561306,
      "rewards/margins": -0.3461867570877075,
      "rewards/rejected": 0.10792979598045349,
      "step": 2377
    },
    {
      "epoch": 0.31118017502249123,
      "grad_norm": 5.989412307739258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36710330843925476,
      "logits/rejected": 0.4473913013935089,
      "logps/chosen": -492.5806579589844,
      "logps/rejected": -481.14788818359375,
      "loss": 0.3263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5313789248466492,
      "rewards/margins": 1.6356364488601685,
      "rewards/rejected": -2.167015552520752,
      "step": 2378
    },
    {
      "epoch": 0.31131103295984297,
      "grad_norm": 9.506731986999512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16830356419086456,
      "logits/rejected": -0.009089590981602669,
      "logps/chosen": -432.5898132324219,
      "logps/rejected": -357.2467041015625,
      "loss": 0.7574,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5680843591690063,
      "rewards/margins": 0.2292976677417755,
      "rewards/rejected": -0.7973819971084595,
      "step": 2379
    },
    {
      "epoch": 0.3114418908971947,
      "grad_norm": 5.240045070648193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08254526555538177,
      "logits/rejected": 0.1824323683977127,
      "logps/chosen": -429.008544921875,
      "logps/rejected": -405.5037841796875,
      "loss": 0.4765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09347954392433167,
      "rewards/margins": 0.7227871417999268,
      "rewards/rejected": -0.8162665963172913,
      "step": 2380
    },
    {
      "epoch": 0.3115727488345465,
      "grad_norm": 10.789899826049805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31533879041671753,
      "logits/rejected": 0.15682446956634521,
      "logps/chosen": -436.3138427734375,
      "logps/rejected": -457.42266845703125,
      "loss": 0.8387,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4380168616771698,
      "rewards/margins": 0.25471997261047363,
      "rewards/rejected": -0.6927368640899658,
      "step": 2381
    },
    {
      "epoch": 0.31170360677189823,
      "grad_norm": 6.242578983306885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0869290828704834,
      "logits/rejected": 0.24871334433555603,
      "logps/chosen": -395.0695495605469,
      "logps/rejected": -284.2666931152344,
      "loss": 0.4654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5590925216674805,
      "rewards/margins": 0.7309902310371399,
      "rewards/rejected": -1.2900826930999756,
      "step": 2382
    },
    {
      "epoch": 0.31183446470925,
      "grad_norm": 6.390888690948486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3123643398284912,
      "logits/rejected": -0.2396700531244278,
      "logps/chosen": -429.9776611328125,
      "logps/rejected": -391.0000915527344,
      "loss": 0.5874,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7815452218055725,
      "rewards/margins": 0.6011613011360168,
      "rewards/rejected": -1.3827067613601685,
      "step": 2383
    },
    {
      "epoch": 0.31196532264660176,
      "grad_norm": 10.824450492858887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05869322270154953,
      "logits/rejected": 0.34663572907447815,
      "logps/chosen": -423.7947082519531,
      "logps/rejected": -540.3034057617188,
      "loss": 0.8162,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23455047607421875,
      "rewards/margins": 0.47336235642433167,
      "rewards/rejected": -0.7079128623008728,
      "step": 2384
    },
    {
      "epoch": 0.31209618058395355,
      "grad_norm": 13.85149097442627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2195490300655365,
      "logits/rejected": 0.19690553843975067,
      "logps/chosen": -347.61126708984375,
      "logps/rejected": -425.66949462890625,
      "loss": 1.5139,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4871343672275543,
      "rewards/margins": -1.0601402521133423,
      "rewards/rejected": 0.5730058550834656,
      "step": 2385
    },
    {
      "epoch": 0.3122270385213053,
      "grad_norm": 3.1105332374572754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06355970352888107,
      "logits/rejected": -0.22290919721126556,
      "logps/chosen": -379.6195373535156,
      "logps/rejected": -465.6034240722656,
      "loss": 0.2327,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7551392912864685,
      "rewards/margins": 2.442396879196167,
      "rewards/rejected": -3.1975362300872803,
      "step": 2386
    },
    {
      "epoch": 0.3123578964586571,
      "grad_norm": 15.421743392944336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19597801566123962,
      "logits/rejected": 0.02898399904370308,
      "logps/chosen": -531.502685546875,
      "logps/rejected": -554.0693359375,
      "loss": 1.4494,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6148689389228821,
      "rewards/margins": -0.6428415775299072,
      "rewards/rejected": 0.02797262743115425,
      "step": 2387
    },
    {
      "epoch": 0.3124887543960088,
      "grad_norm": 6.815151214599609,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07305033504962921,
      "logits/rejected": 0.19271394610404968,
      "logps/chosen": -442.96746826171875,
      "logps/rejected": -379.7496643066406,
      "loss": 0.6346,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14697039127349854,
      "rewards/margins": 0.24688169360160828,
      "rewards/rejected": -0.09991130977869034,
      "step": 2388
    },
    {
      "epoch": 0.3126196123333606,
      "grad_norm": 9.865225791931152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18326610326766968,
      "logits/rejected": 0.07876928150653839,
      "logps/chosen": -641.785400390625,
      "logps/rejected": -433.2840576171875,
      "loss": 0.6151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.331542044878006,
      "rewards/margins": 0.771274209022522,
      "rewards/rejected": -1.1028162240982056,
      "step": 2389
    },
    {
      "epoch": 0.31275047027071234,
      "grad_norm": 12.671277046203613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16569764912128448,
      "logits/rejected": 0.13704097270965576,
      "logps/chosen": -595.7001953125,
      "logps/rejected": -444.39532470703125,
      "loss": 1.0354,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8019113540649414,
      "rewards/margins": 0.04225027561187744,
      "rewards/rejected": -0.8441615700721741,
      "step": 2390
    },
    {
      "epoch": 0.31288132820806414,
      "grad_norm": 4.830750942230225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17648863792419434,
      "logits/rejected": -0.02197571098804474,
      "logps/chosen": -335.5856628417969,
      "logps/rejected": -408.67694091796875,
      "loss": 0.5667,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38778191804885864,
      "rewards/margins": 0.4750596880912781,
      "rewards/rejected": -0.8628416061401367,
      "step": 2391
    },
    {
      "epoch": 0.31301218614541587,
      "grad_norm": 7.724853515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05368998646736145,
      "logits/rejected": -0.1111767590045929,
      "logps/chosen": -687.0963134765625,
      "logps/rejected": -502.87908935546875,
      "loss": 0.6079,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1015840619802475,
      "rewards/margins": 0.655268132686615,
      "rewards/rejected": -0.7568521499633789,
      "step": 2392
    },
    {
      "epoch": 0.31314304408276766,
      "grad_norm": 7.479285717010498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10435722023248672,
      "logits/rejected": 0.08132031559944153,
      "logps/chosen": -758.9275512695312,
      "logps/rejected": -474.46942138671875,
      "loss": 0.5315,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27845537662506104,
      "rewards/margins": 1.0872070789337158,
      "rewards/rejected": -1.3656623363494873,
      "step": 2393
    },
    {
      "epoch": 0.3132739020201194,
      "grad_norm": 8.09272289276123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1674153059720993,
      "logits/rejected": 0.285397469997406,
      "logps/chosen": -446.39910888671875,
      "logps/rejected": -462.18707275390625,
      "loss": 0.6412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42172300815582275,
      "rewards/margins": 0.33065128326416016,
      "rewards/rejected": -0.7523742914199829,
      "step": 2394
    },
    {
      "epoch": 0.3134047599574712,
      "grad_norm": 7.767658233642578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15841908752918243,
      "logits/rejected": 0.16786286234855652,
      "logps/chosen": -532.4682006835938,
      "logps/rejected": -339.94818115234375,
      "loss": 0.6059,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.556009829044342,
      "rewards/margins": 0.9302005767822266,
      "rewards/rejected": -0.37419071793556213,
      "step": 2395
    },
    {
      "epoch": 0.3135356178948229,
      "grad_norm": 9.676031112670898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1161649227142334,
      "logits/rejected": -0.03540610522031784,
      "logps/chosen": -539.6286010742188,
      "logps/rejected": -566.210693359375,
      "loss": 0.8402,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5412940979003906,
      "rewards/margins": -0.1491302251815796,
      "rewards/rejected": -0.39216387271881104,
      "step": 2396
    },
    {
      "epoch": 0.3136664758321747,
      "grad_norm": 6.651412010192871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5904742479324341,
      "logits/rejected": 0.4155162572860718,
      "logps/chosen": -483.97259521484375,
      "logps/rejected": -483.6243896484375,
      "loss": 0.4689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23312342166900635,
      "rewards/margins": 0.8820640444755554,
      "rewards/rejected": -0.6489406824111938,
      "step": 2397
    },
    {
      "epoch": 0.31379733376952645,
      "grad_norm": 5.197534084320068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24187904596328735,
      "logits/rejected": 0.06636153161525726,
      "logps/chosen": -400.1841735839844,
      "logps/rejected": -358.3254089355469,
      "loss": 0.3934,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5963064432144165,
      "rewards/margins": 0.9991482496261597,
      "rewards/rejected": -0.4028417766094208,
      "step": 2398
    },
    {
      "epoch": 0.31392819170687825,
      "grad_norm": 10.759324073791504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0865439772605896,
      "logits/rejected": -0.08762335032224655,
      "logps/chosen": -419.74530029296875,
      "logps/rejected": -453.4925842285156,
      "loss": 1.0806,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.9907037019729614,
      "rewards/margins": -0.29210320115089417,
      "rewards/rejected": -0.6986006498336792,
      "step": 2399
    },
    {
      "epoch": 0.31405904964423,
      "grad_norm": 7.854278087615967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27098479866981506,
      "logits/rejected": -0.6147130131721497,
      "logps/chosen": -564.4337158203125,
      "logps/rejected": -515.8719482421875,
      "loss": 0.5796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2013767957687378,
      "rewards/margins": 0.6558716297149658,
      "rewards/rejected": -1.8572484254837036,
      "step": 2400
    },
    {
      "epoch": 0.3141899075815817,
      "grad_norm": 4.3811235427856445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17295554280281067,
      "logits/rejected": 0.1455753594636917,
      "logps/chosen": -504.25628662109375,
      "logps/rejected": -364.294677734375,
      "loss": 0.2641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23686830699443817,
      "rewards/margins": 1.3501585721969604,
      "rewards/rejected": -1.1132901906967163,
      "step": 2401
    },
    {
      "epoch": 0.3143207655189335,
      "grad_norm": 6.184657096862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06173654645681381,
      "logits/rejected": -0.029063493013381958,
      "logps/chosen": -488.9921569824219,
      "logps/rejected": -545.0451049804688,
      "loss": 0.3605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4947085976600647,
      "rewards/margins": 1.0840357542037964,
      "rewards/rejected": -1.5787441730499268,
      "step": 2402
    },
    {
      "epoch": 0.31445162345628525,
      "grad_norm": 7.26708459854126,
      "learning_rate": 5e-06,
      "logits/chosen": -0.158487930893898,
      "logits/rejected": -0.1389773190021515,
      "logps/chosen": -420.81158447265625,
      "logps/rejected": -383.4513244628906,
      "loss": 0.5501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01867753267288208,
      "rewards/margins": 0.8835372924804688,
      "rewards/rejected": -0.8648598194122314,
      "step": 2403
    },
    {
      "epoch": 0.31458248139363704,
      "grad_norm": 5.586512088775635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6075112223625183,
      "logits/rejected": 0.6486082077026367,
      "logps/chosen": -357.35113525390625,
      "logps/rejected": -396.630126953125,
      "loss": 0.4223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10124415159225464,
      "rewards/margins": 0.814473569393158,
      "rewards/rejected": -0.9157177209854126,
      "step": 2404
    },
    {
      "epoch": 0.3147133393309888,
      "grad_norm": 6.527041435241699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03740255534648895,
      "logits/rejected": 0.016956061124801636,
      "logps/chosen": -656.5278930664062,
      "logps/rejected": -449.7645568847656,
      "loss": 0.4425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06327073276042938,
      "rewards/margins": 0.9087160229682922,
      "rewards/rejected": -0.8454452753067017,
      "step": 2405
    },
    {
      "epoch": 0.31484419726834056,
      "grad_norm": 7.5786967277526855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05913873016834259,
      "logits/rejected": 0.019099723547697067,
      "logps/chosen": -478.89898681640625,
      "logps/rejected": -446.8092956542969,
      "loss": 0.546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08535176515579224,
      "rewards/margins": 0.9802015423774719,
      "rewards/rejected": -0.8948497772216797,
      "step": 2406
    },
    {
      "epoch": 0.3149750552056923,
      "grad_norm": 6.176441192626953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2989301383495331,
      "logits/rejected": 0.17763160169124603,
      "logps/chosen": -412.3796081542969,
      "logps/rejected": -369.8915100097656,
      "loss": 0.4976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8503965735435486,
      "rewards/margins": 0.6639679670333862,
      "rewards/rejected": 0.18642862141132355,
      "step": 2407
    },
    {
      "epoch": 0.3151059131430441,
      "grad_norm": 7.387334823608398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20395351946353912,
      "logits/rejected": 0.004488751292228699,
      "logps/chosen": -666.4266357421875,
      "logps/rejected": -630.3021240234375,
      "loss": 0.373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6863766312599182,
      "rewards/margins": 1.1248985528945923,
      "rewards/rejected": -1.8112752437591553,
      "step": 2408
    },
    {
      "epoch": 0.31523677108039583,
      "grad_norm": 5.817554950714111,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05496295168995857,
      "logits/rejected": 0.08444879949092865,
      "logps/chosen": -425.5941162109375,
      "logps/rejected": -360.38677978515625,
      "loss": 0.5093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.028128810226917267,
      "rewards/margins": 0.8523767590522766,
      "rewards/rejected": -0.8805055618286133,
      "step": 2409
    },
    {
      "epoch": 0.3153676290177476,
      "grad_norm": 4.212170124053955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07145430147647858,
      "logits/rejected": 0.11785056442022324,
      "logps/chosen": -547.03662109375,
      "logps/rejected": -448.193603515625,
      "loss": 0.3318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19657516479492188,
      "rewards/margins": 1.3871105909347534,
      "rewards/rejected": -1.1905354261398315,
      "step": 2410
    },
    {
      "epoch": 0.31549848695509936,
      "grad_norm": 5.118679046630859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10235840082168579,
      "logits/rejected": 0.03341849893331528,
      "logps/chosen": -315.9217834472656,
      "logps/rejected": -264.1916809082031,
      "loss": 0.5302,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2374143898487091,
      "rewards/margins": 0.7424405813217163,
      "rewards/rejected": -0.979854941368103,
      "step": 2411
    },
    {
      "epoch": 0.31562934489245115,
      "grad_norm": 5.3678741455078125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12833371758460999,
      "logits/rejected": -0.08545690774917603,
      "logps/chosen": -431.3625793457031,
      "logps/rejected": -339.46075439453125,
      "loss": 0.4757,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21081124246120453,
      "rewards/margins": 0.8337143659591675,
      "rewards/rejected": -0.6229031085968018,
      "step": 2412
    },
    {
      "epoch": 0.3157602028298029,
      "grad_norm": 7.620385646820068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16609179973602295,
      "logits/rejected": 0.03754737228155136,
      "logps/chosen": -505.68853759765625,
      "logps/rejected": -408.2378845214844,
      "loss": 0.5587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.002582937479019165,
      "rewards/margins": 0.7011816501617432,
      "rewards/rejected": -0.6985986232757568,
      "step": 2413
    },
    {
      "epoch": 0.3158910607671547,
      "grad_norm": 7.857797622680664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014430727809667587,
      "logits/rejected": -0.042833082377910614,
      "logps/chosen": -521.9833374023438,
      "logps/rejected": -532.36474609375,
      "loss": 0.7338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2368268966674805,
      "rewards/margins": 0.07460539042949677,
      "rewards/rejected": -1.3114323616027832,
      "step": 2414
    },
    {
      "epoch": 0.3160219187045064,
      "grad_norm": 6.616927146911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01542789489030838,
      "logits/rejected": 0.0047269295901060104,
      "logps/chosen": -318.5491943359375,
      "logps/rejected": -395.9233093261719,
      "loss": 0.4902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.478833943605423,
      "rewards/margins": 0.5603877902030945,
      "rewards/rejected": -0.0815538614988327,
      "step": 2415
    },
    {
      "epoch": 0.3161527766418582,
      "grad_norm": 7.573304176330566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3434891402721405,
      "logits/rejected": 0.14928428828716278,
      "logps/chosen": -467.8763122558594,
      "logps/rejected": -409.1280517578125,
      "loss": 0.639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04584755003452301,
      "rewards/margins": 0.40615832805633545,
      "rewards/rejected": -0.36031076312065125,
      "step": 2416
    },
    {
      "epoch": 0.31628363457920994,
      "grad_norm": 8.540081024169922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10662692785263062,
      "logits/rejected": -0.03365586698055267,
      "logps/chosen": -387.32586669921875,
      "logps/rejected": -289.9729919433594,
      "loss": 0.6475,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2607021927833557,
      "rewards/margins": 0.3515627086162567,
      "rewards/rejected": -0.09086055308580399,
      "step": 2417
    },
    {
      "epoch": 0.31641449251656173,
      "grad_norm": 6.019521236419678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0960858166217804,
      "logits/rejected": 0.10277631878852844,
      "logps/chosen": -437.0699768066406,
      "logps/rejected": -418.23773193359375,
      "loss": 0.383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41421639919281006,
      "rewards/margins": 1.4701143503189087,
      "rewards/rejected": -1.8843307495117188,
      "step": 2418
    },
    {
      "epoch": 0.31654535045391347,
      "grad_norm": 8.845489501953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2841222286224365,
      "logits/rejected": 0.27633172273635864,
      "logps/chosen": -632.8048706054688,
      "logps/rejected": -474.822509765625,
      "loss": 0.5844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2557903528213501,
      "rewards/margins": 0.8650940656661987,
      "rewards/rejected": -1.1208842992782593,
      "step": 2419
    },
    {
      "epoch": 0.31667620839126526,
      "grad_norm": 6.9690728187561035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25333645939826965,
      "logits/rejected": 0.2503640353679657,
      "logps/chosen": -528.8712768554688,
      "logps/rejected": -531.548583984375,
      "loss": 0.5453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17710503935813904,
      "rewards/margins": 1.0429768562316895,
      "rewards/rejected": -0.8658719062805176,
      "step": 2420
    },
    {
      "epoch": 0.316807066328617,
      "grad_norm": 6.7126617431640625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14486238360404968,
      "logits/rejected": -0.004242405295372009,
      "logps/chosen": -579.3800659179688,
      "logps/rejected": -474.90948486328125,
      "loss": 0.5835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28348925709724426,
      "rewards/margins": 0.3000290095806122,
      "rewards/rejected": -0.01653974875807762,
      "step": 2421
    },
    {
      "epoch": 0.3169379242659688,
      "grad_norm": 5.098348617553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23173490166664124,
      "logits/rejected": 0.01923481747508049,
      "logps/chosen": -665.3902587890625,
      "logps/rejected": -473.80181884765625,
      "loss": 0.3395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8998488187789917,
      "rewards/margins": 1.266073226928711,
      "rewards/rejected": -0.3662244975566864,
      "step": 2422
    },
    {
      "epoch": 0.3170687822033205,
      "grad_norm": 8.20306396484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07306073606014252,
      "logits/rejected": -0.03675180673599243,
      "logps/chosen": -429.85235595703125,
      "logps/rejected": -598.18115234375,
      "loss": 0.5444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009334191679954529,
      "rewards/margins": 0.7435442209243774,
      "rewards/rejected": -0.7528783679008484,
      "step": 2423
    },
    {
      "epoch": 0.31719964014067226,
      "grad_norm": 7.088954448699951,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34011587500572205,
      "logits/rejected": 0.11865942180156708,
      "logps/chosen": -622.20263671875,
      "logps/rejected": -603.2636108398438,
      "loss": 0.4223,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24987277388572693,
      "rewards/margins": 1.0576579570770264,
      "rewards/rejected": -0.8077850937843323,
      "step": 2424
    },
    {
      "epoch": 0.31733049807802405,
      "grad_norm": 7.707048416137695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06413338333368301,
      "logits/rejected": 0.2729392945766449,
      "logps/chosen": -518.2608032226562,
      "logps/rejected": -475.10650634765625,
      "loss": 0.4603,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.043748073279857635,
      "rewards/margins": 1.003960132598877,
      "rewards/rejected": -1.0477081537246704,
      "step": 2425
    },
    {
      "epoch": 0.3174613560153758,
      "grad_norm": 5.320909023284912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2717878222465515,
      "logits/rejected": 0.15716737508773804,
      "logps/chosen": -633.4297485351562,
      "logps/rejected": -599.43896484375,
      "loss": 0.3547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2597496211528778,
      "rewards/margins": 1.0651214122772217,
      "rewards/rejected": -0.805371880531311,
      "step": 2426
    },
    {
      "epoch": 0.3175922139527276,
      "grad_norm": 8.106107711791992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09730610251426697,
      "logits/rejected": -0.22384819388389587,
      "logps/chosen": -512.9853515625,
      "logps/rejected": -411.0518798828125,
      "loss": 0.6168,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8405957818031311,
      "rewards/margins": 0.7372577786445618,
      "rewards/rejected": -1.5778535604476929,
      "step": 2427
    },
    {
      "epoch": 0.3177230718900793,
      "grad_norm": 6.618346691131592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02835792675614357,
      "logits/rejected": -0.131758913397789,
      "logps/chosen": -422.7210998535156,
      "logps/rejected": -348.2212219238281,
      "loss": 0.5405,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5290300846099854,
      "rewards/margins": 0.5921768546104431,
      "rewards/rejected": -1.1212068796157837,
      "step": 2428
    },
    {
      "epoch": 0.3178539298274311,
      "grad_norm": 3.509937286376953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2764245867729187,
      "logits/rejected": 0.29986968636512756,
      "logps/chosen": -331.2568054199219,
      "logps/rejected": -352.90240478515625,
      "loss": 0.4304,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.033001333475112915,
      "rewards/margins": 1.020687460899353,
      "rewards/rejected": -0.9876860976219177,
      "step": 2429
    },
    {
      "epoch": 0.31798478776478284,
      "grad_norm": 8.015899658203125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19666872918605804,
      "logits/rejected": 0.18024423718452454,
      "logps/chosen": -539.1367797851562,
      "logps/rejected": -529.1903076171875,
      "loss": 0.5898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6186599731445312,
      "rewards/margins": 0.9230024814605713,
      "rewards/rejected": -0.30434250831604004,
      "step": 2430
    },
    {
      "epoch": 0.31811564570213463,
      "grad_norm": 11.366094589233398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08020375669002533,
      "logits/rejected": -0.0746522918343544,
      "logps/chosen": -357.1223449707031,
      "logps/rejected": -557.407958984375,
      "loss": 1.1299,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5197166204452515,
      "rewards/margins": -0.3483319878578186,
      "rewards/rejected": -0.17138463258743286,
      "step": 2431
    },
    {
      "epoch": 0.31824650363948637,
      "grad_norm": 4.083846092224121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002009458839893341,
      "logits/rejected": 0.1608116179704666,
      "logps/chosen": -479.33306884765625,
      "logps/rejected": -636.503173828125,
      "loss": 0.2134,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22411364316940308,
      "rewards/margins": 2.3345792293548584,
      "rewards/rejected": -2.1104655265808105,
      "step": 2432
    },
    {
      "epoch": 0.31837736157683816,
      "grad_norm": 6.8745293617248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049243442714214325,
      "logits/rejected": 0.2811698913574219,
      "logps/chosen": -631.095703125,
      "logps/rejected": -572.6261596679688,
      "loss": 0.4072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23023416101932526,
      "rewards/margins": 1.2141966819763184,
      "rewards/rejected": -0.9839627146720886,
      "step": 2433
    },
    {
      "epoch": 0.3185082195141899,
      "grad_norm": 6.562743186950684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13548210263252258,
      "logits/rejected": 0.00895207840949297,
      "logps/chosen": -394.2599182128906,
      "logps/rejected": -352.48516845703125,
      "loss": 0.6033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5341777801513672,
      "rewards/margins": 0.3263092041015625,
      "rewards/rejected": -0.8604870438575745,
      "step": 2434
    },
    {
      "epoch": 0.3186390774515417,
      "grad_norm": 7.464289665222168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3026331663131714,
      "logits/rejected": 0.27715516090393066,
      "logps/chosen": -587.1319580078125,
      "logps/rejected": -575.9994506835938,
      "loss": 0.6342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1401304006576538,
      "rewards/margins": 0.5111067295074463,
      "rewards/rejected": -0.3709762990474701,
      "step": 2435
    },
    {
      "epoch": 0.3187699353888934,
      "grad_norm": 6.655405044555664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08107034116983414,
      "logits/rejected": 0.059117820113897324,
      "logps/chosen": -325.2014465332031,
      "logps/rejected": -355.4642028808594,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1826610565185547,
      "rewards/margins": 0.3015483617782593,
      "rewards/rejected": -0.11888732016086578,
      "step": 2436
    },
    {
      "epoch": 0.3189007933262452,
      "grad_norm": 3.835686683654785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1437280774116516,
      "logits/rejected": 0.17194700241088867,
      "logps/chosen": -651.9276123046875,
      "logps/rejected": -480.6429748535156,
      "loss": 0.3412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44138145446777344,
      "rewards/margins": 1.292061448097229,
      "rewards/rejected": -1.7334429025650024,
      "step": 2437
    },
    {
      "epoch": 0.31903165126359695,
      "grad_norm": 5.802048206329346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0044019147753715515,
      "logits/rejected": 0.015692956745624542,
      "logps/chosen": -504.9537353515625,
      "logps/rejected": -489.75616455078125,
      "loss": 0.5074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12130701541900635,
      "rewards/margins": 1.1480114459991455,
      "rewards/rejected": -1.0267044305801392,
      "step": 2438
    },
    {
      "epoch": 0.31916250920094874,
      "grad_norm": 5.723240375518799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04080817848443985,
      "logits/rejected": 0.05934103578329086,
      "logps/chosen": -552.0142822265625,
      "logps/rejected": -346.0111083984375,
      "loss": 0.5383,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12483996152877808,
      "rewards/margins": 0.552402913570404,
      "rewards/rejected": -0.6772428750991821,
      "step": 2439
    },
    {
      "epoch": 0.3192933671383005,
      "grad_norm": 7.054484844207764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07921209931373596,
      "logits/rejected": -0.2027094066143036,
      "logps/chosen": -630.486083984375,
      "logps/rejected": -611.7091674804688,
      "loss": 0.5642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04946688935160637,
      "rewards/margins": 1.0183852910995483,
      "rewards/rejected": -0.9689185619354248,
      "step": 2440
    },
    {
      "epoch": 0.31942422507565227,
      "grad_norm": 6.726057529449463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2600906491279602,
      "logits/rejected": 0.08699175715446472,
      "logps/chosen": -419.06903076171875,
      "logps/rejected": -424.9272766113281,
      "loss": 0.4916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45504212379455566,
      "rewards/margins": 0.7154882550239563,
      "rewards/rejected": -1.1705304384231567,
      "step": 2441
    },
    {
      "epoch": 0.319555083013004,
      "grad_norm": 4.986155033111572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3152889311313629,
      "logits/rejected": 0.06284032762050629,
      "logps/chosen": -300.03173828125,
      "logps/rejected": -295.818115234375,
      "loss": 0.554,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3921319842338562,
      "rewards/margins": 0.7808140516281128,
      "rewards/rejected": -1.1729459762573242,
      "step": 2442
    },
    {
      "epoch": 0.3196859409503558,
      "grad_norm": 6.552917003631592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19028249382972717,
      "logits/rejected": 0.22689637541770935,
      "logps/chosen": -611.065673828125,
      "logps/rejected": -658.6286010742188,
      "loss": 0.4831,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05437755584716797,
      "rewards/margins": 0.9518052339553833,
      "rewards/rejected": -1.0061829090118408,
      "step": 2443
    },
    {
      "epoch": 0.31981679888770753,
      "grad_norm": 14.09118366241455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01669749990105629,
      "logits/rejected": -0.02464229241013527,
      "logps/chosen": -495.1642150878906,
      "logps/rejected": -317.85107421875,
      "loss": 0.8697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.753902792930603,
      "rewards/margins": 0.2605958580970764,
      "rewards/rejected": -1.0144987106323242,
      "step": 2444
    },
    {
      "epoch": 0.31994765682505927,
      "grad_norm": 5.679886341094971,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16642993688583374,
      "logits/rejected": 0.10134975612163544,
      "logps/chosen": -467.13519287109375,
      "logps/rejected": -473.5374450683594,
      "loss": 0.4488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.710393488407135,
      "rewards/margins": 0.8243653774261475,
      "rewards/rejected": -1.5347589254379272,
      "step": 2445
    },
    {
      "epoch": 0.32007851476241106,
      "grad_norm": 9.805510520935059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23554061353206635,
      "logits/rejected": 0.1434071958065033,
      "logps/chosen": -448.83697509765625,
      "logps/rejected": -403.6861877441406,
      "loss": 0.8742,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3455971479415894,
      "rewards/margins": 0.3095816969871521,
      "rewards/rejected": -1.6551789045333862,
      "step": 2446
    },
    {
      "epoch": 0.3202093726997628,
      "grad_norm": 6.024610996246338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31149032711982727,
      "logits/rejected": 0.3729160726070404,
      "logps/chosen": -602.2615966796875,
      "logps/rejected": -540.0716552734375,
      "loss": 0.4303,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1146286129951477,
      "rewards/margins": 0.6934820413589478,
      "rewards/rejected": -0.8081105947494507,
      "step": 2447
    },
    {
      "epoch": 0.3203402306371146,
      "grad_norm": 6.765598297119141,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23977795243263245,
      "logits/rejected": -0.029689239338040352,
      "logps/chosen": -533.7744750976562,
      "logps/rejected": -465.6083984375,
      "loss": 0.5504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27494508028030396,
      "rewards/margins": 0.48305433988571167,
      "rewards/rejected": -0.2081092745065689,
      "step": 2448
    },
    {
      "epoch": 0.3204710885744663,
      "grad_norm": 5.473410129547119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15031832456588745,
      "logits/rejected": -0.09340550005435944,
      "logps/chosen": -415.2563781738281,
      "logps/rejected": -316.3853759765625,
      "loss": 0.3735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09514063596725464,
      "rewards/margins": 1.2388417720794678,
      "rewards/rejected": -1.3339823484420776,
      "step": 2449
    },
    {
      "epoch": 0.3206019465118181,
      "grad_norm": 4.890300273895264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028389912098646164,
      "logits/rejected": 0.01730015128850937,
      "logps/chosen": -306.7734375,
      "logps/rejected": -350.6125183105469,
      "loss": 0.498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5869848728179932,
      "rewards/margins": 0.6941773295402527,
      "rewards/rejected": -1.2811623811721802,
      "step": 2450
    },
    {
      "epoch": 0.32073280444916985,
      "grad_norm": 9.204246520996094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2644176185131073,
      "logits/rejected": 0.0819953978061676,
      "logps/chosen": -558.144287109375,
      "logps/rejected": -470.2430114746094,
      "loss": 0.7275,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06579448282718658,
      "rewards/margins": 0.2019859254360199,
      "rewards/rejected": -0.2677803933620453,
      "step": 2451
    },
    {
      "epoch": 0.32086366238652164,
      "grad_norm": 7.931615829467773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1517537534236908,
      "logits/rejected": 0.33620619773864746,
      "logps/chosen": -637.7979736328125,
      "logps/rejected": -485.92828369140625,
      "loss": 0.7345,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.27259165048599243,
      "rewards/margins": 0.6740453243255615,
      "rewards/rejected": -0.40145379304885864,
      "step": 2452
    },
    {
      "epoch": 0.3209945203238734,
      "grad_norm": 7.022432327270508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06853369623422623,
      "logits/rejected": 0.16177797317504883,
      "logps/chosen": -463.66265869140625,
      "logps/rejected": -500.53094482421875,
      "loss": 0.6372,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.340475857257843,
      "rewards/margins": 0.21913088858127594,
      "rewards/rejected": 0.12134499102830887,
      "step": 2453
    },
    {
      "epoch": 0.32112537826122517,
      "grad_norm": 11.559292793273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38969188928604126,
      "logits/rejected": 0.1463676393032074,
      "logps/chosen": -584.0013427734375,
      "logps/rejected": -570.0590209960938,
      "loss": 1.025,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8187423944473267,
      "rewards/margins": 0.07498812675476074,
      "rewards/rejected": -0.8937305808067322,
      "step": 2454
    },
    {
      "epoch": 0.3212562361985769,
      "grad_norm": 9.39745807647705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24027392268180847,
      "logits/rejected": 0.39473816752433777,
      "logps/chosen": -589.0304565429688,
      "logps/rejected": -529.66015625,
      "loss": 0.866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09779816120862961,
      "rewards/margins": 0.12652644515037537,
      "rewards/rejected": -0.028728291392326355,
      "step": 2455
    },
    {
      "epoch": 0.3213870941359287,
      "grad_norm": 5.4981584548950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08390244096517563,
      "logits/rejected": 0.18769173324108124,
      "logps/chosen": -347.8639221191406,
      "logps/rejected": -331.2856750488281,
      "loss": 0.5655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7318752408027649,
      "rewards/margins": 0.5810806155204773,
      "rewards/rejected": -1.3129559755325317,
      "step": 2456
    },
    {
      "epoch": 0.32151795207328043,
      "grad_norm": 6.7002153396606445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.044711291790008545,
      "logits/rejected": 0.0930987000465393,
      "logps/chosen": -670.4922485351562,
      "logps/rejected": -567.7132568359375,
      "loss": 0.4549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33805352449417114,
      "rewards/margins": 0.9297956824302673,
      "rewards/rejected": -0.591742217540741,
      "step": 2457
    },
    {
      "epoch": 0.3216488100106322,
      "grad_norm": 7.001465320587158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10683083534240723,
      "logits/rejected": -0.18157172203063965,
      "logps/chosen": -335.2027587890625,
      "logps/rejected": -349.1892395019531,
      "loss": 0.4469,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5323950052261353,
      "rewards/margins": 1.260011076927185,
      "rewards/rejected": -1.7924060821533203,
      "step": 2458
    },
    {
      "epoch": 0.32177966794798396,
      "grad_norm": 8.138720512390137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02124505303800106,
      "logits/rejected": 0.12359695136547089,
      "logps/chosen": -456.2301940917969,
      "logps/rejected": -327.6898193359375,
      "loss": 0.8503,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1333177238702774,
      "rewards/margins": 0.06692236661911011,
      "rewards/rejected": -0.2002401351928711,
      "step": 2459
    },
    {
      "epoch": 0.32191052588533575,
      "grad_norm": 9.730517387390137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0038505010306835175,
      "logits/rejected": 0.09051326662302017,
      "logps/chosen": -407.7497863769531,
      "logps/rejected": -606.5328979492188,
      "loss": 0.6821,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3956993520259857,
      "rewards/margins": 0.4458025097846985,
      "rewards/rejected": -0.8415018320083618,
      "step": 2460
    },
    {
      "epoch": 0.3220413838226875,
      "grad_norm": 10.856915473937988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08750230818986893,
      "logits/rejected": 0.13645505905151367,
      "logps/chosen": -789.6065673828125,
      "logps/rejected": -591.1309814453125,
      "loss": 0.7576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08068619668483734,
      "rewards/margins": 0.345350056886673,
      "rewards/rejected": -0.2646638751029968,
      "step": 2461
    },
    {
      "epoch": 0.3221722417600393,
      "grad_norm": 10.748746871948242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03826887533068657,
      "logits/rejected": -0.1626848578453064,
      "logps/chosen": -517.8118896484375,
      "logps/rejected": -467.1318359375,
      "loss": 0.8749,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5197100043296814,
      "rewards/margins": -0.16422788798809052,
      "rewards/rejected": -0.3554820716381073,
      "step": 2462
    },
    {
      "epoch": 0.322303099697391,
      "grad_norm": 6.470998287200928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13159964978694916,
      "logits/rejected": -0.005615979433059692,
      "logps/chosen": -380.81304931640625,
      "logps/rejected": -336.03021240234375,
      "loss": 0.5663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4308178424835205,
      "rewards/margins": 0.3681997060775757,
      "rewards/rejected": -0.7990174889564514,
      "step": 2463
    },
    {
      "epoch": 0.3224339576347428,
      "grad_norm": 4.423897743225098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03393943980336189,
      "logits/rejected": -0.17855669558048248,
      "logps/chosen": -229.04623413085938,
      "logps/rejected": -288.8765869140625,
      "loss": 0.4364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03773936629295349,
      "rewards/margins": 0.7618875503540039,
      "rewards/rejected": -0.7241482138633728,
      "step": 2464
    },
    {
      "epoch": 0.32256481557209454,
      "grad_norm": 9.559030532836914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2821090817451477,
      "logits/rejected": -0.0198991596698761,
      "logps/chosen": -526.332275390625,
      "logps/rejected": -408.99981689453125,
      "loss": 0.6983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009598135948181152,
      "rewards/margins": 0.9803676009178162,
      "rewards/rejected": -0.9707695245742798,
      "step": 2465
    },
    {
      "epoch": 0.32269567350944633,
      "grad_norm": 5.279623985290527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02342107519507408,
      "logits/rejected": 0.005463015288114548,
      "logps/chosen": -508.9311828613281,
      "logps/rejected": -408.8418884277344,
      "loss": 0.481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05201798677444458,
      "rewards/margins": 0.832121729850769,
      "rewards/rejected": -0.8841397166252136,
      "step": 2466
    },
    {
      "epoch": 0.32282653144679807,
      "grad_norm": 7.540898323059082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20093940198421478,
      "logits/rejected": 0.13467571139335632,
      "logps/chosen": -517.2740478515625,
      "logps/rejected": -476.94146728515625,
      "loss": 0.6127,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03774988278746605,
      "rewards/margins": 0.5830757021903992,
      "rewards/rejected": -0.6208256483078003,
      "step": 2467
    },
    {
      "epoch": 0.3229573893841498,
      "grad_norm": 5.433911323547363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19292223453521729,
      "logits/rejected": 0.10505993664264679,
      "logps/chosen": -327.263427734375,
      "logps/rejected": -408.32904052734375,
      "loss": 0.6591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5437082648277283,
      "rewards/margins": 0.41376417875289917,
      "rewards/rejected": -0.9574724435806274,
      "step": 2468
    },
    {
      "epoch": 0.3230882473215016,
      "grad_norm": 6.1458420753479,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20419880747795105,
      "logits/rejected": -0.17905490100383759,
      "logps/chosen": -467.15911865234375,
      "logps/rejected": -450.5397644042969,
      "loss": 0.452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5365469455718994,
      "rewards/margins": 0.6521382331848145,
      "rewards/rejected": -1.1886850595474243,
      "step": 2469
    },
    {
      "epoch": 0.32321910525885333,
      "grad_norm": 9.012616157531738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1660773754119873,
      "logits/rejected": 0.1259327381849289,
      "logps/chosen": -440.5300598144531,
      "logps/rejected": -472.6062316894531,
      "loss": 0.8704,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05126228556036949,
      "rewards/margins": -0.0426115021109581,
      "rewards/rejected": -0.00865077879279852,
      "step": 2470
    },
    {
      "epoch": 0.3233499631962051,
      "grad_norm": 7.8469929695129395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32366055250167847,
      "logits/rejected": 0.03788438439369202,
      "logps/chosen": -625.0882568359375,
      "logps/rejected": -399.765625,
      "loss": 0.624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5939518809318542,
      "rewards/margins": 0.18433761596679688,
      "rewards/rejected": 0.4096142053604126,
      "step": 2471
    },
    {
      "epoch": 0.32348082113355686,
      "grad_norm": 4.809234619140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05943750590085983,
      "logits/rejected": 0.14179079234600067,
      "logps/chosen": -377.3314514160156,
      "logps/rejected": -289.1966857910156,
      "loss": 0.4811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16974754631519318,
      "rewards/margins": 0.7698509097099304,
      "rewards/rejected": -0.6001034379005432,
      "step": 2472
    },
    {
      "epoch": 0.32361167907090865,
      "grad_norm": 8.087639808654785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03292442858219147,
      "logits/rejected": 0.08916760981082916,
      "logps/chosen": -552.3231201171875,
      "logps/rejected": -356.47247314453125,
      "loss": 0.6193,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3722309470176697,
      "rewards/margins": 0.328753262758255,
      "rewards/rejected": 0.04347766935825348,
      "step": 2473
    },
    {
      "epoch": 0.3237425370082604,
      "grad_norm": 11.011942863464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25309646129608154,
      "logits/rejected": 0.15829439461231232,
      "logps/chosen": -569.1168823242188,
      "logps/rejected": -495.87542724609375,
      "loss": 0.5405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20573383569717407,
      "rewards/margins": 1.0687717199325562,
      "rewards/rejected": -0.8630378842353821,
      "step": 2474
    },
    {
      "epoch": 0.3238733949456122,
      "grad_norm": 9.263467788696289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09168031066656113,
      "logits/rejected": 0.4088115394115448,
      "logps/chosen": -329.9264221191406,
      "logps/rejected": -460.26800537109375,
      "loss": 0.7183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03365956246852875,
      "rewards/margins": 0.2105102837085724,
      "rewards/rejected": -0.17685070633888245,
      "step": 2475
    },
    {
      "epoch": 0.3240042528829639,
      "grad_norm": 4.291787624359131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1613859236240387,
      "logits/rejected": 0.0649317055940628,
      "logps/chosen": -500.9825744628906,
      "logps/rejected": -394.4596252441406,
      "loss": 0.3487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.42814064025878906,
      "rewards/margins": 1.4653253555297852,
      "rewards/rejected": -1.037184715270996,
      "step": 2476
    },
    {
      "epoch": 0.3241351108203157,
      "grad_norm": 5.543372631072998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2638726532459259,
      "logits/rejected": 0.15863755345344543,
      "logps/chosen": -357.1027526855469,
      "logps/rejected": -312.1030578613281,
      "loss": 0.6915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17677079141139984,
      "rewards/margins": 0.20526815950870514,
      "rewards/rejected": -0.38203898072242737,
      "step": 2477
    },
    {
      "epoch": 0.32426596875766744,
      "grad_norm": 8.308259963989258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07310795783996582,
      "logits/rejected": 0.05453091114759445,
      "logps/chosen": -600.0820922851562,
      "logps/rejected": -554.3905639648438,
      "loss": 0.7197,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.46595054864883423,
      "rewards/margins": 0.2074909210205078,
      "rewards/rejected": 0.2584596872329712,
      "step": 2478
    },
    {
      "epoch": 0.32439682669501924,
      "grad_norm": 4.708370685577393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20232422649860382,
      "logits/rejected": 0.2884972095489502,
      "logps/chosen": -355.1630554199219,
      "logps/rejected": -321.4535217285156,
      "loss": 0.4866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30701160430908203,
      "rewards/margins": 0.6958572268486023,
      "rewards/rejected": -1.002868890762329,
      "step": 2479
    },
    {
      "epoch": 0.32452768463237097,
      "grad_norm": 9.3994779586792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04847422242164612,
      "logits/rejected": -0.16609902679920197,
      "logps/chosen": -485.30609130859375,
      "logps/rejected": -528.9275512695312,
      "loss": 0.8748,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7031160593032837,
      "rewards/margins": 0.17198827862739563,
      "rewards/rejected": -0.8751043081283569,
      "step": 2480
    },
    {
      "epoch": 0.32465854256972276,
      "grad_norm": 6.157026767730713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14799268543720245,
      "logits/rejected": 0.22505557537078857,
      "logps/chosen": -472.88897705078125,
      "logps/rejected": -373.6191711425781,
      "loss": 0.4528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3520938754081726,
      "rewards/margins": 0.8953367471694946,
      "rewards/rejected": -0.543242871761322,
      "step": 2481
    },
    {
      "epoch": 0.3247894005070745,
      "grad_norm": 5.733777046203613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11947065591812134,
      "logits/rejected": 0.2680183947086334,
      "logps/chosen": -620.3837280273438,
      "logps/rejected": -542.0668334960938,
      "loss": 0.4021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5454596281051636,
      "rewards/margins": 1.2246774435043335,
      "rewards/rejected": -0.6792178750038147,
      "step": 2482
    },
    {
      "epoch": 0.3249202584444263,
      "grad_norm": 6.624060153961182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0645226240158081,
      "logits/rejected": 0.05545420199632645,
      "logps/chosen": -788.6693725585938,
      "logps/rejected": -518.8319702148438,
      "loss": 0.4305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2512672543525696,
      "rewards/margins": 0.7675563097000122,
      "rewards/rejected": -0.5162891149520874,
      "step": 2483
    },
    {
      "epoch": 0.325051116381778,
      "grad_norm": 6.97664213180542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08736570179462433,
      "logits/rejected": 0.021844863891601562,
      "logps/chosen": -286.335205078125,
      "logps/rejected": -334.17486572265625,
      "loss": 0.6792,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19801369309425354,
      "rewards/margins": 0.32132279872894287,
      "rewards/rejected": -0.519336462020874,
      "step": 2484
    },
    {
      "epoch": 0.3251819743191298,
      "grad_norm": 4.093583106994629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2282065749168396,
      "logits/rejected": 0.18260058760643005,
      "logps/chosen": -360.9111633300781,
      "logps/rejected": -405.1796569824219,
      "loss": 0.3357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40637874603271484,
      "rewards/margins": 1.1032902002334595,
      "rewards/rejected": -1.5096691846847534,
      "step": 2485
    },
    {
      "epoch": 0.32531283225648155,
      "grad_norm": 9.434013366699219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07268164306879044,
      "logits/rejected": 0.030982330441474915,
      "logps/chosen": -493.0437927246094,
      "logps/rejected": -607.1300659179688,
      "loss": 0.5223,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20191629230976105,
      "rewards/margins": 1.042881965637207,
      "rewards/rejected": -0.8409656882286072,
      "step": 2486
    },
    {
      "epoch": 0.32544369019383335,
      "grad_norm": 6.867817401885986,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2038412094116211,
      "logits/rejected": 0.05969616770744324,
      "logps/chosen": -410.6897888183594,
      "logps/rejected": -376.1829833984375,
      "loss": 0.5588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10552767664194107,
      "rewards/margins": 0.5108307003974915,
      "rewards/rejected": -0.4053030014038086,
      "step": 2487
    },
    {
      "epoch": 0.3255745481311851,
      "grad_norm": 5.442486763000488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0908142700791359,
      "logits/rejected": -0.1546078324317932,
      "logps/chosen": -361.3199462890625,
      "logps/rejected": -401.6769714355469,
      "loss": 0.3563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22651539742946625,
      "rewards/margins": 1.094472885131836,
      "rewards/rejected": -0.8679574728012085,
      "step": 2488
    },
    {
      "epoch": 0.3257054060685368,
      "grad_norm": 8.253119468688965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028314262628555298,
      "logits/rejected": 0.03948911279439926,
      "logps/chosen": -555.4807739257812,
      "logps/rejected": -579.7061157226562,
      "loss": 0.6459,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.028507234528660774,
      "rewards/margins": 0.4788719415664673,
      "rewards/rejected": -0.45036470890045166,
      "step": 2489
    },
    {
      "epoch": 0.3258362640058886,
      "grad_norm": 9.638470649719238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10860349982976913,
      "logits/rejected": -0.08903142064809799,
      "logps/chosen": -392.4257507324219,
      "logps/rejected": -389.58319091796875,
      "loss": 0.9277,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.21866586804389954,
      "rewards/margins": -0.31160545349121094,
      "rewards/rejected": 0.5302713513374329,
      "step": 2490
    },
    {
      "epoch": 0.32596712194324035,
      "grad_norm": 3.4831628799438477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08448195457458496,
      "logits/rejected": 0.06039280444383621,
      "logps/chosen": -387.9223937988281,
      "logps/rejected": -324.11944580078125,
      "loss": 0.2799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5705835223197937,
      "rewards/margins": 1.3363019227981567,
      "rewards/rejected": -0.765718400478363,
      "step": 2491
    },
    {
      "epoch": 0.32609797988059214,
      "grad_norm": 6.136431694030762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17967361211776733,
      "logits/rejected": 0.11490464955568314,
      "logps/chosen": -340.0256042480469,
      "logps/rejected": -477.718994140625,
      "loss": 0.4049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4219665825366974,
      "rewards/margins": 0.9669420719146729,
      "rewards/rejected": -1.3889086246490479,
      "step": 2492
    },
    {
      "epoch": 0.3262288378179439,
      "grad_norm": 5.994833946228027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05099274963140488,
      "logits/rejected": 0.21468907594680786,
      "logps/chosen": -532.6287231445312,
      "logps/rejected": -516.1240234375,
      "loss": 0.3936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36996692419052124,
      "rewards/margins": 1.069409966468811,
      "rewards/rejected": -0.6994431018829346,
      "step": 2493
    },
    {
      "epoch": 0.32635969575529566,
      "grad_norm": 7.536137580871582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1357034593820572,
      "logits/rejected": 0.030194252729415894,
      "logps/chosen": -780.158203125,
      "logps/rejected": -797.8309936523438,
      "loss": 0.5644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4681776165962219,
      "rewards/margins": 0.576206386089325,
      "rewards/rejected": -1.0443840026855469,
      "step": 2494
    },
    {
      "epoch": 0.3264905536926474,
      "grad_norm": 8.20527458190918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3489101827144623,
      "logits/rejected": 0.13335220515727997,
      "logps/chosen": -383.6134948730469,
      "logps/rejected": -572.2295532226562,
      "loss": 0.8603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3249529004096985,
      "rewards/margins": 0.47631391882896423,
      "rewards/rejected": -0.15136107802391052,
      "step": 2495
    },
    {
      "epoch": 0.3266214116299992,
      "grad_norm": 9.111416816711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0695047527551651,
      "logits/rejected": -0.09727606922388077,
      "logps/chosen": -611.5765380859375,
      "logps/rejected": -637.9192504882812,
      "loss": 0.7145,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07753525674343109,
      "rewards/margins": 0.16073375940322876,
      "rewards/rejected": -0.23826901614665985,
      "step": 2496
    },
    {
      "epoch": 0.32675226956735093,
      "grad_norm": 5.159693241119385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07357221841812134,
      "logits/rejected": 0.37998729944229126,
      "logps/chosen": -396.1620178222656,
      "logps/rejected": -564.1177978515625,
      "loss": 0.3602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2991172969341278,
      "rewards/margins": 1.1167957782745361,
      "rewards/rejected": -1.4159129858016968,
      "step": 2497
    },
    {
      "epoch": 0.3268831275047027,
      "grad_norm": 6.147343158721924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18153567612171173,
      "logits/rejected": -0.23862631618976593,
      "logps/chosen": -276.7642822265625,
      "logps/rejected": -357.3646545410156,
      "loss": 0.368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6965830326080322,
      "rewards/margins": 1.2718309164047241,
      "rewards/rejected": -1.9684138298034668,
      "step": 2498
    },
    {
      "epoch": 0.32701398544205446,
      "grad_norm": 9.228607177734375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18525899946689606,
      "logits/rejected": 0.09579914808273315,
      "logps/chosen": -570.9462890625,
      "logps/rejected": -443.4614562988281,
      "loss": 0.8304,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0843023955821991,
      "rewards/margins": 0.12470868229866028,
      "rewards/rejected": -0.20901107788085938,
      "step": 2499
    },
    {
      "epoch": 0.32714484337940625,
      "grad_norm": 8.02325439453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5051637887954712,
      "logits/rejected": 0.50093674659729,
      "logps/chosen": -488.4404296875,
      "logps/rejected": -525.2974853515625,
      "loss": 0.6089,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5307140350341797,
      "rewards/margins": 0.7933258414268494,
      "rewards/rejected": -0.2626117765903473,
      "step": 2500
    },
    {
      "epoch": 0.327275701316758,
      "grad_norm": 8.615509033203125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1865854263305664,
      "logits/rejected": -0.09276336431503296,
      "logps/chosen": -222.25108337402344,
      "logps/rejected": -474.22314453125,
      "loss": 0.8918,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7195339202880859,
      "rewards/margins": 0.25164994597435,
      "rewards/rejected": -0.9711838960647583,
      "step": 2501
    },
    {
      "epoch": 0.3274065592541098,
      "grad_norm": 4.931187629699707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08797906339168549,
      "logits/rejected": 0.06420227885246277,
      "logps/chosen": -231.53515625,
      "logps/rejected": -300.6468505859375,
      "loss": 0.5325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4233739972114563,
      "rewards/margins": 0.4985083043575287,
      "rewards/rejected": -0.921882152557373,
      "step": 2502
    },
    {
      "epoch": 0.3275374171914615,
      "grad_norm": 4.757908344268799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005368612706661224,
      "logits/rejected": -0.17956411838531494,
      "logps/chosen": -455.5235595703125,
      "logps/rejected": -403.7515869140625,
      "loss": 0.3171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3829643428325653,
      "rewards/margins": 1.1796324253082275,
      "rewards/rejected": -1.5625966787338257,
      "step": 2503
    },
    {
      "epoch": 0.3276682751288133,
      "grad_norm": 4.2830891609191895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21950766444206238,
      "logits/rejected": 0.053162503987550735,
      "logps/chosen": -513.62255859375,
      "logps/rejected": -516.200927734375,
      "loss": 0.4112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36244910955429077,
      "rewards/margins": 1.3137216567993164,
      "rewards/rejected": -0.9512726068496704,
      "step": 2504
    },
    {
      "epoch": 0.32779913306616504,
      "grad_norm": 4.875279903411865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03699098527431488,
      "logits/rejected": -0.08241860568523407,
      "logps/chosen": -421.2676086425781,
      "logps/rejected": -438.9678955078125,
      "loss": 0.3262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02878018468618393,
      "rewards/margins": 1.3375213146209717,
      "rewards/rejected": -1.3663015365600586,
      "step": 2505
    },
    {
      "epoch": 0.32792999100351683,
      "grad_norm": 7.0042901039123535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03239334374666214,
      "logits/rejected": -0.14417025446891785,
      "logps/chosen": -450.4353332519531,
      "logps/rejected": -420.0086975097656,
      "loss": 0.5126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3920850455760956,
      "rewards/margins": 0.7782054543495178,
      "rewards/rejected": -1.1702905893325806,
      "step": 2506
    },
    {
      "epoch": 0.32806084894086857,
      "grad_norm": 10.096254348754883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2338704764842987,
      "logits/rejected": 0.2581188380718231,
      "logps/chosen": -456.9781494140625,
      "logps/rejected": -388.0417175292969,
      "loss": 0.6855,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9753814935684204,
      "rewards/margins": 0.3867180347442627,
      "rewards/rejected": -1.3620994091033936,
      "step": 2507
    },
    {
      "epoch": 0.32819170687822036,
      "grad_norm": 8.112740516662598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25259363651275635,
      "logits/rejected": 0.26265040040016174,
      "logps/chosen": -438.00079345703125,
      "logps/rejected": -297.9720153808594,
      "loss": 0.5844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39098799228668213,
      "rewards/margins": 0.42862704396247864,
      "rewards/rejected": -0.8196150064468384,
      "step": 2508
    },
    {
      "epoch": 0.3283225648155721,
      "grad_norm": 5.986601829528809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02899957448244095,
      "logits/rejected": -0.03388112038373947,
      "logps/chosen": -408.6573791503906,
      "logps/rejected": -298.93853759765625,
      "loss": 0.4646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07677935063838959,
      "rewards/margins": 0.8145126104354858,
      "rewards/rejected": -0.7377332448959351,
      "step": 2509
    },
    {
      "epoch": 0.3284534227529239,
      "grad_norm": 4.523125648498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0995665192604065,
      "logits/rejected": -0.049570776522159576,
      "logps/chosen": -695.178955078125,
      "logps/rejected": -436.0859069824219,
      "loss": 0.2691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3622661828994751,
      "rewards/margins": 1.44520902633667,
      "rewards/rejected": -1.0829427242279053,
      "step": 2510
    },
    {
      "epoch": 0.3285842806902756,
      "grad_norm": 6.993701457977295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0988791435956955,
      "logits/rejected": 0.19846364855766296,
      "logps/chosen": -279.376708984375,
      "logps/rejected": -368.6686706542969,
      "loss": 0.625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47384393215179443,
      "rewards/margins": 0.7283573150634766,
      "rewards/rejected": -1.202201247215271,
      "step": 2511
    },
    {
      "epoch": 0.32871513862762736,
      "grad_norm": 8.468918800354004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29013991355895996,
      "logits/rejected": 0.06574361026287079,
      "logps/chosen": -438.7494812011719,
      "logps/rejected": -559.5509033203125,
      "loss": 0.6797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5622866153717041,
      "rewards/margins": 0.22903726994991302,
      "rewards/rejected": -0.7913239002227783,
      "step": 2512
    },
    {
      "epoch": 0.32884599656497915,
      "grad_norm": 7.276623249053955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3157932758331299,
      "logits/rejected": 0.1994692087173462,
      "logps/chosen": -367.78363037109375,
      "logps/rejected": -439.024169921875,
      "loss": 0.5689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17048433423042297,
      "rewards/margins": 0.7127960920333862,
      "rewards/rejected": -0.8832803964614868,
      "step": 2513
    },
    {
      "epoch": 0.3289768545023309,
      "grad_norm": 7.318367004394531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09326810389757156,
      "logits/rejected": -0.07752146571874619,
      "logps/chosen": -385.1767272949219,
      "logps/rejected": -549.1988525390625,
      "loss": 0.5993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013368230313062668,
      "rewards/margins": 1.0182881355285645,
      "rewards/rejected": -1.0049197673797607,
      "step": 2514
    },
    {
      "epoch": 0.3291077124396827,
      "grad_norm": 6.407398700714111,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35692673921585083,
      "logits/rejected": 0.04384896159172058,
      "logps/chosen": -719.612060546875,
      "logps/rejected": -717.5303344726562,
      "loss": 0.4065,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.522473931312561,
      "rewards/margins": 1.072542667388916,
      "rewards/rejected": -0.5500686764717102,
      "step": 2515
    },
    {
      "epoch": 0.3292385703770344,
      "grad_norm": 3.2013919353485107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18870210647583008,
      "logits/rejected": 0.020054776221513748,
      "logps/chosen": -411.1144104003906,
      "logps/rejected": -337.8673095703125,
      "loss": 0.2285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.705976128578186,
      "rewards/margins": 1.825796127319336,
      "rewards/rejected": -1.11981999874115,
      "step": 2516
    },
    {
      "epoch": 0.3293694283143862,
      "grad_norm": 5.125767230987549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26432541012763977,
      "logits/rejected": 0.0640191063284874,
      "logps/chosen": -353.14178466796875,
      "logps/rejected": -331.65191650390625,
      "loss": 0.4697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17120647430419922,
      "rewards/margins": 0.7576516270637512,
      "rewards/rejected": -0.5864452123641968,
      "step": 2517
    },
    {
      "epoch": 0.32950028625173794,
      "grad_norm": 5.433367729187012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2567797601222992,
      "logits/rejected": -0.07830585539340973,
      "logps/chosen": -445.7174072265625,
      "logps/rejected": -522.7347412109375,
      "loss": 0.4947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8125913739204407,
      "rewards/margins": 0.8978301286697388,
      "rewards/rejected": -1.7104215621948242,
      "step": 2518
    },
    {
      "epoch": 0.32963114418908973,
      "grad_norm": 11.433152198791504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27220386266708374,
      "logits/rejected": -0.0792158916592598,
      "logps/chosen": -518.6865234375,
      "logps/rejected": -494.21636962890625,
      "loss": 0.9683,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2538859844207764,
      "rewards/margins": 0.05043278634548187,
      "rewards/rejected": -1.304318904876709,
      "step": 2519
    },
    {
      "epoch": 0.32976200212644147,
      "grad_norm": 7.293785095214844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24564087390899658,
      "logits/rejected": -0.016918815672397614,
      "logps/chosen": -509.4219055175781,
      "logps/rejected": -410.3478088378906,
      "loss": 0.6342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8750074505805969,
      "rewards/margins": 0.47765788435935974,
      "rewards/rejected": -1.3526653051376343,
      "step": 2520
    },
    {
      "epoch": 0.32989286006379326,
      "grad_norm": 4.541950225830078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10608018934726715,
      "logits/rejected": -0.05541691929101944,
      "logps/chosen": -462.6900634765625,
      "logps/rejected": -383.9090881347656,
      "loss": 0.405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021189691498875618,
      "rewards/margins": 1.1000028848648071,
      "rewards/rejected": -1.0788131952285767,
      "step": 2521
    },
    {
      "epoch": 0.330023718001145,
      "grad_norm": 6.110613822937012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12002279609441757,
      "logits/rejected": 0.15753643214702606,
      "logps/chosen": -353.2917175292969,
      "logps/rejected": -435.0023193359375,
      "loss": 0.3891,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.403054416179657,
      "rewards/margins": 1.04807710647583,
      "rewards/rejected": -1.4511313438415527,
      "step": 2522
    },
    {
      "epoch": 0.3301545759384968,
      "grad_norm": 6.895642280578613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2905937135219574,
      "logits/rejected": 0.1114480197429657,
      "logps/chosen": -460.0822448730469,
      "logps/rejected": -540.869873046875,
      "loss": 0.4752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.613324761390686,
      "rewards/margins": 0.8782111406326294,
      "rewards/rejected": -0.2648862898349762,
      "step": 2523
    },
    {
      "epoch": 0.3302854338758485,
      "grad_norm": 6.252125263214111,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2566644847393036,
      "logits/rejected": 0.16690778732299805,
      "logps/chosen": -408.74835205078125,
      "logps/rejected": -383.71026611328125,
      "loss": 0.3034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015364646911621094,
      "rewards/margins": 1.2468860149383545,
      "rewards/rejected": -1.2622507810592651,
      "step": 2524
    },
    {
      "epoch": 0.3304162918132003,
      "grad_norm": 10.771413803100586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023908331990242004,
      "logits/rejected": -0.22026723623275757,
      "logps/chosen": -641.0030517578125,
      "logps/rejected": -399.5374755859375,
      "loss": 0.8728,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6100872159004211,
      "rewards/margins": -0.18081587553024292,
      "rewards/rejected": -0.4292713403701782,
      "step": 2525
    },
    {
      "epoch": 0.33054714975055205,
      "grad_norm": 9.414250373840332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04826336354017258,
      "logits/rejected": 0.02098914235830307,
      "logps/chosen": -431.8968200683594,
      "logps/rejected": -398.4419250488281,
      "loss": 0.6655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05859869718551636,
      "rewards/margins": 0.7024402618408203,
      "rewards/rejected": -0.643841564655304,
      "step": 2526
    },
    {
      "epoch": 0.33067800768790384,
      "grad_norm": 10.271930694580078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06518343091011047,
      "logits/rejected": -0.20228607952594757,
      "logps/chosen": -604.8746337890625,
      "logps/rejected": -622.26220703125,
      "loss": 0.7473,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10296723246574402,
      "rewards/margins": 0.3994634747505188,
      "rewards/rejected": -0.2964962124824524,
      "step": 2527
    },
    {
      "epoch": 0.3308088656252556,
      "grad_norm": 10.162202835083008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3950709104537964,
      "logits/rejected": 0.19296711683273315,
      "logps/chosen": -592.5030517578125,
      "logps/rejected": -537.4680786132812,
      "loss": 0.6862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17799130082130432,
      "rewards/margins": 0.9889156818389893,
      "rewards/rejected": -1.1669069528579712,
      "step": 2528
    },
    {
      "epoch": 0.33093972356260737,
      "grad_norm": 5.844662666320801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07456352561712265,
      "logits/rejected": 0.05940619111061096,
      "logps/chosen": -394.2937927246094,
      "logps/rejected": -484.1541442871094,
      "loss": 0.4885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4270673990249634,
      "rewards/margins": 0.849886953830719,
      "rewards/rejected": -1.2769544124603271,
      "step": 2529
    },
    {
      "epoch": 0.3310705814999591,
      "grad_norm": 6.407329559326172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12844318151474,
      "logits/rejected": 0.08595572412014008,
      "logps/chosen": -460.2008972167969,
      "logps/rejected": -481.4493713378906,
      "loss": 0.391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11414909362792969,
      "rewards/margins": 1.0744472742080688,
      "rewards/rejected": -0.9602980613708496,
      "step": 2530
    },
    {
      "epoch": 0.3312014394373109,
      "grad_norm": 7.742395877838135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01695677638053894,
      "logits/rejected": -0.0021930113434791565,
      "logps/chosen": -525.0025024414062,
      "logps/rejected": -569.8895874023438,
      "loss": 0.6146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1608991175889969,
      "rewards/margins": 1.3613884449005127,
      "rewards/rejected": -1.5222876071929932,
      "step": 2531
    },
    {
      "epoch": 0.33133229737466263,
      "grad_norm": 10.889674186706543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08468712121248245,
      "logits/rejected": 0.17839136719703674,
      "logps/chosen": -432.44183349609375,
      "logps/rejected": -372.37921142578125,
      "loss": 0.8812,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.106163740158081,
      "rewards/margins": -0.0016040541231632233,
      "rewards/rejected": -1.1045596599578857,
      "step": 2532
    },
    {
      "epoch": 0.33146315531201437,
      "grad_norm": 5.295927047729492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040352463722229004,
      "logits/rejected": -0.07303325831890106,
      "logps/chosen": -517.2009887695312,
      "logps/rejected": -396.2073974609375,
      "loss": 0.4866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16379663348197937,
      "rewards/margins": 0.7769342660903931,
      "rewards/rejected": -0.6131376624107361,
      "step": 2533
    },
    {
      "epoch": 0.33159401324936616,
      "grad_norm": 5.320186138153076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22590897977352142,
      "logits/rejected": 0.2893362045288086,
      "logps/chosen": -643.19677734375,
      "logps/rejected": -594.4817504882812,
      "loss": 0.2661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6305705904960632,
      "rewards/margins": 1.5645359754562378,
      "rewards/rejected": -0.9339653253555298,
      "step": 2534
    },
    {
      "epoch": 0.3317248711867179,
      "grad_norm": 4.320387840270996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11940056830644608,
      "logits/rejected": -0.17408785223960876,
      "logps/chosen": -546.9006958007812,
      "logps/rejected": -432.8227233886719,
      "loss": 0.3769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6430530548095703,
      "rewards/margins": 0.9672592282295227,
      "rewards/rejected": -1.6103122234344482,
      "step": 2535
    },
    {
      "epoch": 0.3318557291240697,
      "grad_norm": 5.310115814208984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1806434839963913,
      "logits/rejected": 0.07493750751018524,
      "logps/chosen": -308.70892333984375,
      "logps/rejected": -430.53289794921875,
      "loss": 0.4448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4807628393173218,
      "rewards/margins": 0.8010145425796509,
      "rewards/rejected": -1.2817773818969727,
      "step": 2536
    },
    {
      "epoch": 0.3319865870614214,
      "grad_norm": 7.923567771911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005679715424776077,
      "logits/rejected": -0.11642526835203171,
      "logps/chosen": -496.5425109863281,
      "logps/rejected": -402.6960754394531,
      "loss": 0.702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2893594801425934,
      "rewards/margins": 0.44609734416007996,
      "rewards/rejected": -0.7354568243026733,
      "step": 2537
    },
    {
      "epoch": 0.3321174449987732,
      "grad_norm": 5.8672943115234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08791117370128632,
      "logits/rejected": -0.09126180410385132,
      "logps/chosen": -612.7835083007812,
      "logps/rejected": -558.3740234375,
      "loss": 0.3524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44248563051223755,
      "rewards/margins": 1.1031672954559326,
      "rewards/rejected": -1.545652985572815,
      "step": 2538
    },
    {
      "epoch": 0.33224830293612495,
      "grad_norm": 4.889994144439697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010304383933544159,
      "logits/rejected": -0.1026885062456131,
      "logps/chosen": -443.90386962890625,
      "logps/rejected": -443.6610107421875,
      "loss": 0.3422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10819227993488312,
      "rewards/margins": 1.0201060771942139,
      "rewards/rejected": -1.1282984018325806,
      "step": 2539
    },
    {
      "epoch": 0.33237916087347674,
      "grad_norm": 5.801917552947998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024954620748758316,
      "logits/rejected": -0.12719403207302094,
      "logps/chosen": -544.6694946289062,
      "logps/rejected": -389.3026123046875,
      "loss": 0.4957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2936685085296631,
      "rewards/margins": 0.7251605987548828,
      "rewards/rejected": -0.43149203062057495,
      "step": 2540
    },
    {
      "epoch": 0.3325100188108285,
      "grad_norm": 8.437052726745605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019887864589691162,
      "logits/rejected": 0.02153567224740982,
      "logps/chosen": -511.41253662109375,
      "logps/rejected": -518.8211669921875,
      "loss": 0.6075,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0669628456234932,
      "rewards/margins": 0.7224314212799072,
      "rewards/rejected": -0.7893942594528198,
      "step": 2541
    },
    {
      "epoch": 0.33264087674818027,
      "grad_norm": 3.7096827030181885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13719883561134338,
      "logits/rejected": -0.06634397804737091,
      "logps/chosen": -578.0264892578125,
      "logps/rejected": -360.4145202636719,
      "loss": 0.2344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24766255915164948,
      "rewards/margins": 1.6775000095367432,
      "rewards/rejected": -1.4298374652862549,
      "step": 2542
    },
    {
      "epoch": 0.332771734685532,
      "grad_norm": 5.130920886993408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03416069597005844,
      "logits/rejected": -0.07790425419807434,
      "logps/chosen": -385.7614440917969,
      "logps/rejected": -423.4328918457031,
      "loss": 0.4572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.045178383588790894,
      "rewards/margins": 0.7825549244880676,
      "rewards/rejected": -0.8277333378791809,
      "step": 2543
    },
    {
      "epoch": 0.3329025926228838,
      "grad_norm": 7.447519779205322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2479611337184906,
      "logits/rejected": 0.026491668075323105,
      "logps/chosen": -286.9114990234375,
      "logps/rejected": -411.67431640625,
      "loss": 0.5773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6895923614501953,
      "rewards/margins": 0.6450040936470032,
      "rewards/rejected": -1.3345963954925537,
      "step": 2544
    },
    {
      "epoch": 0.33303345056023553,
      "grad_norm": 7.786194801330566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17727653682231903,
      "logits/rejected": 0.192416712641716,
      "logps/chosen": -432.3983459472656,
      "logps/rejected": -424.7130126953125,
      "loss": 0.5378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1844947785139084,
      "rewards/margins": 0.7599108219146729,
      "rewards/rejected": -0.5754160284996033,
      "step": 2545
    },
    {
      "epoch": 0.3331643084975873,
      "grad_norm": 4.396462440490723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12336691468954086,
      "logits/rejected": -0.11928138136863708,
      "logps/chosen": -397.48382568359375,
      "logps/rejected": -417.3779296875,
      "loss": 0.2787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11103784292936325,
      "rewards/margins": 1.4607921838760376,
      "rewards/rejected": -1.5718300342559814,
      "step": 2546
    },
    {
      "epoch": 0.33329516643493906,
      "grad_norm": 8.278423309326172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1920856386423111,
      "logits/rejected": 0.21959391236305237,
      "logps/chosen": -512.4321899414062,
      "logps/rejected": -436.943603515625,
      "loss": 0.5165,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07610683143138885,
      "rewards/margins": 0.5735028982162476,
      "rewards/rejected": -0.6496097445487976,
      "step": 2547
    },
    {
      "epoch": 0.33342602437229085,
      "grad_norm": 5.496624946594238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11879419535398483,
      "logits/rejected": -0.06391031295061111,
      "logps/chosen": -323.02740478515625,
      "logps/rejected": -329.6795654296875,
      "loss": 0.5048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21289825439453125,
      "rewards/margins": 0.7143158912658691,
      "rewards/rejected": -0.9272143244743347,
      "step": 2548
    },
    {
      "epoch": 0.3335568823096426,
      "grad_norm": 5.74986457824707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22090426087379456,
      "logits/rejected": 0.25213533639907837,
      "logps/chosen": -347.8294677734375,
      "logps/rejected": -311.6541748046875,
      "loss": 0.4639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1078527420759201,
      "rewards/margins": 0.7195472717285156,
      "rewards/rejected": -0.8274000883102417,
      "step": 2549
    },
    {
      "epoch": 0.3336877402469944,
      "grad_norm": 5.731574535369873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38259148597717285,
      "logits/rejected": 0.47997742891311646,
      "logps/chosen": -414.2923278808594,
      "logps/rejected": -496.4913024902344,
      "loss": 0.4033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6157875061035156,
      "rewards/margins": 1.3177638053894043,
      "rewards/rejected": -0.7019761800765991,
      "step": 2550
    },
    {
      "epoch": 0.3338185981843461,
      "grad_norm": 6.265960216522217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21438641846179962,
      "logits/rejected": 0.19266186654567719,
      "logps/chosen": -424.46002197265625,
      "logps/rejected": -442.8009033203125,
      "loss": 0.5645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27239131927490234,
      "rewards/margins": 0.732150673866272,
      "rewards/rejected": -1.0045419931411743,
      "step": 2551
    },
    {
      "epoch": 0.3339494561216979,
      "grad_norm": 6.380173683166504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21387967467308044,
      "logits/rejected": -0.19633044302463531,
      "logps/chosen": -794.4031372070312,
      "logps/rejected": -556.62548828125,
      "loss": 0.4872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16589924693107605,
      "rewards/margins": 0.9413326382637024,
      "rewards/rejected": -1.1072319746017456,
      "step": 2552
    },
    {
      "epoch": 0.33408031405904964,
      "grad_norm": 7.682239055633545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002049286849796772,
      "logits/rejected": 0.0779402032494545,
      "logps/chosen": -524.166015625,
      "logps/rejected": -565.11767578125,
      "loss": 0.4436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0794057846069336,
      "rewards/margins": 1.2972971200942993,
      "rewards/rejected": -1.3767027854919434,
      "step": 2553
    },
    {
      "epoch": 0.3342111719964014,
      "grad_norm": 12.410163879394531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020513661205768585,
      "logits/rejected": -0.14080750942230225,
      "logps/chosen": -544.93701171875,
      "logps/rejected": -425.32830810546875,
      "loss": 0.5632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.053802743554115295,
      "rewards/margins": 1.1306304931640625,
      "rewards/rejected": -1.184433102607727,
      "step": 2554
    },
    {
      "epoch": 0.33434202993375317,
      "grad_norm": 8.56864070892334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12468636780977249,
      "logits/rejected": 0.07686243206262589,
      "logps/chosen": -318.7847595214844,
      "logps/rejected": -347.9992980957031,
      "loss": 0.5809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8627926111221313,
      "rewards/margins": 0.8597730398178101,
      "rewards/rejected": -1.7225656509399414,
      "step": 2555
    },
    {
      "epoch": 0.3344728878711049,
      "grad_norm": 6.855416297912598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26335591077804565,
      "logits/rejected": -0.4778810739517212,
      "logps/chosen": -360.3583068847656,
      "logps/rejected": -368.2756652832031,
      "loss": 0.4665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9792749881744385,
      "rewards/margins": 0.8071548938751221,
      "rewards/rejected": -1.786429762840271,
      "step": 2556
    },
    {
      "epoch": 0.3346037458084567,
      "grad_norm": 7.181939125061035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0578203946352005,
      "logits/rejected": 0.137881338596344,
      "logps/chosen": -506.595947265625,
      "logps/rejected": -518.34716796875,
      "loss": 0.6199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5067024230957031,
      "rewards/margins": 0.47749561071395874,
      "rewards/rejected": -0.9841979742050171,
      "step": 2557
    },
    {
      "epoch": 0.33473460374580843,
      "grad_norm": 8.3184814453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2739894390106201,
      "logits/rejected": 0.11065680533647537,
      "logps/chosen": -362.23565673828125,
      "logps/rejected": -416.61273193359375,
      "loss": 0.5963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5718311071395874,
      "rewards/margins": 0.7236354947090149,
      "rewards/rejected": -1.2954665422439575,
      "step": 2558
    },
    {
      "epoch": 0.3348654616831602,
      "grad_norm": 5.815393924713135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3402404189109802,
      "logits/rejected": 0.2886093556880951,
      "logps/chosen": -519.9777221679688,
      "logps/rejected": -567.4358520507812,
      "loss": 0.4054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16054420173168182,
      "rewards/margins": 0.8164123296737671,
      "rewards/rejected": -0.6558681130409241,
      "step": 2559
    },
    {
      "epoch": 0.33499631962051196,
      "grad_norm": 8.569677352905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10366542637348175,
      "logits/rejected": -0.10226130485534668,
      "logps/chosen": -533.7744750976562,
      "logps/rejected": -424.83050537109375,
      "loss": 0.5871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8784816861152649,
      "rewards/margins": 0.6242202520370483,
      "rewards/rejected": -1.5027018785476685,
      "step": 2560
    },
    {
      "epoch": 0.33512717755786375,
      "grad_norm": 4.890493392944336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21965965628623962,
      "logits/rejected": 0.3144910931587219,
      "logps/chosen": -387.86029052734375,
      "logps/rejected": -432.0476989746094,
      "loss": 0.4447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31725427508354187,
      "rewards/margins": 0.8469136953353882,
      "rewards/rejected": -1.1641680002212524,
      "step": 2561
    },
    {
      "epoch": 0.3352580354952155,
      "grad_norm": 6.4283766746521,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11003343760967255,
      "logits/rejected": -0.0637350082397461,
      "logps/chosen": -540.2581176757812,
      "logps/rejected": -403.0134582519531,
      "loss": 0.4822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11477621644735336,
      "rewards/margins": 0.8915661573410034,
      "rewards/rejected": -0.7767899036407471,
      "step": 2562
    },
    {
      "epoch": 0.3353888934325673,
      "grad_norm": 5.367639541625977,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32970860600471497,
      "logits/rejected": 0.285391241312027,
      "logps/chosen": -638.2529907226562,
      "logps/rejected": -526.322265625,
      "loss": 0.3655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3819133937358856,
      "rewards/margins": 1.0827897787094116,
      "rewards/rejected": -0.7008764147758484,
      "step": 2563
    },
    {
      "epoch": 0.335519751369919,
      "grad_norm": 7.697556495666504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14938586950302124,
      "logits/rejected": -0.05947383493185043,
      "logps/chosen": -526.292236328125,
      "logps/rejected": -580.2706909179688,
      "loss": 0.6242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26451489329338074,
      "rewards/margins": 0.5717241764068604,
      "rewards/rejected": -0.8362390398979187,
      "step": 2564
    },
    {
      "epoch": 0.3356506093072708,
      "grad_norm": 5.810052394866943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027585066854953766,
      "logits/rejected": 0.14997358620166779,
      "logps/chosen": -397.4346618652344,
      "logps/rejected": -531.4525146484375,
      "loss": 0.3643,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4155685603618622,
      "rewards/margins": 1.4869681596755981,
      "rewards/rejected": -1.9025366306304932,
      "step": 2565
    },
    {
      "epoch": 0.33578146724462254,
      "grad_norm": 5.608323574066162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23330679535865784,
      "logits/rejected": 0.2337925285100937,
      "logps/chosen": -592.7562866210938,
      "logps/rejected": -444.226806640625,
      "loss": 0.3811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04145699739456177,
      "rewards/margins": 1.238499402999878,
      "rewards/rejected": -1.197042465209961,
      "step": 2566
    },
    {
      "epoch": 0.33591232518197434,
      "grad_norm": 9.362001419067383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29874828457832336,
      "logits/rejected": 0.04407113045454025,
      "logps/chosen": -313.0478820800781,
      "logps/rejected": -458.1001892089844,
      "loss": 0.7741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0260515213012695,
      "rewards/margins": 0.19421391189098358,
      "rewards/rejected": -1.2202653884887695,
      "step": 2567
    },
    {
      "epoch": 0.33604318311932607,
      "grad_norm": 7.791006088256836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3247649371623993,
      "logits/rejected": 0.2716165781021118,
      "logps/chosen": -405.67059326171875,
      "logps/rejected": -470.34930419921875,
      "loss": 0.5357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6460660696029663,
      "rewards/margins": 0.602634608745575,
      "rewards/rejected": -1.2487008571624756,
      "step": 2568
    },
    {
      "epoch": 0.33617404105667786,
      "grad_norm": 6.015200614929199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06261536478996277,
      "logits/rejected": 0.16103661060333252,
      "logps/chosen": -467.9384460449219,
      "logps/rejected": -352.8851623535156,
      "loss": 0.5088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08344957232475281,
      "rewards/margins": 1.1287541389465332,
      "rewards/rejected": -1.0453044176101685,
      "step": 2569
    },
    {
      "epoch": 0.3363048989940296,
      "grad_norm": 7.850216388702393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28710514307022095,
      "logits/rejected": 0.4624740183353424,
      "logps/chosen": -448.6490173339844,
      "logps/rejected": -550.4600830078125,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11319670081138611,
      "rewards/margins": 0.7890278697013855,
      "rewards/rejected": -0.9022245407104492,
      "step": 2570
    },
    {
      "epoch": 0.3364357569313814,
      "grad_norm": 5.4958577156066895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09676352143287659,
      "logits/rejected": 0.007356524467468262,
      "logps/chosen": -496.2429504394531,
      "logps/rejected": -563.740478515625,
      "loss": 0.3665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02865046262741089,
      "rewards/margins": 1.2498974800109863,
      "rewards/rejected": -1.2212471961975098,
      "step": 2571
    },
    {
      "epoch": 0.3365666148687331,
      "grad_norm": 9.004769325256348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3007517158985138,
      "logits/rejected": 0.35235071182250977,
      "logps/chosen": -320.0368347167969,
      "logps/rejected": -384.6864929199219,
      "loss": 0.6631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3106144070625305,
      "rewards/margins": 0.8614492416381836,
      "rewards/rejected": -1.1720637083053589,
      "step": 2572
    },
    {
      "epoch": 0.3366974728060849,
      "grad_norm": 6.336631774902344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0011574607342481613,
      "logits/rejected": 0.0913311168551445,
      "logps/chosen": -774.2779541015625,
      "logps/rejected": -480.6007080078125,
      "loss": 0.4379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6698570251464844,
      "rewards/margins": 0.7390371561050415,
      "rewards/rejected": -1.4088943004608154,
      "step": 2573
    },
    {
      "epoch": 0.33682833074343665,
      "grad_norm": 7.092439651489258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07146033644676208,
      "logits/rejected": 0.0013657435774803162,
      "logps/chosen": -580.0609130859375,
      "logps/rejected": -395.2118835449219,
      "loss": 0.669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7651845216751099,
      "rewards/margins": 0.4331185817718506,
      "rewards/rejected": -1.1983031034469604,
      "step": 2574
    },
    {
      "epoch": 0.33695918868078845,
      "grad_norm": 8.706648826599121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.332122802734375,
      "logits/rejected": 0.033788494765758514,
      "logps/chosen": -706.4805297851562,
      "logps/rejected": -498.40985107421875,
      "loss": 0.5741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.022061537951231003,
      "rewards/margins": 1.0387684106826782,
      "rewards/rejected": -1.0608298778533936,
      "step": 2575
    },
    {
      "epoch": 0.3370900466181402,
      "grad_norm": 4.017102241516113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33538809418678284,
      "logits/rejected": 0.24540108442306519,
      "logps/chosen": -457.7286071777344,
      "logps/rejected": -386.629150390625,
      "loss": 0.2651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06863803416490555,
      "rewards/margins": 1.294205665588379,
      "rewards/rejected": -1.3628437519073486,
      "step": 2576
    },
    {
      "epoch": 0.3372209045554919,
      "grad_norm": 6.725722789764404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07071331888437271,
      "logits/rejected": -0.23130179941654205,
      "logps/chosen": -360.6656188964844,
      "logps/rejected": -290.50457763671875,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7335583567619324,
      "rewards/margins": 0.39423367381095886,
      "rewards/rejected": -1.1277921199798584,
      "step": 2577
    },
    {
      "epoch": 0.3373517624928437,
      "grad_norm": 7.082621097564697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4413048326969147,
      "logits/rejected": 0.2952350080013275,
      "logps/chosen": -357.1819763183594,
      "logps/rejected": -376.263671875,
      "loss": 0.7442,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7871723175048828,
      "rewards/margins": 0.01974637806415558,
      "rewards/rejected": -0.806918740272522,
      "step": 2578
    },
    {
      "epoch": 0.33748262043019545,
      "grad_norm": 11.904397964477539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24853071570396423,
      "logits/rejected": -0.03746151924133301,
      "logps/chosen": -466.8661804199219,
      "logps/rejected": -343.82464599609375,
      "loss": 0.63,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7940990924835205,
      "rewards/margins": 0.575384259223938,
      "rewards/rejected": -1.369483232498169,
      "step": 2579
    },
    {
      "epoch": 0.33761347836754724,
      "grad_norm": 13.032421112060547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1416279375553131,
      "logits/rejected": -0.20596756041049957,
      "logps/chosen": -388.6712646484375,
      "logps/rejected": -345.7817687988281,
      "loss": 1.1644,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1255614757537842,
      "rewards/margins": -0.5605311989784241,
      "rewards/rejected": -0.5650302767753601,
      "step": 2580
    },
    {
      "epoch": 0.337744336304899,
      "grad_norm": 7.711273670196533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16337846219539642,
      "logits/rejected": 0.047980595380067825,
      "logps/chosen": -428.8017578125,
      "logps/rejected": -369.312255859375,
      "loss": 0.6762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13722924888134003,
      "rewards/margins": 0.12517938017845154,
      "rewards/rejected": -0.26240867376327515,
      "step": 2581
    },
    {
      "epoch": 0.33787519424225076,
      "grad_norm": 4.5739312171936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16227953135967255,
      "logits/rejected": 0.03953593224287033,
      "logps/chosen": -298.1518859863281,
      "logps/rejected": -365.94561767578125,
      "loss": 0.3508,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39548972249031067,
      "rewards/margins": 1.0163099765777588,
      "rewards/rejected": -1.411799669265747,
      "step": 2582
    },
    {
      "epoch": 0.3380060521796025,
      "grad_norm": 13.957417488098145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044632673263549805,
      "logits/rejected": -0.13841553032398224,
      "logps/chosen": -468.9245300292969,
      "logps/rejected": -465.41552734375,
      "loss": 1.5017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0613393783569336,
      "rewards/margins": -0.6541433930397034,
      "rewards/rejected": -0.407196044921875,
      "step": 2583
    },
    {
      "epoch": 0.3381369101169543,
      "grad_norm": 7.266303062438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02769409492611885,
      "logits/rejected": 0.13703857362270355,
      "logps/chosen": -434.3662109375,
      "logps/rejected": -584.773681640625,
      "loss": 0.6473,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.49697136878967285,
      "rewards/margins": 0.534685492515564,
      "rewards/rejected": -1.0316569805145264,
      "step": 2584
    },
    {
      "epoch": 0.33826776805430603,
      "grad_norm": 8.770031929016113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10830484330654144,
      "logits/rejected": 0.11813569813966751,
      "logps/chosen": -493.603271484375,
      "logps/rejected": -384.7580261230469,
      "loss": 0.6044,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.028345942497253418,
      "rewards/margins": 0.48529618978500366,
      "rewards/rejected": -0.45695018768310547,
      "step": 2585
    },
    {
      "epoch": 0.3383986259916578,
      "grad_norm": 8.223259925842285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08844403922557831,
      "logits/rejected": -0.12309706211090088,
      "logps/chosen": -553.5131225585938,
      "logps/rejected": -429.49310302734375,
      "loss": 0.7341,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7907286286354065,
      "rewards/margins": 0.0835515707731247,
      "rewards/rejected": -0.8742802143096924,
      "step": 2586
    },
    {
      "epoch": 0.33852948392900956,
      "grad_norm": 5.190454959869385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24447326362133026,
      "logits/rejected": -0.38243138790130615,
      "logps/chosen": -490.86956787109375,
      "logps/rejected": -537.24169921875,
      "loss": 0.321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19529074430465698,
      "rewards/margins": 1.4325032234191895,
      "rewards/rejected": -1.6277940273284912,
      "step": 2587
    },
    {
      "epoch": 0.33866034186636135,
      "grad_norm": 4.738941192626953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04022117704153061,
      "logits/rejected": 0.13499730825424194,
      "logps/chosen": -294.59710693359375,
      "logps/rejected": -448.897705078125,
      "loss": 0.3642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.022379979491233826,
      "rewards/margins": 1.1401616334915161,
      "rewards/rejected": -1.1625416278839111,
      "step": 2588
    },
    {
      "epoch": 0.3387911998037131,
      "grad_norm": 5.934850692749023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13746382296085358,
      "logits/rejected": -0.15334491431713104,
      "logps/chosen": -451.2112121582031,
      "logps/rejected": -529.716796875,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7089078426361084,
      "rewards/margins": 0.9591038227081299,
      "rewards/rejected": -0.2501958906650543,
      "step": 2589
    },
    {
      "epoch": 0.3389220577410649,
      "grad_norm": 5.9692606925964355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008722670376300812,
      "logits/rejected": -0.17795413732528687,
      "logps/chosen": -525.825927734375,
      "logps/rejected": -377.7695007324219,
      "loss": 0.4685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4305824041366577,
      "rewards/margins": 1.1944776773452759,
      "rewards/rejected": -0.7638952732086182,
      "step": 2590
    },
    {
      "epoch": 0.3390529156784166,
      "grad_norm": 5.697049140930176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4082014262676239,
      "logits/rejected": 0.5272573232650757,
      "logps/chosen": -513.3453369140625,
      "logps/rejected": -485.6580810546875,
      "loss": 0.507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21181431412696838,
      "rewards/margins": 0.6392147541046143,
      "rewards/rejected": -0.4274004399776459,
      "step": 2591
    },
    {
      "epoch": 0.3391837736157684,
      "grad_norm": 4.774549961090088,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15451428294181824,
      "logits/rejected": -0.13044220209121704,
      "logps/chosen": -419.68170166015625,
      "logps/rejected": -363.74273681640625,
      "loss": 0.4274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6580110788345337,
      "rewards/margins": 0.9118595123291016,
      "rewards/rejected": -0.25384846329689026,
      "step": 2592
    },
    {
      "epoch": 0.33931463155312014,
      "grad_norm": 15.723244667053223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024119973182678223,
      "logits/rejected": -0.180461123585701,
      "logps/chosen": -615.7985229492188,
      "logps/rejected": -546.3036499023438,
      "loss": 1.1798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6841498613357544,
      "rewards/margins": 0.23545096814632416,
      "rewards/rejected": -0.9196008443832397,
      "step": 2593
    },
    {
      "epoch": 0.33944548949047193,
      "grad_norm": 8.798073768615723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12402068078517914,
      "logits/rejected": 0.09744248539209366,
      "logps/chosen": -367.58087158203125,
      "logps/rejected": -289.8374938964844,
      "loss": 0.8185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6290515661239624,
      "rewards/margins": -0.1312643140554428,
      "rewards/rejected": -0.4977872967720032,
      "step": 2594
    },
    {
      "epoch": 0.33957634742782367,
      "grad_norm": 8.831144332885742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1520031988620758,
      "logits/rejected": -0.12474146485328674,
      "logps/chosen": -318.74951171875,
      "logps/rejected": -474.5628662109375,
      "loss": 0.7434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48723310232162476,
      "rewards/margins": 0.19116482138633728,
      "rewards/rejected": -0.678398072719574,
      "step": 2595
    },
    {
      "epoch": 0.33970720536517546,
      "grad_norm": 8.087857246398926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3205213248729706,
      "logits/rejected": 0.2103918045759201,
      "logps/chosen": -549.1050415039062,
      "logps/rejected": -394.94525146484375,
      "loss": 0.4827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3855319917201996,
      "rewards/margins": 0.7510812878608704,
      "rewards/rejected": -1.1366132497787476,
      "step": 2596
    },
    {
      "epoch": 0.3398380633025272,
      "grad_norm": 8.559123992919922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21842217445373535,
      "logits/rejected": 0.2889094948768616,
      "logps/chosen": -417.2069091796875,
      "logps/rejected": -413.741455078125,
      "loss": 0.7366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3114784359931946,
      "rewards/margins": 0.06609094142913818,
      "rewards/rejected": -0.37756940722465515,
      "step": 2597
    },
    {
      "epoch": 0.33996892123987893,
      "grad_norm": 4.55261754989624,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009438768029212952,
      "logits/rejected": -0.03366093710064888,
      "logps/chosen": -377.635009765625,
      "logps/rejected": -391.4192199707031,
      "loss": 0.3149,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.36885929107666016,
      "rewards/margins": 1.3067545890808105,
      "rewards/rejected": -0.9378952383995056,
      "step": 2598
    },
    {
      "epoch": 0.3400997791772307,
      "grad_norm": 5.372322082519531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07398835569620132,
      "logits/rejected": 0.15125395357608795,
      "logps/chosen": -383.0791015625,
      "logps/rejected": -300.97479248046875,
      "loss": 0.4298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4141084551811218,
      "rewards/margins": 0.9137102365493774,
      "rewards/rejected": -0.49960172176361084,
      "step": 2599
    },
    {
      "epoch": 0.34023063711458246,
      "grad_norm": 7.545752048492432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17333218455314636,
      "logits/rejected": 0.10019244998693466,
      "logps/chosen": -437.6191101074219,
      "logps/rejected": -539.9420166015625,
      "loss": 0.3923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1929776668548584,
      "rewards/margins": 1.37742280960083,
      "rewards/rejected": -2.5704007148742676,
      "step": 2600
    },
    {
      "epoch": 0.34036149505193425,
      "grad_norm": 6.060640811920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08665049076080322,
      "logits/rejected": 0.13329851627349854,
      "logps/chosen": -314.2100830078125,
      "logps/rejected": -358.81256103515625,
      "loss": 0.5076,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.241542249917984,
      "rewards/margins": 0.7287570238113403,
      "rewards/rejected": -0.9702993631362915,
      "step": 2601
    },
    {
      "epoch": 0.340492352989286,
      "grad_norm": 7.435965061187744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08986514806747437,
      "logits/rejected": -0.012454114854335785,
      "logps/chosen": -463.4823303222656,
      "logps/rejected": -414.47357177734375,
      "loss": 0.6048,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5639060735702515,
      "rewards/margins": 0.4159092903137207,
      "rewards/rejected": 0.14799679815769196,
      "step": 2602
    },
    {
      "epoch": 0.3406232109266378,
      "grad_norm": 4.333676815032959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08295343816280365,
      "logits/rejected": 0.15209481120109558,
      "logps/chosen": -417.3841552734375,
      "logps/rejected": -320.57049560546875,
      "loss": 0.3478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.012685582041740417,
      "rewards/margins": 1.0330287218093872,
      "rewards/rejected": -1.0457143783569336,
      "step": 2603
    },
    {
      "epoch": 0.3407540688639895,
      "grad_norm": 11.102158546447754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12593455612659454,
      "logits/rejected": -0.17354783415794373,
      "logps/chosen": -424.22467041015625,
      "logps/rejected": -432.3473815917969,
      "loss": 1.0359,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15186256170272827,
      "rewards/margins": -0.045604586601257324,
      "rewards/rejected": -0.10625803470611572,
      "step": 2604
    },
    {
      "epoch": 0.3408849268013413,
      "grad_norm": 6.787790775299072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07624475657939911,
      "logits/rejected": 0.13271203637123108,
      "logps/chosen": -617.359375,
      "logps/rejected": -487.678955078125,
      "loss": 0.3949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5590126514434814,
      "rewards/margins": 1.0382623672485352,
      "rewards/rejected": -0.4792497754096985,
      "step": 2605
    },
    {
      "epoch": 0.34101578473869304,
      "grad_norm": 8.623847007751465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02196040004491806,
      "logits/rejected": -0.00046423450112342834,
      "logps/chosen": -472.3810729980469,
      "logps/rejected": -588.3408813476562,
      "loss": 0.7867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45508885383605957,
      "rewards/margins": 0.12862932682037354,
      "rewards/rejected": -0.5837181210517883,
      "step": 2606
    },
    {
      "epoch": 0.34114664267604483,
      "grad_norm": 5.9953203201293945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1961393654346466,
      "logits/rejected": 0.07012402266263962,
      "logps/chosen": -612.3756713867188,
      "logps/rejected": -430.0191955566406,
      "loss": 0.409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10293063521385193,
      "rewards/margins": 1.1435637474060059,
      "rewards/rejected": -1.040632963180542,
      "step": 2607
    },
    {
      "epoch": 0.34127750061339657,
      "grad_norm": 7.739229202270508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2819836139678955,
      "logits/rejected": 0.1863039880990982,
      "logps/chosen": -540.4957275390625,
      "logps/rejected": -697.001708984375,
      "loss": 0.5552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15116539597511292,
      "rewards/margins": 0.4963691234588623,
      "rewards/rejected": -0.6475346088409424,
      "step": 2608
    },
    {
      "epoch": 0.34140835855074836,
      "grad_norm": 6.6242756843566895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2520138919353485,
      "logits/rejected": 0.21257658302783966,
      "logps/chosen": -503.2015686035156,
      "logps/rejected": -379.81365966796875,
      "loss": 0.4899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12174069881439209,
      "rewards/margins": 0.9330399632453918,
      "rewards/rejected": -0.8112993240356445,
      "step": 2609
    },
    {
      "epoch": 0.3415392164881001,
      "grad_norm": 4.717732906341553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19534818828105927,
      "logits/rejected": -0.1338779777288437,
      "logps/chosen": -436.0089111328125,
      "logps/rejected": -319.98883056640625,
      "loss": 0.4227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009375393390655518,
      "rewards/margins": 1.0896003246307373,
      "rewards/rejected": -1.098975658416748,
      "step": 2610
    },
    {
      "epoch": 0.3416700744254519,
      "grad_norm": 6.124867916107178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09645967185497284,
      "logits/rejected": -0.004873715341091156,
      "logps/chosen": -314.7618408203125,
      "logps/rejected": -342.2394104003906,
      "loss": 0.5216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43100491166114807,
      "rewards/margins": 0.5237419605255127,
      "rewards/rejected": -0.9547468423843384,
      "step": 2611
    },
    {
      "epoch": 0.3418009323628036,
      "grad_norm": 5.127488613128662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27076300978660583,
      "logits/rejected": 0.24376805126667023,
      "logps/chosen": -453.7128601074219,
      "logps/rejected": -350.3389892578125,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8985637426376343,
      "rewards/margins": 1.335994005203247,
      "rewards/rejected": -0.437430202960968,
      "step": 2612
    },
    {
      "epoch": 0.3419317903001554,
      "grad_norm": 6.2612714767456055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17316873371601105,
      "logits/rejected": 0.06897418946027756,
      "logps/chosen": -520.985595703125,
      "logps/rejected": -546.3903198242188,
      "loss": 0.4251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019021227955818176,
      "rewards/margins": 1.073890209197998,
      "rewards/rejected": -1.0548690557479858,
      "step": 2613
    },
    {
      "epoch": 0.34206264823750715,
      "grad_norm": 5.235525608062744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28705665469169617,
      "logits/rejected": 0.18990877270698547,
      "logps/chosen": -638.5985107421875,
      "logps/rejected": -613.4063720703125,
      "loss": 0.3476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7614339590072632,
      "rewards/margins": 0.9898893237113953,
      "rewards/rejected": -0.2284553349018097,
      "step": 2614
    },
    {
      "epoch": 0.34219350617485894,
      "grad_norm": 7.01979923248291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03694605827331543,
      "logits/rejected": -0.06860757619142532,
      "logps/chosen": -432.0781555175781,
      "logps/rejected": -481.13677978515625,
      "loss": 0.5053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6626308560371399,
      "rewards/margins": 0.6997194290161133,
      "rewards/rejected": -0.03708857297897339,
      "step": 2615
    },
    {
      "epoch": 0.3423243641122107,
      "grad_norm": 7.140203475952148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2747926712036133,
      "logits/rejected": -0.12839436531066895,
      "logps/chosen": -470.73406982421875,
      "logps/rejected": -325.7391357421875,
      "loss": 0.4293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08330629765987396,
      "rewards/margins": 0.841909646987915,
      "rewards/rejected": -0.9252158999443054,
      "step": 2616
    },
    {
      "epoch": 0.34245522204956247,
      "grad_norm": 6.63231897354126,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08881187438964844,
      "logits/rejected": -0.05224902927875519,
      "logps/chosen": -412.1993713378906,
      "logps/rejected": -411.4426574707031,
      "loss": 0.4901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2233985960483551,
      "rewards/margins": 0.7488329410552979,
      "rewards/rejected": -0.9722314476966858,
      "step": 2617
    },
    {
      "epoch": 0.3425860799869142,
      "grad_norm": 2.326691150665283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0377679318189621,
      "logits/rejected": -0.247401624917984,
      "logps/chosen": -675.7198486328125,
      "logps/rejected": -474.48223876953125,
      "loss": 0.156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9702900052070618,
      "rewards/margins": 2.0160164833068848,
      "rewards/rejected": -1.0457264184951782,
      "step": 2618
    },
    {
      "epoch": 0.342716937924266,
      "grad_norm": 7.601378440856934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15417645871639252,
      "logits/rejected": -0.07154355198144913,
      "logps/chosen": -593.6126708984375,
      "logps/rejected": -466.98724365234375,
      "loss": 0.5436,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2294580191373825,
      "rewards/margins": 0.7098002433776855,
      "rewards/rejected": -0.9392582178115845,
      "step": 2619
    },
    {
      "epoch": 0.34284779586161773,
      "grad_norm": 8.413865089416504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06434880197048187,
      "logits/rejected": -0.05668933317065239,
      "logps/chosen": -498.85650634765625,
      "logps/rejected": -476.50750732421875,
      "loss": 0.7396,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15011084079742432,
      "rewards/margins": 0.31984540820121765,
      "rewards/rejected": -0.16973459720611572,
      "step": 2620
    },
    {
      "epoch": 0.34297865379896947,
      "grad_norm": 7.347037315368652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3395062983036041,
      "logits/rejected": 0.2215469777584076,
      "logps/chosen": -538.4254760742188,
      "logps/rejected": -586.1910400390625,
      "loss": 0.419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36471229791641235,
      "rewards/margins": 1.0020421743392944,
      "rewards/rejected": -0.6373298168182373,
      "step": 2621
    },
    {
      "epoch": 0.34310951173632126,
      "grad_norm": 5.853360176086426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14669430255889893,
      "logits/rejected": 0.13202843070030212,
      "logps/chosen": -496.2900390625,
      "logps/rejected": -412.72900390625,
      "loss": 0.4855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04585150629281998,
      "rewards/margins": 0.6825146079063416,
      "rewards/rejected": -0.636663019657135,
      "step": 2622
    },
    {
      "epoch": 0.343240369673673,
      "grad_norm": 8.205418586730957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06533890217542648,
      "logits/rejected": -0.09288111329078674,
      "logps/chosen": -400.82965087890625,
      "logps/rejected": -428.8177490234375,
      "loss": 0.5901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44467127323150635,
      "rewards/margins": 1.18804132938385,
      "rewards/rejected": -1.6327126026153564,
      "step": 2623
    },
    {
      "epoch": 0.3433712276110248,
      "grad_norm": 8.136817932128906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31628769636154175,
      "logits/rejected": 0.173055961728096,
      "logps/chosen": -569.856201171875,
      "logps/rejected": -610.5840454101562,
      "loss": 0.5402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6075612902641296,
      "rewards/margins": 1.2241346836090088,
      "rewards/rejected": -1.8316959142684937,
      "step": 2624
    },
    {
      "epoch": 0.3435020855483765,
      "grad_norm": 8.922301292419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1470356285572052,
      "logits/rejected": -0.0036431588232517242,
      "logps/chosen": -433.8645324707031,
      "logps/rejected": -431.9565124511719,
      "loss": 0.6666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5376154184341431,
      "rewards/margins": 0.32729780673980713,
      "rewards/rejected": -0.8649132251739502,
      "step": 2625
    },
    {
      "epoch": 0.3436329434857283,
      "grad_norm": 8.086291313171387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1900230497121811,
      "logits/rejected": -0.020916786044836044,
      "logps/chosen": -352.76470947265625,
      "logps/rejected": -380.64788818359375,
      "loss": 0.7848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5255423188209534,
      "rewards/margins": 0.32034778594970703,
      "rewards/rejected": -0.8458900451660156,
      "step": 2626
    },
    {
      "epoch": 0.34376380142308005,
      "grad_norm": 7.946427345275879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.098875030875206,
      "logits/rejected": 0.13255730271339417,
      "logps/chosen": -319.60955810546875,
      "logps/rejected": -350.5718994140625,
      "loss": 0.8314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.978870153427124,
      "rewards/margins": -0.12806068360805511,
      "rewards/rejected": -0.8508095741271973,
      "step": 2627
    },
    {
      "epoch": 0.34389465936043184,
      "grad_norm": 5.9205708503723145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1681087762117386,
      "logits/rejected": -0.07939848303794861,
      "logps/chosen": -647.3557739257812,
      "logps/rejected": -536.4013061523438,
      "loss": 0.3556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29709047079086304,
      "rewards/margins": 1.7043648958206177,
      "rewards/rejected": -1.4072741270065308,
      "step": 2628
    },
    {
      "epoch": 0.3440255172977836,
      "grad_norm": 8.908895492553711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029765285551548004,
      "logits/rejected": -0.025532439351081848,
      "logps/chosen": -438.14312744140625,
      "logps/rejected": -435.4351806640625,
      "loss": 0.7267,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5284456014633179,
      "rewards/margins": 0.5676472783088684,
      "rewards/rejected": -1.0960930585861206,
      "step": 2629
    },
    {
      "epoch": 0.34415637523513537,
      "grad_norm": 4.14710807800293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2630467116832733,
      "logits/rejected": 0.028194710612297058,
      "logps/chosen": -534.4769287109375,
      "logps/rejected": -375.0222473144531,
      "loss": 0.2681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24974662065505981,
      "rewards/margins": 1.5426907539367676,
      "rewards/rejected": -1.2929441928863525,
      "step": 2630
    },
    {
      "epoch": 0.3442872331724871,
      "grad_norm": 5.372192859649658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33853399753570557,
      "logits/rejected": 0.2862871587276459,
      "logps/chosen": -363.00970458984375,
      "logps/rejected": -391.9215393066406,
      "loss": 0.4682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10957097262144089,
      "rewards/margins": 0.6976453065872192,
      "rewards/rejected": -0.8072162866592407,
      "step": 2631
    },
    {
      "epoch": 0.3444180911098389,
      "grad_norm": 5.475957870483398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14814087748527527,
      "logits/rejected": 0.23507176339626312,
      "logps/chosen": -401.5270690917969,
      "logps/rejected": -352.7998352050781,
      "loss": 0.4387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31173115968704224,
      "rewards/margins": 0.7995718717575073,
      "rewards/rejected": -0.4878406524658203,
      "step": 2632
    },
    {
      "epoch": 0.34454894904719063,
      "grad_norm": 7.405367851257324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18110404908657074,
      "logits/rejected": 0.167522132396698,
      "logps/chosen": -478.6177062988281,
      "logps/rejected": -492.0130615234375,
      "loss": 0.5485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.488578200340271,
      "rewards/margins": 0.5952808856964111,
      "rewards/rejected": -0.10670261085033417,
      "step": 2633
    },
    {
      "epoch": 0.3446798069845424,
      "grad_norm": 4.193258285522461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029030974954366684,
      "logits/rejected": -0.20089462399482727,
      "logps/chosen": -447.1661376953125,
      "logps/rejected": -427.1068115234375,
      "loss": 0.2687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2163192629814148,
      "rewards/margins": 1.5405548810958862,
      "rewards/rejected": -1.7568740844726562,
      "step": 2634
    },
    {
      "epoch": 0.34481066492189416,
      "grad_norm": 5.9553608894348145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5397298336029053,
      "logits/rejected": 0.19081787765026093,
      "logps/chosen": -557.4847412109375,
      "logps/rejected": -517.73193359375,
      "loss": 0.4129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8170082569122314,
      "rewards/margins": 1.3755720853805542,
      "rewards/rejected": -0.5585638284683228,
      "step": 2635
    },
    {
      "epoch": 0.34494152285924595,
      "grad_norm": 7.5131001472473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16280163824558258,
      "logits/rejected": 0.21869327127933502,
      "logps/chosen": -509.03179931640625,
      "logps/rejected": -416.4168395996094,
      "loss": 0.7199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8353285193443298,
      "rewards/margins": 0.4175815284252167,
      "rewards/rejected": -1.2529101371765137,
      "step": 2636
    },
    {
      "epoch": 0.3450723807965977,
      "grad_norm": 8.656208992004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09668523073196411,
      "logits/rejected": 0.04556707292795181,
      "logps/chosen": -438.7984313964844,
      "logps/rejected": -436.21893310546875,
      "loss": 0.7767,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38057634234428406,
      "rewards/margins": 0.19113582372665405,
      "rewards/rejected": -0.5717121958732605,
      "step": 2637
    },
    {
      "epoch": 0.3452032387339495,
      "grad_norm": 8.688828468322754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3193919360637665,
      "logits/rejected": 0.36752986907958984,
      "logps/chosen": -463.4915771484375,
      "logps/rejected": -368.2093200683594,
      "loss": 0.7621,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25914156436920166,
      "rewards/margins": 0.29058897495269775,
      "rewards/rejected": -0.5497304797172546,
      "step": 2638
    },
    {
      "epoch": 0.3453340966713012,
      "grad_norm": 5.294927597045898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012397374026477337,
      "logits/rejected": 0.0397227443754673,
      "logps/chosen": -277.2093811035156,
      "logps/rejected": -357.4714660644531,
      "loss": 0.3036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4681657552719116,
      "rewards/margins": 1.2704217433929443,
      "rewards/rejected": -0.8022558689117432,
      "step": 2639
    },
    {
      "epoch": 0.345464954608653,
      "grad_norm": 10.108304977416992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09181426465511322,
      "logits/rejected": -0.08069124817848206,
      "logps/chosen": -460.61468505859375,
      "logps/rejected": -359.3057861328125,
      "loss": 0.6252,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08498707413673401,
      "rewards/margins": 0.6991734504699707,
      "rewards/rejected": -0.7841604351997375,
      "step": 2640
    },
    {
      "epoch": 0.34559581254600474,
      "grad_norm": 8.766688346862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029687872156500816,
      "logits/rejected": -0.022891923785209656,
      "logps/chosen": -377.610595703125,
      "logps/rejected": -482.1239013671875,
      "loss": 0.7088,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07506828010082245,
      "rewards/margins": 0.2882595360279083,
      "rewards/rejected": -0.36332783102989197,
      "step": 2641
    },
    {
      "epoch": 0.3457266704833565,
      "grad_norm": 5.820639610290527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10171428322792053,
      "logits/rejected": 0.03802775591611862,
      "logps/chosen": -370.7802734375,
      "logps/rejected": -412.07891845703125,
      "loss": 0.4268,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.023546408861875534,
      "rewards/margins": 1.4566056728363037,
      "rewards/rejected": -1.4801522493362427,
      "step": 2642
    },
    {
      "epoch": 0.34585752842070827,
      "grad_norm": 5.5504560470581055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2814755141735077,
      "logits/rejected": 0.2647387981414795,
      "logps/chosen": -355.4819641113281,
      "logps/rejected": -421.99468994140625,
      "loss": 0.4343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1381637454032898,
      "rewards/margins": 0.6761883497238159,
      "rewards/rejected": -0.5380246639251709,
      "step": 2643
    },
    {
      "epoch": 0.34598838635806,
      "grad_norm": 9.06750774383545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21319907903671265,
      "logits/rejected": 0.12454725801944733,
      "logps/chosen": -565.4509887695312,
      "logps/rejected": -511.06805419921875,
      "loss": 0.634,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24351918697357178,
      "rewards/margins": 0.27757760882377625,
      "rewards/rejected": -0.03405839949846268,
      "step": 2644
    },
    {
      "epoch": 0.3461192442954118,
      "grad_norm": 7.983052730560303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10777100920677185,
      "logits/rejected": -0.10988233983516693,
      "logps/chosen": -526.448486328125,
      "logps/rejected": -471.8243103027344,
      "loss": 0.6175,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3938489854335785,
      "rewards/margins": 0.6521587371826172,
      "rewards/rejected": -0.2583097517490387,
      "step": 2645
    },
    {
      "epoch": 0.34625010223276353,
      "grad_norm": 6.687106132507324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10058750212192535,
      "logits/rejected": 0.11442616581916809,
      "logps/chosen": -553.0654907226562,
      "logps/rejected": -413.1399230957031,
      "loss": 0.4461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11953429877758026,
      "rewards/margins": 0.8672062754631042,
      "rewards/rejected": -0.747671902179718,
      "step": 2646
    },
    {
      "epoch": 0.3463809601701153,
      "grad_norm": 5.582443714141846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15017980337142944,
      "logits/rejected": 0.09972355514764786,
      "logps/chosen": -570.265625,
      "logps/rejected": -447.1462097167969,
      "loss": 0.3711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5036584734916687,
      "rewards/margins": 0.9200050234794617,
      "rewards/rejected": -1.4236634969711304,
      "step": 2647
    },
    {
      "epoch": 0.34651181810746706,
      "grad_norm": 7.113691806793213,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2640584707260132,
      "logits/rejected": 0.20122650265693665,
      "logps/chosen": -367.89794921875,
      "logps/rejected": -336.7449951171875,
      "loss": 0.6353,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30454352498054504,
      "rewards/margins": 0.5156343579292297,
      "rewards/rejected": -0.8201778531074524,
      "step": 2648
    },
    {
      "epoch": 0.34664267604481885,
      "grad_norm": 10.397868156433105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031017780303955078,
      "logits/rejected": 0.004858039319515228,
      "logps/chosen": -505.26031494140625,
      "logps/rejected": -469.06463623046875,
      "loss": 0.7023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04751094430685043,
      "rewards/margins": 0.45759323239326477,
      "rewards/rejected": -0.41008228063583374,
      "step": 2649
    },
    {
      "epoch": 0.3467735339821706,
      "grad_norm": 7.825443267822266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23458130657672882,
      "logits/rejected": 0.014166206121444702,
      "logps/chosen": -309.3811340332031,
      "logps/rejected": -366.9822998046875,
      "loss": 0.7451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46830520033836365,
      "rewards/margins": 0.5051030516624451,
      "rewards/rejected": -0.9734082221984863,
      "step": 2650
    },
    {
      "epoch": 0.3469043919195224,
      "grad_norm": 6.048358917236328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08776341378688812,
      "logits/rejected": 0.005052831489592791,
      "logps/chosen": -495.1998291015625,
      "logps/rejected": -423.05450439453125,
      "loss": 0.438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3603587746620178,
      "rewards/margins": 1.1031781435012817,
      "rewards/rejected": -1.4635370969772339,
      "step": 2651
    },
    {
      "epoch": 0.3470352498568741,
      "grad_norm": 6.807242393493652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0855940580368042,
      "logits/rejected": -0.08136876672506332,
      "logps/chosen": -418.7345886230469,
      "logps/rejected": -417.1159973144531,
      "loss": 0.552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.255497545003891,
      "rewards/margins": 1.0431783199310303,
      "rewards/rejected": -0.7876806259155273,
      "step": 2652
    },
    {
      "epoch": 0.3471661077942259,
      "grad_norm": 4.483278751373291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019494540989398956,
      "logits/rejected": -0.09088746458292007,
      "logps/chosen": -602.831787109375,
      "logps/rejected": -342.3515930175781,
      "loss": 0.3534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5303720831871033,
      "rewards/margins": 0.9988572001457214,
      "rewards/rejected": -0.4684850573539734,
      "step": 2653
    },
    {
      "epoch": 0.34729696573157764,
      "grad_norm": 6.002357482910156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0939716249704361,
      "logits/rejected": 0.1377955824136734,
      "logps/chosen": -426.50732421875,
      "logps/rejected": -424.45263671875,
      "loss": 0.5642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5093034505844116,
      "rewards/margins": 0.7263784408569336,
      "rewards/rejected": -0.2170749306678772,
      "step": 2654
    },
    {
      "epoch": 0.34742782366892944,
      "grad_norm": 7.554032802581787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03513967990875244,
      "logits/rejected": 0.23539823293685913,
      "logps/chosen": -578.4501342773438,
      "logps/rejected": -651.5045166015625,
      "loss": 0.635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2961044907569885,
      "rewards/margins": 0.7822480797767639,
      "rewards/rejected": -1.0783525705337524,
      "step": 2655
    },
    {
      "epoch": 0.3475586816062812,
      "grad_norm": 7.761306285858154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16991107165813446,
      "logits/rejected": 0.1176561564207077,
      "logps/chosen": -349.6426086425781,
      "logps/rejected": -407.42669677734375,
      "loss": 0.6412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7048217058181763,
      "rewards/margins": 0.4532754719257355,
      "rewards/rejected": -1.1580971479415894,
      "step": 2656
    },
    {
      "epoch": 0.34768953954363296,
      "grad_norm": 7.100572109222412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16299304366111755,
      "logits/rejected": 0.26233306527137756,
      "logps/chosen": -444.83856201171875,
      "logps/rejected": -460.9815368652344,
      "loss": 0.6617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.38166046142578125,
      "rewards/margins": 0.44479236006736755,
      "rewards/rejected": -0.0631319135427475,
      "step": 2657
    },
    {
      "epoch": 0.3478203974809847,
      "grad_norm": 8.285614013671875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07802119851112366,
      "logits/rejected": 0.01699760928750038,
      "logps/chosen": -402.0280456542969,
      "logps/rejected": -440.0773010253906,
      "loss": 0.7295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02120266854763031,
      "rewards/margins": 0.4615415036678314,
      "rewards/rejected": -0.4403388500213623,
      "step": 2658
    },
    {
      "epoch": 0.3479512554183365,
      "grad_norm": 8.192283630371094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17416298389434814,
      "logits/rejected": 0.17533138394355774,
      "logps/chosen": -331.30426025390625,
      "logps/rejected": -418.7242736816406,
      "loss": 0.6662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8912157416343689,
      "rewards/margins": 0.28000813722610474,
      "rewards/rejected": -1.1712238788604736,
      "step": 2659
    },
    {
      "epoch": 0.3480821133556882,
      "grad_norm": 7.907187461853027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12258504331111908,
      "logits/rejected": 0.3348972201347351,
      "logps/chosen": -422.90362548828125,
      "logps/rejected": -453.6216125488281,
      "loss": 0.7341,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7795678973197937,
      "rewards/margins": 0.01491868868470192,
      "rewards/rejected": -0.7944865822792053,
      "step": 2660
    },
    {
      "epoch": 0.34821297129304,
      "grad_norm": 7.729811668395996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23693841695785522,
      "logits/rejected": 0.36383143067359924,
      "logps/chosen": -467.14495849609375,
      "logps/rejected": -421.9332275390625,
      "loss": 0.8624,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.2163761556148529,
      "rewards/margins": 0.00026085972785949707,
      "rewards/rejected": 0.2161152958869934,
      "step": 2661
    },
    {
      "epoch": 0.34834382923039175,
      "grad_norm": 5.486823081970215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36784252524375916,
      "logits/rejected": 0.23779410123825073,
      "logps/chosen": -480.4083557128906,
      "logps/rejected": -308.8459167480469,
      "loss": 0.3464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1938818097114563,
      "rewards/margins": 1.1289777755737305,
      "rewards/rejected": -0.9350959658622742,
      "step": 2662
    },
    {
      "epoch": 0.3484746871677435,
      "grad_norm": 8.448789596557617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07049735635519028,
      "logits/rejected": -0.0285502877086401,
      "logps/chosen": -351.84564208984375,
      "logps/rejected": -286.92156982421875,
      "loss": 0.72,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7686352133750916,
      "rewards/margins": 0.12005902826786041,
      "rewards/rejected": -0.888694167137146,
      "step": 2663
    },
    {
      "epoch": 0.3486055451050953,
      "grad_norm": 6.425189018249512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13638563454151154,
      "logits/rejected": -0.03129206597805023,
      "logps/chosen": -489.5583801269531,
      "logps/rejected": -451.8594970703125,
      "loss": 0.4291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2969299554824829,
      "rewards/margins": 1.2107079029083252,
      "rewards/rejected": -0.9137779474258423,
      "step": 2664
    },
    {
      "epoch": 0.348736403042447,
      "grad_norm": 6.313145160675049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5710649490356445,
      "logits/rejected": 0.4398820102214813,
      "logps/chosen": -335.387451171875,
      "logps/rejected": -350.42218017578125,
      "loss": 0.7188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08395960181951523,
      "rewards/margins": 0.13350291550159454,
      "rewards/rejected": -0.21746255457401276,
      "step": 2665
    },
    {
      "epoch": 0.3488672609797988,
      "grad_norm": 6.599665641784668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5102659463882446,
      "logits/rejected": 0.4498392641544342,
      "logps/chosen": -608.713134765625,
      "logps/rejected": -412.2943115234375,
      "loss": 0.4926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2288791835308075,
      "rewards/margins": 1.1132363080978394,
      "rewards/rejected": -0.8843570947647095,
      "step": 2666
    },
    {
      "epoch": 0.34899811891715055,
      "grad_norm": 5.10775089263916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0002206568606197834,
      "logits/rejected": -0.03512882441282272,
      "logps/chosen": -447.2374267578125,
      "logps/rejected": -386.1870422363281,
      "loss": 0.4318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7278976440429688,
      "rewards/margins": 0.7477693557739258,
      "rewards/rejected": -1.4756669998168945,
      "step": 2667
    },
    {
      "epoch": 0.34912897685450234,
      "grad_norm": 5.99308967590332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3195475935935974,
      "logits/rejected": 0.19304637610912323,
      "logps/chosen": -473.3056640625,
      "logps/rejected": -354.92669677734375,
      "loss": 0.6015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08487912267446518,
      "rewards/margins": 0.5950179100036621,
      "rewards/rejected": -0.5101386904716492,
      "step": 2668
    },
    {
      "epoch": 0.3492598347918541,
      "grad_norm": 7.736264228820801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10157837718725204,
      "logits/rejected": -0.059804532676935196,
      "logps/chosen": -292.78619384765625,
      "logps/rejected": -338.34002685546875,
      "loss": 0.7089,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20947305858135223,
      "rewards/margins": 0.3606216311454773,
      "rewards/rejected": -0.15114861726760864,
      "step": 2669
    },
    {
      "epoch": 0.34939069272920587,
      "grad_norm": 6.258121967315674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2533315122127533,
      "logits/rejected": -0.26934725046157837,
      "logps/chosen": -440.6495361328125,
      "logps/rejected": -385.5466003417969,
      "loss": 0.3763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03165186941623688,
      "rewards/margins": 1.180763840675354,
      "rewards/rejected": -1.2124158143997192,
      "step": 2670
    },
    {
      "epoch": 0.3495215506665576,
      "grad_norm": 6.377252101898193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.038573943078517914,
      "logits/rejected": -0.1105358898639679,
      "logps/chosen": -278.76348876953125,
      "logps/rejected": -304.25836181640625,
      "loss": 0.7078,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11246856302022934,
      "rewards/margins": 0.17948663234710693,
      "rewards/rejected": -0.2919551432132721,
      "step": 2671
    },
    {
      "epoch": 0.3496524086039094,
      "grad_norm": 8.894331932067871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3120110332965851,
      "logits/rejected": 0.14574196934700012,
      "logps/chosen": -518.6799926757812,
      "logps/rejected": -602.6104736328125,
      "loss": 0.9921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2635098099708557,
      "rewards/margins": 0.09087830781936646,
      "rewards/rejected": -0.3543880581855774,
      "step": 2672
    },
    {
      "epoch": 0.34978326654126113,
      "grad_norm": 7.110199451446533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17050905525684357,
      "logits/rejected": 0.05179085582494736,
      "logps/chosen": -492.4148254394531,
      "logps/rejected": -509.4571228027344,
      "loss": 0.577,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3822803497314453,
      "rewards/margins": 1.286440134048462,
      "rewards/rejected": -0.9041597843170166,
      "step": 2673
    },
    {
      "epoch": 0.3499141244786129,
      "grad_norm": 5.68979024887085,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05104538053274155,
      "logits/rejected": 0.03223550319671631,
      "logps/chosen": -474.4488830566406,
      "logps/rejected": -383.70709228515625,
      "loss": 0.5375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5068917870521545,
      "rewards/margins": 0.6536579132080078,
      "rewards/rejected": -1.1605497598648071,
      "step": 2674
    },
    {
      "epoch": 0.35004498241596466,
      "grad_norm": 11.691845893859863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1446695625782013,
      "logits/rejected": -0.0957852154970169,
      "logps/chosen": -590.1998901367188,
      "logps/rejected": -394.5289611816406,
      "loss": 1.0041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6645544171333313,
      "rewards/margins": -0.0597345232963562,
      "rewards/rejected": -0.6048198938369751,
      "step": 2675
    },
    {
      "epoch": 0.35017584035331645,
      "grad_norm": 4.2031354904174805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0941459983587265,
      "logits/rejected": 0.10868701338768005,
      "logps/chosen": -514.5833740234375,
      "logps/rejected": -397.8175048828125,
      "loss": 0.3971,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21639388799667358,
      "rewards/margins": 1.340547800064087,
      "rewards/rejected": -1.1241538524627686,
      "step": 2676
    },
    {
      "epoch": 0.3503066982906682,
      "grad_norm": 4.365022659301758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3010736107826233,
      "logits/rejected": 0.3281205892562866,
      "logps/chosen": -447.03399658203125,
      "logps/rejected": -398.5978698730469,
      "loss": 0.2743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38657525181770325,
      "rewards/margins": 1.44698965549469,
      "rewards/rejected": -1.8335648775100708,
      "step": 2677
    },
    {
      "epoch": 0.35043755622802,
      "grad_norm": 5.612468242645264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3237605392932892,
      "logits/rejected": 0.01228567585349083,
      "logps/chosen": -597.3797607421875,
      "logps/rejected": -492.6654968261719,
      "loss": 0.3442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3295702040195465,
      "rewards/margins": 1.4199845790863037,
      "rewards/rejected": -1.0904144048690796,
      "step": 2678
    },
    {
      "epoch": 0.3505684141653717,
      "grad_norm": 7.433338165283203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10545580089092255,
      "logits/rejected": 0.22507773339748383,
      "logps/chosen": -486.49267578125,
      "logps/rejected": -370.25958251953125,
      "loss": 0.7186,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7877784967422485,
      "rewards/margins": 0.2719854414463043,
      "rewards/rejected": -1.05976402759552,
      "step": 2679
    },
    {
      "epoch": 0.3506992721027235,
      "grad_norm": 8.583916664123535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024289697408676147,
      "logits/rejected": 0.19500724971294403,
      "logps/chosen": -424.99609375,
      "logps/rejected": -474.6544189453125,
      "loss": 0.5657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07478520274162292,
      "rewards/margins": 0.6521247625350952,
      "rewards/rejected": -0.5773395299911499,
      "step": 2680
    },
    {
      "epoch": 0.35083013004007524,
      "grad_norm": 7.826239109039307,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0268509853631258,
      "logits/rejected": -0.015060343779623508,
      "logps/chosen": -380.0887451171875,
      "logps/rejected": -500.14227294921875,
      "loss": 0.8165,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.23306217789649963,
      "rewards/margins": 0.1357855200767517,
      "rewards/rejected": 0.09727667272090912,
      "step": 2681
    },
    {
      "epoch": 0.35096098797742703,
      "grad_norm": 5.973482131958008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3309051990509033,
      "logits/rejected": -0.21279527246952057,
      "logps/chosen": -446.9659423828125,
      "logps/rejected": -501.468994140625,
      "loss": 0.5695,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34214210510253906,
      "rewards/margins": 0.43470001220703125,
      "rewards/rejected": -0.7768421173095703,
      "step": 2682
    },
    {
      "epoch": 0.35109184591477877,
      "grad_norm": 4.164709568023682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08967477083206177,
      "logits/rejected": 0.1780359447002411,
      "logps/chosen": -433.6332092285156,
      "logps/rejected": -283.6617431640625,
      "loss": 0.4814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.017778590321540833,
      "rewards/margins": 0.8489572405815125,
      "rewards/rejected": -0.8667358160018921,
      "step": 2683
    },
    {
      "epoch": 0.35122270385213056,
      "grad_norm": 6.488926410675049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23080535233020782,
      "logits/rejected": 0.3973449170589447,
      "logps/chosen": -385.880615234375,
      "logps/rejected": -385.3786315917969,
      "loss": 0.7349,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14838168025016785,
      "rewards/margins": 0.23785273730754852,
      "rewards/rejected": -0.38623449206352234,
      "step": 2684
    },
    {
      "epoch": 0.3513535617894823,
      "grad_norm": 6.083072185516357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2694694399833679,
      "logits/rejected": 0.2769680619239807,
      "logps/chosen": -584.282958984375,
      "logps/rejected": -493.7030944824219,
      "loss": 0.3932,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5116247534751892,
      "rewards/margins": 0.9648757576942444,
      "rewards/rejected": -0.45325109362602234,
      "step": 2685
    },
    {
      "epoch": 0.35148441972683403,
      "grad_norm": 12.03162956237793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1394769847393036,
      "logits/rejected": -0.004313036799430847,
      "logps/chosen": -424.7672119140625,
      "logps/rejected": -413.04345703125,
      "loss": 0.641,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15424403548240662,
      "rewards/margins": 1.0495589971542358,
      "rewards/rejected": -1.2038031816482544,
      "step": 2686
    },
    {
      "epoch": 0.3516152776641858,
      "grad_norm": 5.645355224609375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22781231999397278,
      "logits/rejected": -0.20550529658794403,
      "logps/chosen": -329.13372802734375,
      "logps/rejected": -372.6427001953125,
      "loss": 0.4407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07194788753986359,
      "rewards/margins": 0.6760247945785522,
      "rewards/rejected": -0.7479727268218994,
      "step": 2687
    },
    {
      "epoch": 0.35174613560153756,
      "grad_norm": 7.4261474609375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003757145255804062,
      "logits/rejected": -0.015076518058776855,
      "logps/chosen": -355.6297607421875,
      "logps/rejected": -325.42230224609375,
      "loss": 0.7404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14936010539531708,
      "rewards/margins": 0.3808321952819824,
      "rewards/rejected": -0.5301922559738159,
      "step": 2688
    },
    {
      "epoch": 0.35187699353888935,
      "grad_norm": 5.436710357666016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.506597101688385,
      "logits/rejected": 0.5189760327339172,
      "logps/chosen": -311.9907531738281,
      "logps/rejected": -436.80633544921875,
      "loss": 0.404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21645358204841614,
      "rewards/margins": 1.139021873474121,
      "rewards/rejected": -1.3554750680923462,
      "step": 2689
    },
    {
      "epoch": 0.3520078514762411,
      "grad_norm": 7.049386501312256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3324087858200073,
      "logits/rejected": -0.02927321568131447,
      "logps/chosen": -550.7625122070312,
      "logps/rejected": -588.751708984375,
      "loss": 0.3821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6530138254165649,
      "rewards/margins": 1.1189817190170288,
      "rewards/rejected": -1.7719956636428833,
      "step": 2690
    },
    {
      "epoch": 0.3521387094135929,
      "grad_norm": 8.884647369384766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006469523534178734,
      "logits/rejected": 0.2965337634086609,
      "logps/chosen": -323.5191345214844,
      "logps/rejected": -411.0177001953125,
      "loss": 0.8379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22617721557617188,
      "rewards/margins": -0.012292057275772095,
      "rewards/rejected": -0.2138851433992386,
      "step": 2691
    },
    {
      "epoch": 0.3522695673509446,
      "grad_norm": 5.204551696777344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14416703581809998,
      "logits/rejected": 0.10303627699613571,
      "logps/chosen": -358.3992004394531,
      "logps/rejected": -319.3504638671875,
      "loss": 0.3822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1967657208442688,
      "rewards/margins": 1.519296407699585,
      "rewards/rejected": -1.3225305080413818,
      "step": 2692
    },
    {
      "epoch": 0.3524004252882964,
      "grad_norm": 9.940360069274902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18475179374217987,
      "logits/rejected": -0.0360981822013855,
      "logps/chosen": -523.55615234375,
      "logps/rejected": -391.4426574707031,
      "loss": 0.8259,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2677916288375854,
      "rewards/margins": 0.14084967970848083,
      "rewards/rejected": -1.4086412191390991,
      "step": 2693
    },
    {
      "epoch": 0.35253128322564814,
      "grad_norm": 7.5222578048706055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.098152294754982,
      "logits/rejected": 0.01759650558233261,
      "logps/chosen": -311.6871032714844,
      "logps/rejected": -339.4206848144531,
      "loss": 0.5282,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.43931710720062256,
      "rewards/margins": 0.7234464883804321,
      "rewards/rejected": -0.2841292917728424,
      "step": 2694
    },
    {
      "epoch": 0.35266214116299993,
      "grad_norm": 3.8317441940307617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43667614459991455,
      "logits/rejected": 0.2517206370830536,
      "logps/chosen": -636.2850952148438,
      "logps/rejected": -515.9013671875,
      "loss": 0.2692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15890826284885406,
      "rewards/margins": 1.8628509044647217,
      "rewards/rejected": -1.7039426565170288,
      "step": 2695
    },
    {
      "epoch": 0.35279299910035167,
      "grad_norm": 8.37200927734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012066058814525604,
      "logits/rejected": 0.29318010807037354,
      "logps/chosen": -390.576416015625,
      "logps/rejected": -359.67974853515625,
      "loss": 0.6348,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6295021176338196,
      "rewards/margins": 0.32956600189208984,
      "rewards/rejected": -0.9590681195259094,
      "step": 2696
    },
    {
      "epoch": 0.35292385703770346,
      "grad_norm": 6.80910587310791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0996660366654396,
      "logits/rejected": -0.20709681510925293,
      "logps/chosen": -401.9947814941406,
      "logps/rejected": -369.4858703613281,
      "loss": 0.6157,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4933910667896271,
      "rewards/margins": 0.22680187225341797,
      "rewards/rejected": -0.7201929092407227,
      "step": 2697
    },
    {
      "epoch": 0.3530547149750552,
      "grad_norm": 5.450160026550293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13883796334266663,
      "logits/rejected": 0.16673575341701508,
      "logps/chosen": -453.38262939453125,
      "logps/rejected": -397.1293029785156,
      "loss": 0.4272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4877309501171112,
      "rewards/margins": 0.8316198587417603,
      "rewards/rejected": -1.3193508386611938,
      "step": 2698
    },
    {
      "epoch": 0.353185572912407,
      "grad_norm": 7.081991195678711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06328479200601578,
      "logits/rejected": 0.14922836422920227,
      "logps/chosen": -468.38323974609375,
      "logps/rejected": -394.7619323730469,
      "loss": 0.5425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2994661331176758,
      "rewards/margins": 0.7894978523254395,
      "rewards/rejected": -1.0889638662338257,
      "step": 2699
    },
    {
      "epoch": 0.3533164308497587,
      "grad_norm": 6.434457778930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1400778740644455,
      "logits/rejected": -0.04159817099571228,
      "logps/chosen": -454.852294921875,
      "logps/rejected": -382.4910583496094,
      "loss": 0.472,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07289618253707886,
      "rewards/margins": 1.05739426612854,
      "rewards/rejected": -1.1302905082702637,
      "step": 2700
    },
    {
      "epoch": 0.3534472887871105,
      "grad_norm": 3.462728977203369,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047490444034338,
      "logits/rejected": -0.05327797681093216,
      "logps/chosen": -310.1162414550781,
      "logps/rejected": -549.7027587890625,
      "loss": 0.3108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43649202585220337,
      "rewards/margins": 1.4688475131988525,
      "rewards/rejected": -1.0323554277420044,
      "step": 2701
    },
    {
      "epoch": 0.35357814672446225,
      "grad_norm": 5.67935848236084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2831316590309143,
      "logits/rejected": 0.28183943033218384,
      "logps/chosen": -476.32476806640625,
      "logps/rejected": -491.9985046386719,
      "loss": 0.4356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16845932602882385,
      "rewards/margins": 1.1765501499176025,
      "rewards/rejected": -1.008090853691101,
      "step": 2702
    },
    {
      "epoch": 0.35370900466181404,
      "grad_norm": 10.498888969421387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2607921063899994,
      "logits/rejected": 0.2515813410282135,
      "logps/chosen": -579.1837768554688,
      "logps/rejected": -619.05859375,
      "loss": 0.7474,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07858695089817047,
      "rewards/margins": 0.33997899293899536,
      "rewards/rejected": -0.418565958738327,
      "step": 2703
    },
    {
      "epoch": 0.3538398625991658,
      "grad_norm": 5.014289379119873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35711944103240967,
      "logits/rejected": 0.43663957715034485,
      "logps/chosen": -488.25347900390625,
      "logps/rejected": -509.6387634277344,
      "loss": 0.2871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05304640531539917,
      "rewards/margins": 1.5242691040039062,
      "rewards/rejected": -1.5773155689239502,
      "step": 2704
    },
    {
      "epoch": 0.35397072053651757,
      "grad_norm": 7.232754230499268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04139932990074158,
      "logits/rejected": -0.12225425988435745,
      "logps/chosen": -520.7195434570312,
      "logps/rejected": -432.8876037597656,
      "loss": 0.5933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24703390896320343,
      "rewards/margins": 0.5032725930213928,
      "rewards/rejected": -0.7503064870834351,
      "step": 2705
    },
    {
      "epoch": 0.3541015784738693,
      "grad_norm": 5.42924165725708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5197070240974426,
      "logits/rejected": -0.396261602640152,
      "logps/chosen": -372.05853271484375,
      "logps/rejected": -416.9222717285156,
      "loss": 0.4515,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40619945526123047,
      "rewards/margins": 0.6679179072380066,
      "rewards/rejected": -1.0741171836853027,
      "step": 2706
    },
    {
      "epoch": 0.35423243641122104,
      "grad_norm": 7.5417094230651855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3477979898452759,
      "logits/rejected": 0.46729806065559387,
      "logps/chosen": -533.3604736328125,
      "logps/rejected": -544.2340087890625,
      "loss": 0.4577,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3123788833618164,
      "rewards/margins": 1.049914836883545,
      "rewards/rejected": -1.3622937202453613,
      "step": 2707
    },
    {
      "epoch": 0.35436329434857283,
      "grad_norm": 4.0337371826171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31059718132019043,
      "logits/rejected": 0.2563799023628235,
      "logps/chosen": -343.2075500488281,
      "logps/rejected": -283.43878173828125,
      "loss": 0.4041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4057941436767578,
      "rewards/margins": 0.8581246137619019,
      "rewards/rejected": -1.2639187574386597,
      "step": 2708
    },
    {
      "epoch": 0.35449415228592457,
      "grad_norm": 7.463155746459961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08894612640142441,
      "logits/rejected": 0.004278816282749176,
      "logps/chosen": -463.9675598144531,
      "logps/rejected": -471.6847839355469,
      "loss": 0.4971,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7191306948661804,
      "rewards/margins": 0.9002485871315002,
      "rewards/rejected": -1.6193792819976807,
      "step": 2709
    },
    {
      "epoch": 0.35462501022327636,
      "grad_norm": 5.7533087730407715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22784200310707092,
      "logits/rejected": 0.09377565234899521,
      "logps/chosen": -568.4315185546875,
      "logps/rejected": -495.7325134277344,
      "loss": 0.3039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0046516358852386475,
      "rewards/margins": 1.4418290853500366,
      "rewards/rejected": -1.4464807510375977,
      "step": 2710
    },
    {
      "epoch": 0.3547558681606281,
      "grad_norm": 8.89538860321045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09443949162960052,
      "logits/rejected": 0.02852713130414486,
      "logps/chosen": -668.59619140625,
      "logps/rejected": -514.5230102539062,
      "loss": 0.7117,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6523814797401428,
      "rewards/margins": 0.3972581624984741,
      "rewards/rejected": -1.0496398210525513,
      "step": 2711
    },
    {
      "epoch": 0.3548867260979799,
      "grad_norm": 7.30684232711792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17486056685447693,
      "logits/rejected": 0.3070828318595886,
      "logps/chosen": -428.9905700683594,
      "logps/rejected": -510.351806640625,
      "loss": 0.6482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21253623068332672,
      "rewards/margins": 0.5418322086334229,
      "rewards/rejected": -0.7543684244155884,
      "step": 2712
    },
    {
      "epoch": 0.3550175840353316,
      "grad_norm": 4.129023551940918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3792814016342163,
      "logits/rejected": 0.10775415599346161,
      "logps/chosen": -484.0877685546875,
      "logps/rejected": -470.78204345703125,
      "loss": 0.2553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8446931838989258,
      "rewards/margins": 1.853964924812317,
      "rewards/rejected": -1.0092718601226807,
      "step": 2713
    },
    {
      "epoch": 0.3551484419726834,
      "grad_norm": 4.672706127166748,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1855137050151825,
      "logits/rejected": 0.033924400806427,
      "logps/chosen": -345.132568359375,
      "logps/rejected": -427.6917419433594,
      "loss": 0.4495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23075884580612183,
      "rewards/margins": 0.9948965907096863,
      "rewards/rejected": -1.2256553173065186,
      "step": 2714
    },
    {
      "epoch": 0.35527929991003515,
      "grad_norm": 11.773869514465332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08827917277812958,
      "logits/rejected": -0.2961627244949341,
      "logps/chosen": -498.42230224609375,
      "logps/rejected": -565.6944580078125,
      "loss": 0.899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8213775753974915,
      "rewards/margins": -0.17311136424541473,
      "rewards/rejected": -0.6482662558555603,
      "step": 2715
    },
    {
      "epoch": 0.35541015784738694,
      "grad_norm": 5.8323163986206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09612659364938736,
      "logits/rejected": -0.02003305032849312,
      "logps/chosen": -813.213134765625,
      "logps/rejected": -557.322021484375,
      "loss": 0.3684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5269078016281128,
      "rewards/margins": 1.3578331470489502,
      "rewards/rejected": -0.8309253454208374,
      "step": 2716
    },
    {
      "epoch": 0.3555410157847387,
      "grad_norm": 10.896524429321289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1369919776916504,
      "logits/rejected": -0.06029859185218811,
      "logps/chosen": -321.398193359375,
      "logps/rejected": -288.2972106933594,
      "loss": 0.6685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.868398129940033,
      "rewards/margins": 0.4382808804512024,
      "rewards/rejected": -1.3066790103912354,
      "step": 2717
    },
    {
      "epoch": 0.35567187372209047,
      "grad_norm": 8.101724624633789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0831093117594719,
      "logits/rejected": 0.09779459238052368,
      "logps/chosen": -317.7693786621094,
      "logps/rejected": -447.664794921875,
      "loss": 0.5946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5084055066108704,
      "rewards/margins": 0.4512321650981903,
      "rewards/rejected": -0.9596377015113831,
      "step": 2718
    },
    {
      "epoch": 0.3558027316594422,
      "grad_norm": 7.402640342712402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08837289363145828,
      "logits/rejected": -0.03985951840877533,
      "logps/chosen": -349.6485290527344,
      "logps/rejected": -371.54022216796875,
      "loss": 0.5783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3679916560649872,
      "rewards/margins": 0.5459359884262085,
      "rewards/rejected": -0.9139276742935181,
      "step": 2719
    },
    {
      "epoch": 0.355933589596794,
      "grad_norm": 6.100827693939209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0854969471693039,
      "logits/rejected": -0.03689295053482056,
      "logps/chosen": -575.1445922851562,
      "logps/rejected": -441.8395690917969,
      "loss": 0.5452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15614131093025208,
      "rewards/margins": 0.9325077533721924,
      "rewards/rejected": -1.0886489152908325,
      "step": 2720
    },
    {
      "epoch": 0.35606444753414573,
      "grad_norm": 6.1783881187438965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06779146194458008,
      "logits/rejected": 0.1047264039516449,
      "logps/chosen": -308.8037109375,
      "logps/rejected": -303.7574768066406,
      "loss": 0.4384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18939606845378876,
      "rewards/margins": 1.0434761047363281,
      "rewards/rejected": -0.854080080986023,
      "step": 2721
    },
    {
      "epoch": 0.3561953054714975,
      "grad_norm": 9.357611656188965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03408699482679367,
      "logits/rejected": 0.009828239679336548,
      "logps/chosen": -256.0306396484375,
      "logps/rejected": -312.428466796875,
      "loss": 0.8767,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2295433282852173,
      "rewards/margins": -0.2479734718799591,
      "rewards/rejected": -0.9815698862075806,
      "step": 2722
    },
    {
      "epoch": 0.35632616340884926,
      "grad_norm": 6.555875778198242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18602582812309265,
      "logits/rejected": 0.19900819659233093,
      "logps/chosen": -437.53424072265625,
      "logps/rejected": -456.5203552246094,
      "loss": 0.5254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.037952981889247894,
      "rewards/margins": 0.49603942036628723,
      "rewards/rejected": -0.5339924097061157,
      "step": 2723
    },
    {
      "epoch": 0.35645702134620105,
      "grad_norm": 3.135030508041382,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2795054018497467,
      "logits/rejected": 0.13153879344463348,
      "logps/chosen": -626.3333740234375,
      "logps/rejected": -503.6556701660156,
      "loss": 0.2209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38407692313194275,
      "rewards/margins": 1.6815886497497559,
      "rewards/rejected": -1.2975118160247803,
      "step": 2724
    },
    {
      "epoch": 0.3565878792835528,
      "grad_norm": 8.39677619934082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24955874681472778,
      "logits/rejected": 0.10485856235027313,
      "logps/chosen": -572.8707275390625,
      "logps/rejected": -456.41131591796875,
      "loss": 0.6151,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49609678983688354,
      "rewards/margins": 0.3418043255805969,
      "rewards/rejected": -0.8379011154174805,
      "step": 2725
    },
    {
      "epoch": 0.3567187372209046,
      "grad_norm": 6.568187236785889,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33202019333839417,
      "logits/rejected": 0.23866578936576843,
      "logps/chosen": -599.38427734375,
      "logps/rejected": -526.7042846679688,
      "loss": 0.4621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3681178689002991,
      "rewards/margins": 0.9635446071624756,
      "rewards/rejected": -0.5954267978668213,
      "step": 2726
    },
    {
      "epoch": 0.3568495951582563,
      "grad_norm": 11.805747985839844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12440123409032822,
      "logits/rejected": 0.19903728365898132,
      "logps/chosen": -349.27215576171875,
      "logps/rejected": -364.364501953125,
      "loss": 1.031,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2864658534526825,
      "rewards/margins": 0.19184505939483643,
      "rewards/rejected": -0.4783109426498413,
      "step": 2727
    },
    {
      "epoch": 0.3569804530956081,
      "grad_norm": 4.166773319244385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33444294333457947,
      "logits/rejected": 0.19198481738567352,
      "logps/chosen": -356.17669677734375,
      "logps/rejected": -420.3468017578125,
      "loss": 0.2033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2826310992240906,
      "rewards/margins": 1.8564547300338745,
      "rewards/rejected": -1.5738235712051392,
      "step": 2728
    },
    {
      "epoch": 0.35711131103295984,
      "grad_norm": 4.201642990112305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5209622979164124,
      "logits/rejected": 0.30121251940727234,
      "logps/chosen": -328.0385437011719,
      "logps/rejected": -278.2884216308594,
      "loss": 0.3316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006779484450817108,
      "rewards/margins": 1.4103901386260986,
      "rewards/rejected": -1.4171696901321411,
      "step": 2729
    },
    {
      "epoch": 0.3572421689703116,
      "grad_norm": 4.1600165367126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15364858508110046,
      "logits/rejected": -0.23376059532165527,
      "logps/chosen": -391.9306640625,
      "logps/rejected": -342.41705322265625,
      "loss": 0.3603,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4143400192260742,
      "rewards/margins": 1.1278825998306274,
      "rewards/rejected": -0.7135425806045532,
      "step": 2730
    },
    {
      "epoch": 0.35737302690766337,
      "grad_norm": 6.034658908843994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07422609627246857,
      "logits/rejected": -0.19067759811878204,
      "logps/chosen": -501.6921691894531,
      "logps/rejected": -392.45556640625,
      "loss": 0.4367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5255464315414429,
      "rewards/margins": 1.0831388235092163,
      "rewards/rejected": -1.6086853742599487,
      "step": 2731
    },
    {
      "epoch": 0.3575038848450151,
      "grad_norm": 4.828071117401123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08534786105155945,
      "logits/rejected": 0.025991884991526604,
      "logps/chosen": -351.0477294921875,
      "logps/rejected": -486.4917907714844,
      "loss": 0.3105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4796500504016876,
      "rewards/margins": 1.9570143222808838,
      "rewards/rejected": -1.4773643016815186,
      "step": 2732
    },
    {
      "epoch": 0.3576347427823669,
      "grad_norm": 8.0206880569458,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09389814734458923,
      "logits/rejected": 0.04273747280240059,
      "logps/chosen": -459.2450866699219,
      "logps/rejected": -535.8843994140625,
      "loss": 0.7092,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34195077419281006,
      "rewards/margins": 0.21018582582473755,
      "rewards/rejected": -0.5521366000175476,
      "step": 2733
    },
    {
      "epoch": 0.35776560071971863,
      "grad_norm": 4.015640735626221,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09769424051046371,
      "logits/rejected": -0.2390764355659485,
      "logps/chosen": -487.6441650390625,
      "logps/rejected": -491.59930419921875,
      "loss": 0.451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43855994939804077,
      "rewards/margins": 1.2430896759033203,
      "rewards/rejected": -1.6816494464874268,
      "step": 2734
    },
    {
      "epoch": 0.3578964586570704,
      "grad_norm": 5.8953471183776855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00019913353025913239,
      "logits/rejected": -0.17764660716056824,
      "logps/chosen": -485.3468322753906,
      "logps/rejected": -414.8787536621094,
      "loss": 0.5029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04419422149658203,
      "rewards/margins": 0.7215891480445862,
      "rewards/rejected": -0.7657833099365234,
      "step": 2735
    },
    {
      "epoch": 0.35802731659442216,
      "grad_norm": 8.87609577178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05612713471055031,
      "logits/rejected": 0.1141907274723053,
      "logps/chosen": -347.28802490234375,
      "logps/rejected": -379.20452880859375,
      "loss": 0.7918,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6663507223129272,
      "rewards/margins": 0.04520589858293533,
      "rewards/rejected": -0.711556613445282,
      "step": 2736
    },
    {
      "epoch": 0.35815817453177395,
      "grad_norm": 10.284435272216797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37617042660713196,
      "logits/rejected": -0.03865157067775726,
      "logps/chosen": -578.939697265625,
      "logps/rejected": -365.2450866699219,
      "loss": 0.8341,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0470958948135376,
      "rewards/margins": 0.10300540924072266,
      "rewards/rejected": -1.1501013040542603,
      "step": 2737
    },
    {
      "epoch": 0.3582890324691257,
      "grad_norm": 6.732175827026367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19131548702716827,
      "logits/rejected": -0.13379967212677002,
      "logps/chosen": -388.9807434082031,
      "logps/rejected": -454.29736328125,
      "loss": 0.5205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5004133582115173,
      "rewards/margins": 0.5592832565307617,
      "rewards/rejected": -1.0596965551376343,
      "step": 2738
    },
    {
      "epoch": 0.3584198904064775,
      "grad_norm": 10.268183708190918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029473837465047836,
      "logits/rejected": 0.1360110342502594,
      "logps/chosen": -441.8511047363281,
      "logps/rejected": -397.44342041015625,
      "loss": 1.0076,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4810352325439453,
      "rewards/margins": -0.15019148588180542,
      "rewards/rejected": -0.3308437466621399,
      "step": 2739
    },
    {
      "epoch": 0.3585507483438292,
      "grad_norm": 7.428144931793213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17691081762313843,
      "logits/rejected": -0.055841781198978424,
      "logps/chosen": -611.6878662109375,
      "logps/rejected": -555.0696411132812,
      "loss": 0.3725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8332288265228271,
      "rewards/margins": 1.196678638458252,
      "rewards/rejected": -2.029907464981079,
      "step": 2740
    },
    {
      "epoch": 0.358681606281181,
      "grad_norm": 7.333220481872559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1921781599521637,
      "logits/rejected": 0.06003360077738762,
      "logps/chosen": -497.365234375,
      "logps/rejected": -363.0223083496094,
      "loss": 0.4202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30466002225875854,
      "rewards/margins": 1.0553338527679443,
      "rewards/rejected": -1.3599939346313477,
      "step": 2741
    },
    {
      "epoch": 0.35881246421853275,
      "grad_norm": 4.605767726898193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08531650900840759,
      "logits/rejected": -0.06587706506252289,
      "logps/chosen": -430.1304626464844,
      "logps/rejected": -327.5960693359375,
      "loss": 0.418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18968944251537323,
      "rewards/margins": 1.1927101612091064,
      "rewards/rejected": -1.0030207633972168,
      "step": 2742
    },
    {
      "epoch": 0.35894332215588454,
      "grad_norm": 6.880471229553223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19676250219345093,
      "logits/rejected": -0.07641235738992691,
      "logps/chosen": -635.4413452148438,
      "logps/rejected": -586.3739013671875,
      "loss": 0.4271,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3912489116191864,
      "rewards/margins": 1.020049810409546,
      "rewards/rejected": -0.6288009881973267,
      "step": 2743
    },
    {
      "epoch": 0.3590741800932363,
      "grad_norm": 8.915969848632812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1835768222808838,
      "logits/rejected": 0.11411396414041519,
      "logps/chosen": -468.80999755859375,
      "logps/rejected": -428.00494384765625,
      "loss": 0.7566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.36814671754837036,
      "rewards/margins": 0.3580942153930664,
      "rewards/rejected": -0.726240873336792,
      "step": 2744
    },
    {
      "epoch": 0.35920503803058806,
      "grad_norm": 9.127798080444336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13298818469047546,
      "logits/rejected": 0.10465942323207855,
      "logps/chosen": -307.0078125,
      "logps/rejected": -437.5999450683594,
      "loss": 0.804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0799318552017212,
      "rewards/margins": 0.3148549795150757,
      "rewards/rejected": -1.3947869539260864,
      "step": 2745
    },
    {
      "epoch": 0.3593358959679398,
      "grad_norm": 5.579865455627441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002477778121829033,
      "logits/rejected": 0.0016700704582035542,
      "logps/chosen": -467.3926086425781,
      "logps/rejected": -388.0185546875,
      "loss": 0.5532,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9299761056900024,
      "rewards/margins": 0.5536041259765625,
      "rewards/rejected": -1.4835801124572754,
      "step": 2746
    },
    {
      "epoch": 0.3594667539052916,
      "grad_norm": 8.414642333984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11457857489585876,
      "logits/rejected": 0.02917683869600296,
      "logps/chosen": -420.76727294921875,
      "logps/rejected": -468.4477844238281,
      "loss": 0.6384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07536572962999344,
      "rewards/margins": 0.6620625257492065,
      "rewards/rejected": -0.7374283075332642,
      "step": 2747
    },
    {
      "epoch": 0.3595976118426433,
      "grad_norm": 6.270965576171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3393304646015167,
      "logits/rejected": 0.20594926178455353,
      "logps/chosen": -499.23492431640625,
      "logps/rejected": -431.4731140136719,
      "loss": 0.4097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2951973080635071,
      "rewards/margins": 1.3823940753936768,
      "rewards/rejected": -1.087196707725525,
      "step": 2748
    },
    {
      "epoch": 0.3597284697799951,
      "grad_norm": 10.00745677947998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.054488517343997955,
      "logits/rejected": 0.151784747838974,
      "logps/chosen": -445.56024169921875,
      "logps/rejected": -465.33740234375,
      "loss": 0.6925,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6070791482925415,
      "rewards/margins": 0.1523301899433136,
      "rewards/rejected": -0.7594093680381775,
      "step": 2749
    },
    {
      "epoch": 0.35985932771734686,
      "grad_norm": 7.208578109741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4385139048099518,
      "logits/rejected": 0.45252713561058044,
      "logps/chosen": -361.82427978515625,
      "logps/rejected": -434.2233581542969,
      "loss": 0.572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8902071118354797,
      "rewards/margins": 0.39560604095458984,
      "rewards/rejected": -1.2858130931854248,
      "step": 2750
    },
    {
      "epoch": 0.3599901856546986,
      "grad_norm": 6.463598251342773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3362201452255249,
      "logits/rejected": 0.3612745404243469,
      "logps/chosen": -571.1495361328125,
      "logps/rejected": -447.21990966796875,
      "loss": 0.4519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12947046756744385,
      "rewards/margins": 0.8889108896255493,
      "rewards/rejected": -1.0183813571929932,
      "step": 2751
    },
    {
      "epoch": 0.3601210435920504,
      "grad_norm": 4.545187950134277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29081055521965027,
      "logits/rejected": 0.21545732021331787,
      "logps/chosen": -649.38671875,
      "logps/rejected": -466.45831298828125,
      "loss": 0.3255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.527840793132782,
      "rewards/margins": 1.3679627180099487,
      "rewards/rejected": -0.840121865272522,
      "step": 2752
    },
    {
      "epoch": 0.3602519015294021,
      "grad_norm": 7.36676549911499,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24747785925865173,
      "logits/rejected": 0.17861631512641907,
      "logps/chosen": -762.6597900390625,
      "logps/rejected": -755.1806030273438,
      "loss": 0.5689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.226542666554451,
      "rewards/margins": 0.5235311985015869,
      "rewards/rejected": -0.7500737905502319,
      "step": 2753
    },
    {
      "epoch": 0.3603827594667539,
      "grad_norm": 6.3461761474609375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20115213096141815,
      "logits/rejected": 0.05317385867238045,
      "logps/chosen": -543.3143310546875,
      "logps/rejected": -581.1146240234375,
      "loss": 0.3328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30584049224853516,
      "rewards/margins": 1.5607547760009766,
      "rewards/rejected": -1.8665953874588013,
      "step": 2754
    },
    {
      "epoch": 0.36051361740410565,
      "grad_norm": 6.1148271560668945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.059171222150325775,
      "logits/rejected": 0.07899870723485947,
      "logps/chosen": -466.4810485839844,
      "logps/rejected": -362.1054992675781,
      "loss": 0.4382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7445825338363647,
      "rewards/margins": 0.9536678194999695,
      "rewards/rejected": -1.698250412940979,
      "step": 2755
    },
    {
      "epoch": 0.36064447534145744,
      "grad_norm": 8.238714218139648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06895656138658524,
      "logits/rejected": -0.026485420763492584,
      "logps/chosen": -293.5260925292969,
      "logps/rejected": -450.1634521484375,
      "loss": 0.6747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5741904973983765,
      "rewards/margins": 0.3695327639579773,
      "rewards/rejected": -0.9437233209609985,
      "step": 2756
    },
    {
      "epoch": 0.3607753332788092,
      "grad_norm": 9.935691833496094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02296096459031105,
      "logits/rejected": 0.043601542711257935,
      "logps/chosen": -534.1787109375,
      "logps/rejected": -433.69293212890625,
      "loss": 0.9233,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47317177057266235,
      "rewards/margins": 0.19491565227508545,
      "rewards/rejected": -0.6680874228477478,
      "step": 2757
    },
    {
      "epoch": 0.36090619121616097,
      "grad_norm": 5.934568405151367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23713141679763794,
      "logits/rejected": 0.10082349181175232,
      "logps/chosen": -415.7178955078125,
      "logps/rejected": -391.2603759765625,
      "loss": 0.388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5073534846305847,
      "rewards/margins": 0.9404268264770508,
      "rewards/rejected": -1.4477802515029907,
      "step": 2758
    },
    {
      "epoch": 0.3610370491535127,
      "grad_norm": 5.638318061828613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3442676067352295,
      "logits/rejected": 0.035606078803539276,
      "logps/chosen": -308.4080505371094,
      "logps/rejected": -368.97369384765625,
      "loss": 0.4676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37529662251472473,
      "rewards/margins": 0.8758377432823181,
      "rewards/rejected": -1.2511342763900757,
      "step": 2759
    },
    {
      "epoch": 0.3611679070908645,
      "grad_norm": 8.56982421875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27081814408302307,
      "logits/rejected": 0.11551883816719055,
      "logps/chosen": -573.2689819335938,
      "logps/rejected": -363.7658386230469,
      "loss": 0.6766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13657283782958984,
      "rewards/margins": 0.903118371963501,
      "rewards/rejected": -0.7665454745292664,
      "step": 2760
    },
    {
      "epoch": 0.36129876502821623,
      "grad_norm": 8.184473991394043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11306578665971756,
      "logits/rejected": -0.05173508822917938,
      "logps/chosen": -365.0396728515625,
      "logps/rejected": -498.7091064453125,
      "loss": 0.6594,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8362937569618225,
      "rewards/margins": 0.3790704309940338,
      "rewards/rejected": -1.2153643369674683,
      "step": 2761
    },
    {
      "epoch": 0.361429622965568,
      "grad_norm": 4.849736213684082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3123788833618164,
      "logits/rejected": -0.16758258640766144,
      "logps/chosen": -454.4842224121094,
      "logps/rejected": -521.2139282226562,
      "loss": 0.2952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3962804973125458,
      "rewards/margins": 1.9717488288879395,
      "rewards/rejected": -2.3680293560028076,
      "step": 2762
    },
    {
      "epoch": 0.36156048090291976,
      "grad_norm": 13.593147277832031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06273290514945984,
      "logits/rejected": 0.008985638618469238,
      "logps/chosen": -482.0741882324219,
      "logps/rejected": -409.18536376953125,
      "loss": 0.9624,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0763189792633057,
      "rewards/margins": 0.045690082013607025,
      "rewards/rejected": -1.122009038925171,
      "step": 2763
    },
    {
      "epoch": 0.36169133884027155,
      "grad_norm": 7.930326461791992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24522970616817474,
      "logits/rejected": 0.05673407390713692,
      "logps/chosen": -610.4720458984375,
      "logps/rejected": -571.7576293945312,
      "loss": 0.6056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.74810791015625,
      "rewards/margins": 0.901148796081543,
      "rewards/rejected": -1.649256706237793,
      "step": 2764
    },
    {
      "epoch": 0.3618221967776233,
      "grad_norm": 10.260356903076172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3441179096698761,
      "logits/rejected": 0.32316887378692627,
      "logps/chosen": -531.1129150390625,
      "logps/rejected": -556.60888671875,
      "loss": 0.8202,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.838291585445404,
      "rewards/margins": 0.05476437509059906,
      "rewards/rejected": -0.8930559158325195,
      "step": 2765
    },
    {
      "epoch": 0.3619530547149751,
      "grad_norm": 6.880308151245117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18702805042266846,
      "logits/rejected": 0.06427639722824097,
      "logps/chosen": -568.9207153320312,
      "logps/rejected": -555.0843505859375,
      "loss": 0.6116,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.142715573310852,
      "rewards/margins": 0.40715527534484863,
      "rewards/rejected": -1.5498707294464111,
      "step": 2766
    },
    {
      "epoch": 0.3620839126523268,
      "grad_norm": 6.770012855529785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01791352406144142,
      "logits/rejected": 0.19407479465007782,
      "logps/chosen": -347.2608642578125,
      "logps/rejected": -357.76702880859375,
      "loss": 0.7143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6715795397758484,
      "rewards/margins": 0.28059276938438416,
      "rewards/rejected": -0.9521722793579102,
      "step": 2767
    },
    {
      "epoch": 0.3622147705896786,
      "grad_norm": 7.687463283538818,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16162100434303284,
      "logits/rejected": 0.232902392745018,
      "logps/chosen": -526.7493896484375,
      "logps/rejected": -544.6519165039062,
      "loss": 0.456,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5456001162528992,
      "rewards/margins": 0.7261077761650085,
      "rewards/rejected": -1.2717078924179077,
      "step": 2768
    },
    {
      "epoch": 0.36234562852703034,
      "grad_norm": 5.769848823547363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0588650219142437,
      "logits/rejected": 0.10045996308326721,
      "logps/chosen": -633.333251953125,
      "logps/rejected": -467.2552795410156,
      "loss": 0.4899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5403400659561157,
      "rewards/margins": 0.8852169513702393,
      "rewards/rejected": -1.425557017326355,
      "step": 2769
    },
    {
      "epoch": 0.36247648646438213,
      "grad_norm": 5.2879462242126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06688106805086136,
      "logits/rejected": -0.204318568110466,
      "logps/chosen": -387.45306396484375,
      "logps/rejected": -407.51806640625,
      "loss": 0.3807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7937483191490173,
      "rewards/margins": 1.1991664171218872,
      "rewards/rejected": -1.9929147958755493,
      "step": 2770
    },
    {
      "epoch": 0.36260734440173387,
      "grad_norm": 5.984252452850342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15046003460884094,
      "logits/rejected": 0.11384816467761993,
      "logps/chosen": -322.6633605957031,
      "logps/rejected": -315.2691955566406,
      "loss": 0.4924,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08653393387794495,
      "rewards/margins": 0.803269624710083,
      "rewards/rejected": -0.8898035287857056,
      "step": 2771
    },
    {
      "epoch": 0.3627382023390856,
      "grad_norm": 5.870091915130615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16427333652973175,
      "logits/rejected": 0.008726358413696289,
      "logps/chosen": -249.47190856933594,
      "logps/rejected": -271.5380859375,
      "loss": 0.4538,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42159825563430786,
      "rewards/margins": 0.6589614152908325,
      "rewards/rejected": -1.0805597305297852,
      "step": 2772
    },
    {
      "epoch": 0.3628690602764374,
      "grad_norm": 5.296298980712891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08938217908143997,
      "logits/rejected": 0.15759840607643127,
      "logps/chosen": -542.5704956054688,
      "logps/rejected": -500.36749267578125,
      "loss": 0.4267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05983620136976242,
      "rewards/margins": 0.887620747089386,
      "rewards/rejected": -0.947456955909729,
      "step": 2773
    },
    {
      "epoch": 0.36299991821378913,
      "grad_norm": 7.548501014709473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1600099354982376,
      "logits/rejected": 0.11224357038736343,
      "logps/chosen": -394.09649658203125,
      "logps/rejected": -379.133056640625,
      "loss": 0.729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37142616510391235,
      "rewards/margins": 0.5136336088180542,
      "rewards/rejected": -0.8850597739219666,
      "step": 2774
    },
    {
      "epoch": 0.3631307761511409,
      "grad_norm": 5.25741720199585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43210089206695557,
      "logits/rejected": 0.06570196896791458,
      "logps/chosen": -595.2789916992188,
      "logps/rejected": -438.0195007324219,
      "loss": 0.3789,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07129764556884766,
      "rewards/margins": 1.3350492715835571,
      "rewards/rejected": -1.2637516260147095,
      "step": 2775
    },
    {
      "epoch": 0.36326163408849266,
      "grad_norm": 10.900171279907227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0772908478975296,
      "logits/rejected": 0.2342577874660492,
      "logps/chosen": -416.3059997558594,
      "logps/rejected": -407.9209899902344,
      "loss": 0.7625,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1185569763183594,
      "rewards/margins": 0.22662487626075745,
      "rewards/rejected": -1.3451818227767944,
      "step": 2776
    },
    {
      "epoch": 0.36339249202584445,
      "grad_norm": 7.929818153381348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0852779895067215,
      "logits/rejected": -0.0014002807438373566,
      "logps/chosen": -391.4981994628906,
      "logps/rejected": -363.8001403808594,
      "loss": 0.9545,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6964296102523804,
      "rewards/margins": -0.26709750294685364,
      "rewards/rejected": -0.4293321669101715,
      "step": 2777
    },
    {
      "epoch": 0.3635233499631962,
      "grad_norm": 6.596334457397461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05236540362238884,
      "logits/rejected": 0.09519258886575699,
      "logps/chosen": -695.5198974609375,
      "logps/rejected": -492.3504638671875,
      "loss": 0.3474,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11547717452049255,
      "rewards/margins": 1.6218239068984985,
      "rewards/rejected": -1.5063467025756836,
      "step": 2778
    },
    {
      "epoch": 0.363654207900548,
      "grad_norm": 3.921861410140991,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05246512591838837,
      "logits/rejected": -0.05588000267744064,
      "logps/chosen": -377.06915283203125,
      "logps/rejected": -307.61322021484375,
      "loss": 0.3072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0733167752623558,
      "rewards/margins": 1.4349180459976196,
      "rewards/rejected": -1.3616012334823608,
      "step": 2779
    },
    {
      "epoch": 0.3637850658378997,
      "grad_norm": 5.215448379516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12407553941011429,
      "logits/rejected": -0.09041950106620789,
      "logps/chosen": -403.6422119140625,
      "logps/rejected": -461.8515930175781,
      "loss": 0.2626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26140403747558594,
      "rewards/margins": 1.7897412776947021,
      "rewards/rejected": -2.051145315170288,
      "step": 2780
    },
    {
      "epoch": 0.3639159237752515,
      "grad_norm": 6.936619758605957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1976502537727356,
      "logits/rejected": -0.09098650515079498,
      "logps/chosen": -546.28955078125,
      "logps/rejected": -697.9541015625,
      "loss": 0.4725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6137415170669556,
      "rewards/margins": 1.0585447549819946,
      "rewards/rejected": -1.6722862720489502,
      "step": 2781
    },
    {
      "epoch": 0.36404678171260324,
      "grad_norm": 6.221074104309082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2453719675540924,
      "logits/rejected": 0.31209543347358704,
      "logps/chosen": -529.0194702148438,
      "logps/rejected": -578.22998046875,
      "loss": 0.4935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4854278564453125,
      "rewards/margins": 0.8863978981971741,
      "rewards/rejected": -0.40097007155418396,
      "step": 2782
    },
    {
      "epoch": 0.36417763964995503,
      "grad_norm": 8.00535774230957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10132952034473419,
      "logits/rejected": 0.12404489517211914,
      "logps/chosen": -330.0249938964844,
      "logps/rejected": -518.0921630859375,
      "loss": 0.7469,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6947839856147766,
      "rewards/margins": 0.1992795467376709,
      "rewards/rejected": -0.8940635919570923,
      "step": 2783
    },
    {
      "epoch": 0.36430849758730677,
      "grad_norm": 7.221765041351318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45506176352500916,
      "logits/rejected": 0.45456022024154663,
      "logps/chosen": -433.0401611328125,
      "logps/rejected": -407.2375793457031,
      "loss": 0.5873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5739477276802063,
      "rewards/margins": 0.6675943732261658,
      "rewards/rejected": -1.241542100906372,
      "step": 2784
    },
    {
      "epoch": 0.36443935552465856,
      "grad_norm": 11.77253532409668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05931416153907776,
      "logits/rejected": -0.06301681697368622,
      "logps/chosen": -403.1095275878906,
      "logps/rejected": -358.5743408203125,
      "loss": 1.1939,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.386161208152771,
      "rewards/margins": -0.04311022162437439,
      "rewards/rejected": -1.3430509567260742,
      "step": 2785
    },
    {
      "epoch": 0.3645702134620103,
      "grad_norm": 6.052740573883057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.186672180891037,
      "logits/rejected": 0.36329537630081177,
      "logps/chosen": -596.8616333007812,
      "logps/rejected": -457.14923095703125,
      "loss": 0.4833,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09872326254844666,
      "rewards/margins": 0.6572959423065186,
      "rewards/rejected": -0.7560192346572876,
      "step": 2786
    },
    {
      "epoch": 0.3647010713993621,
      "grad_norm": 8.169303894042969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2124849259853363,
      "logits/rejected": 0.17776256799697876,
      "logps/chosen": -526.2889404296875,
      "logps/rejected": -549.0598754882812,
      "loss": 0.6139,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7091125249862671,
      "rewards/margins": 0.4880622327327728,
      "rewards/rejected": -1.1971747875213623,
      "step": 2787
    },
    {
      "epoch": 0.3648319293367138,
      "grad_norm": 4.80629301071167,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10820567607879639,
      "logits/rejected": -0.022824304178357124,
      "logps/chosen": -292.1623229980469,
      "logps/rejected": -309.671142578125,
      "loss": 0.5123,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45449182391166687,
      "rewards/margins": 0.6362228989601135,
      "rewards/rejected": -1.0907148122787476,
      "step": 2788
    },
    {
      "epoch": 0.3649627872740656,
      "grad_norm": 5.3676018714904785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12682726979255676,
      "logits/rejected": -0.03908171132206917,
      "logps/chosen": -543.9927978515625,
      "logps/rejected": -363.86236572265625,
      "loss": 0.2991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08147545903921127,
      "rewards/margins": 1.262892723083496,
      "rewards/rejected": -1.1814172267913818,
      "step": 2789
    },
    {
      "epoch": 0.36509364521141735,
      "grad_norm": 4.263092041015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08999337255954742,
      "logits/rejected": 0.2567237615585327,
      "logps/chosen": -382.72845458984375,
      "logps/rejected": -546.99365234375,
      "loss": 0.2942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7047275304794312,
      "rewards/margins": 1.5215699672698975,
      "rewards/rejected": -2.226297616958618,
      "step": 2790
    },
    {
      "epoch": 0.36522450314876914,
      "grad_norm": 6.930509567260742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20645296573638916,
      "logits/rejected": 0.359977662563324,
      "logps/chosen": -409.33453369140625,
      "logps/rejected": -462.3792724609375,
      "loss": 0.5969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12442150712013245,
      "rewards/margins": 0.4342138171195984,
      "rewards/rejected": -0.5586353540420532,
      "step": 2791
    },
    {
      "epoch": 0.3653553610861209,
      "grad_norm": 4.509669780731201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1555452048778534,
      "logits/rejected": -0.11666052043437958,
      "logps/chosen": -556.9442138671875,
      "logps/rejected": -595.1947631835938,
      "loss": 0.3293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3046327829360962,
      "rewards/margins": 1.5617828369140625,
      "rewards/rejected": -1.8664155006408691,
      "step": 2792
    },
    {
      "epoch": 0.36548621902347267,
      "grad_norm": 7.806862831115723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28462255001068115,
      "logits/rejected": 0.07995791733264923,
      "logps/chosen": -447.9703674316406,
      "logps/rejected": -467.0739440917969,
      "loss": 0.5377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19899749755859375,
      "rewards/margins": 0.42013970017433167,
      "rewards/rejected": -0.6191372275352478,
      "step": 2793
    },
    {
      "epoch": 0.3656170769608244,
      "grad_norm": 9.419739723205566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08691241592168808,
      "logits/rejected": 0.0031088506802916527,
      "logps/chosen": -601.4563598632812,
      "logps/rejected": -512.3939208984375,
      "loss": 0.7804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1339054107666016,
      "rewards/margins": 0.16448955237865448,
      "rewards/rejected": -1.298395037651062,
      "step": 2794
    },
    {
      "epoch": 0.36574793489817614,
      "grad_norm": 7.690603733062744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04908343404531479,
      "logits/rejected": 0.007559348829090595,
      "logps/chosen": -357.7052307128906,
      "logps/rejected": -386.2284851074219,
      "loss": 0.7064,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6284074783325195,
      "rewards/margins": 0.20721013844013214,
      "rewards/rejected": -0.8356176614761353,
      "step": 2795
    },
    {
      "epoch": 0.36587879283552793,
      "grad_norm": 9.554032325744629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0055231451988220215,
      "logits/rejected": 0.10046891123056412,
      "logps/chosen": -370.5068359375,
      "logps/rejected": -268.87408447265625,
      "loss": 0.5707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46592772006988525,
      "rewards/margins": 0.45667773485183716,
      "rewards/rejected": -0.9226054549217224,
      "step": 2796
    },
    {
      "epoch": 0.36600965077287967,
      "grad_norm": 7.6647844314575195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3714674711227417,
      "logits/rejected": 0.3225608468055725,
      "logps/chosen": -321.7481994628906,
      "logps/rejected": -328.55615234375,
      "loss": 0.6826,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0071080923080444,
      "rewards/margins": 0.2559865415096283,
      "rewards/rejected": -1.2630945444107056,
      "step": 2797
    },
    {
      "epoch": 0.36614050871023146,
      "grad_norm": 7.360607147216797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3879617750644684,
      "logits/rejected": 0.18573245406150818,
      "logps/chosen": -590.60107421875,
      "logps/rejected": -501.72259521484375,
      "loss": 0.5944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17051389813423157,
      "rewards/margins": 0.6249417066574097,
      "rewards/rejected": -0.45442789793014526,
      "step": 2798
    },
    {
      "epoch": 0.3662713666475832,
      "grad_norm": 4.027377128601074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0932949110865593,
      "logits/rejected": 0.019296536222100258,
      "logps/chosen": -387.4836730957031,
      "logps/rejected": -340.36712646484375,
      "loss": 0.3711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31217578053474426,
      "rewards/margins": 1.0811946392059326,
      "rewards/rejected": -1.3933703899383545,
      "step": 2799
    },
    {
      "epoch": 0.366402224584935,
      "grad_norm": 6.960686206817627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.041133128106594086,
      "logits/rejected": 0.08220366388559341,
      "logps/chosen": -545.214599609375,
      "logps/rejected": -498.5950927734375,
      "loss": 0.5029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6563416123390198,
      "rewards/margins": 0.49127310514450073,
      "rewards/rejected": -1.1476147174835205,
      "step": 2800
    },
    {
      "epoch": 0.3665330825222867,
      "grad_norm": 6.625180244445801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046407513320446014,
      "logits/rejected": 0.12816108763217926,
      "logps/chosen": -428.63165283203125,
      "logps/rejected": -432.24090576171875,
      "loss": 0.3352,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7726240158081055,
      "rewards/margins": 1.237504482269287,
      "rewards/rejected": -2.0101284980773926,
      "step": 2801
    },
    {
      "epoch": 0.3666639404596385,
      "grad_norm": 7.2252302169799805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12589988112449646,
      "logits/rejected": 0.3050035834312439,
      "logps/chosen": -455.42352294921875,
      "logps/rejected": -465.11767578125,
      "loss": 0.4746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3127439618110657,
      "rewards/margins": 1.423595905303955,
      "rewards/rejected": -1.736340045928955,
      "step": 2802
    },
    {
      "epoch": 0.36679479839699025,
      "grad_norm": 9.71711254119873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12049482762813568,
      "logits/rejected": 0.06493023782968521,
      "logps/chosen": -481.705078125,
      "logps/rejected": -356.2939758300781,
      "loss": 0.6206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6983877420425415,
      "rewards/margins": 0.31802958250045776,
      "rewards/rejected": -1.0164172649383545,
      "step": 2803
    },
    {
      "epoch": 0.36692565633434204,
      "grad_norm": 4.879370212554932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04184231907129288,
      "logits/rejected": 0.08157380670309067,
      "logps/chosen": -339.26287841796875,
      "logps/rejected": -313.30224609375,
      "loss": 0.4769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30000191926956177,
      "rewards/margins": 0.8734194040298462,
      "rewards/rejected": -1.1734213829040527,
      "step": 2804
    },
    {
      "epoch": 0.3670565142716938,
      "grad_norm": 9.350865364074707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14452162384986877,
      "logits/rejected": 0.18812435865402222,
      "logps/chosen": -430.2849426269531,
      "logps/rejected": -418.47552490234375,
      "loss": 0.8331,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8884884119033813,
      "rewards/margins": -0.11700595170259476,
      "rewards/rejected": -0.7714824676513672,
      "step": 2805
    },
    {
      "epoch": 0.36718737220904557,
      "grad_norm": 6.946412086486816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021255813539028168,
      "logits/rejected": 0.06957218050956726,
      "logps/chosen": -546.498046875,
      "logps/rejected": -432.049072265625,
      "loss": 0.4955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2891901135444641,
      "rewards/margins": 1.0473109483718872,
      "rewards/rejected": -0.7581207156181335,
      "step": 2806
    },
    {
      "epoch": 0.3673182301463973,
      "grad_norm": 4.423072338104248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2687194049358368,
      "logits/rejected": 0.030779030174016953,
      "logps/chosen": -530.8607177734375,
      "logps/rejected": -464.576904296875,
      "loss": 0.3006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05183238908648491,
      "rewards/margins": 1.2650285959243774,
      "rewards/rejected": -1.3168610334396362,
      "step": 2807
    },
    {
      "epoch": 0.3674490880837491,
      "grad_norm": 5.429612159729004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03710970655083656,
      "logits/rejected": 0.08628678321838379,
      "logps/chosen": -503.5862121582031,
      "logps/rejected": -682.9641723632812,
      "loss": 0.3706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10738854110240936,
      "rewards/margins": 1.350014328956604,
      "rewards/rejected": -1.4574027061462402,
      "step": 2808
    },
    {
      "epoch": 0.36757994602110083,
      "grad_norm": 7.578009605407715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2654806971549988,
      "logits/rejected": 0.3503606617450714,
      "logps/chosen": -684.0241088867188,
      "logps/rejected": -654.58203125,
      "loss": 0.567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5160263180732727,
      "rewards/margins": 0.6867616772651672,
      "rewards/rejected": -1.20278799533844,
      "step": 2809
    },
    {
      "epoch": 0.3677108039584526,
      "grad_norm": 6.232666015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1455957442522049,
      "logits/rejected": -0.22443696856498718,
      "logps/chosen": -245.51869201660156,
      "logps/rejected": -302.07110595703125,
      "loss": 0.465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23295046389102936,
      "rewards/margins": 1.334571123123169,
      "rewards/rejected": -1.567521572113037,
      "step": 2810
    },
    {
      "epoch": 0.36784166189580436,
      "grad_norm": 4.007007598876953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1800675243139267,
      "logits/rejected": 0.16576862335205078,
      "logps/chosen": -415.59356689453125,
      "logps/rejected": -407.5498962402344,
      "loss": 0.3639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03566001355648041,
      "rewards/margins": 1.4468994140625,
      "rewards/rejected": -1.4112393856048584,
      "step": 2811
    },
    {
      "epoch": 0.36797251983315615,
      "grad_norm": 9.89025592803955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3221951723098755,
      "logits/rejected": 0.16149833798408508,
      "logps/chosen": -491.1096496582031,
      "logps/rejected": -502.19146728515625,
      "loss": 0.6077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6727631092071533,
      "rewards/margins": 0.5181535482406616,
      "rewards/rejected": -1.190916657447815,
      "step": 2812
    },
    {
      "epoch": 0.3681033777705079,
      "grad_norm": 7.186360836029053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11841890960931778,
      "logits/rejected": -0.0348089337348938,
      "logps/chosen": -493.6733703613281,
      "logps/rejected": -363.16400146484375,
      "loss": 0.7161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.592451274394989,
      "rewards/margins": 0.1352190375328064,
      "rewards/rejected": -0.7276703119277954,
      "step": 2813
    },
    {
      "epoch": 0.3682342357078597,
      "grad_norm": 6.9647393226623535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2697056829929352,
      "logits/rejected": -0.12139634042978287,
      "logps/chosen": -518.2811279296875,
      "logps/rejected": -467.3178405761719,
      "loss": 0.5336,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7983192205429077,
      "rewards/margins": 0.7945051193237305,
      "rewards/rejected": -1.5928243398666382,
      "step": 2814
    },
    {
      "epoch": 0.3683650936452114,
      "grad_norm": 7.716316223144531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026022017002105713,
      "logits/rejected": 0.02896716259419918,
      "logps/chosen": -249.09469604492188,
      "logps/rejected": -298.34515380859375,
      "loss": 0.635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34069252014160156,
      "rewards/margins": 0.3999733328819275,
      "rewards/rejected": -0.7406657934188843,
      "step": 2815
    },
    {
      "epoch": 0.36849595158256315,
      "grad_norm": 5.43265962600708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31723111867904663,
      "logits/rejected": -0.5885579586029053,
      "logps/chosen": -627.3888549804688,
      "logps/rejected": -418.2122497558594,
      "loss": 0.5004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7781805396080017,
      "rewards/margins": 0.5890058875083923,
      "rewards/rejected": -1.3671863079071045,
      "step": 2816
    },
    {
      "epoch": 0.36862680951991494,
      "grad_norm": 4.718057155609131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10220373421907425,
      "logits/rejected": -0.1469222754240036,
      "logps/chosen": -569.7093505859375,
      "logps/rejected": -503.5281982421875,
      "loss": 0.3392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5080950856208801,
      "rewards/margins": 1.180635690689087,
      "rewards/rejected": -1.6887308359146118,
      "step": 2817
    },
    {
      "epoch": 0.3687576674572667,
      "grad_norm": 4.657660007476807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10318311303853989,
      "logits/rejected": -0.24681706726551056,
      "logps/chosen": -349.74505615234375,
      "logps/rejected": -473.61212158203125,
      "loss": 0.4659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4003814458847046,
      "rewards/margins": 0.758527398109436,
      "rewards/rejected": -1.1589088439941406,
      "step": 2818
    },
    {
      "epoch": 0.36888852539461847,
      "grad_norm": 9.105548858642578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08805989474058151,
      "logits/rejected": 0.016204696148633957,
      "logps/chosen": -472.3027038574219,
      "logps/rejected": -526.8472290039062,
      "loss": 0.6886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5955597162246704,
      "rewards/margins": 0.778252124786377,
      "rewards/rejected": -1.3738117218017578,
      "step": 2819
    },
    {
      "epoch": 0.3690193833319702,
      "grad_norm": 6.057389259338379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16085170209407806,
      "logits/rejected": 0.2888953685760498,
      "logps/chosen": -465.3146057128906,
      "logps/rejected": -372.061767578125,
      "loss": 0.4165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5469803810119629,
      "rewards/margins": 0.8163527250289917,
      "rewards/rejected": -1.3633331060409546,
      "step": 2820
    },
    {
      "epoch": 0.369150241269322,
      "grad_norm": 5.901223182678223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24525706470012665,
      "logits/rejected": 0.2415349781513214,
      "logps/chosen": -455.41400146484375,
      "logps/rejected": -423.4019775390625,
      "loss": 0.4906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6843838691711426,
      "rewards/margins": 0.7912365198135376,
      "rewards/rejected": -0.10685272514820099,
      "step": 2821
    },
    {
      "epoch": 0.36928109920667374,
      "grad_norm": 8.321514129638672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15156714618206024,
      "logits/rejected": 0.286064088344574,
      "logps/chosen": -518.1234130859375,
      "logps/rejected": -510.106201171875,
      "loss": 0.5621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6514735817909241,
      "rewards/margins": 0.7667854428291321,
      "rewards/rejected": -1.4182590246200562,
      "step": 2822
    },
    {
      "epoch": 0.3694119571440255,
      "grad_norm": 7.752533435821533,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05574001744389534,
      "logits/rejected": -0.04919564723968506,
      "logps/chosen": -679.8377685546875,
      "logps/rejected": -423.151123046875,
      "loss": 0.6485,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24888435006141663,
      "rewards/margins": 0.6211826801300049,
      "rewards/rejected": -0.8700670599937439,
      "step": 2823
    },
    {
      "epoch": 0.36954281508137726,
      "grad_norm": 5.231320858001709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13839420676231384,
      "logits/rejected": -0.20623676478862762,
      "logps/chosen": -507.87506103515625,
      "logps/rejected": -306.9224548339844,
      "loss": 0.5397,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7330480813980103,
      "rewards/margins": 0.5321211218833923,
      "rewards/rejected": -1.2651691436767578,
      "step": 2824
    },
    {
      "epoch": 0.36967367301872905,
      "grad_norm": 10.888860702514648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2247294932603836,
      "logits/rejected": -0.04227958619594574,
      "logps/chosen": -412.2753601074219,
      "logps/rejected": -422.7315673828125,
      "loss": 0.7637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9790005683898926,
      "rewards/margins": 0.27567917108535767,
      "rewards/rejected": -1.2546796798706055,
      "step": 2825
    },
    {
      "epoch": 0.3698045309560808,
      "grad_norm": 5.906376361846924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10822758823633194,
      "logits/rejected": 0.15701407194137573,
      "logps/chosen": -361.2039794921875,
      "logps/rejected": -337.323486328125,
      "loss": 0.4843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7984338998794556,
      "rewards/margins": 0.6912074685096741,
      "rewards/rejected": -1.4896413087844849,
      "step": 2826
    },
    {
      "epoch": 0.3699353888934326,
      "grad_norm": 4.618931293487549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027220025658607483,
      "logits/rejected": 0.024250373244285583,
      "logps/chosen": -372.72015380859375,
      "logps/rejected": -424.394775390625,
      "loss": 0.2384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4960029721260071,
      "rewards/margins": 1.6015352010726929,
      "rewards/rejected": -2.0975382328033447,
      "step": 2827
    },
    {
      "epoch": 0.3700662468307843,
      "grad_norm": 6.806783199310303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15814264118671417,
      "logits/rejected": 0.18277044594287872,
      "logps/chosen": -435.5346984863281,
      "logps/rejected": -398.327392578125,
      "loss": 0.5079,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.262145459651947,
      "rewards/margins": 0.8554811477661133,
      "rewards/rejected": -1.1176265478134155,
      "step": 2828
    },
    {
      "epoch": 0.3701971047681361,
      "grad_norm": 5.7694315910339355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5923458933830261,
      "logits/rejected": 0.5305476188659668,
      "logps/chosen": -458.7413635253906,
      "logps/rejected": -415.70050048828125,
      "loss": 0.4151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6343289017677307,
      "rewards/margins": 1.0344184637069702,
      "rewards/rejected": -0.40008965134620667,
      "step": 2829
    },
    {
      "epoch": 0.37032796270548785,
      "grad_norm": 4.893787860870361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19781619310379028,
      "logits/rejected": -0.05318821594119072,
      "logps/chosen": -440.7790222167969,
      "logps/rejected": -495.183837890625,
      "loss": 0.2827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7060449719429016,
      "rewards/margins": 1.5609571933746338,
      "rewards/rejected": -2.2670021057128906,
      "step": 2830
    },
    {
      "epoch": 0.37045882064283964,
      "grad_norm": 4.3072662353515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1387847661972046,
      "logits/rejected": 0.28410041332244873,
      "logps/chosen": -726.6044311523438,
      "logps/rejected": -556.8263549804688,
      "loss": 0.2178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.009393304586410522,
      "rewards/margins": 1.679075002670288,
      "rewards/rejected": -1.6884684562683105,
      "step": 2831
    },
    {
      "epoch": 0.3705896785801914,
      "grad_norm": 4.888592720031738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.055704399943351746,
      "logits/rejected": -0.3284776210784912,
      "logps/chosen": -443.4804992675781,
      "logps/rejected": -546.4381713867188,
      "loss": 0.3258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32075750827789307,
      "rewards/margins": 1.2984193563461304,
      "rewards/rejected": -1.6191767454147339,
      "step": 2832
    },
    {
      "epoch": 0.37072053651754316,
      "grad_norm": 4.377003192901611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05148378387093544,
      "logits/rejected": 0.030198007822036743,
      "logps/chosen": -544.860107421875,
      "logps/rejected": -470.4773254394531,
      "loss": 0.3539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7932488322257996,
      "rewards/margins": 1.2294654846191406,
      "rewards/rejected": -0.4362167716026306,
      "step": 2833
    },
    {
      "epoch": 0.3708513944548949,
      "grad_norm": 3.8630142211914062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4079601764678955,
      "logits/rejected": -0.5709341764450073,
      "logps/chosen": -370.258544921875,
      "logps/rejected": -457.5086669921875,
      "loss": 0.2751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05905665457248688,
      "rewards/margins": 1.5090949535369873,
      "rewards/rejected": -1.568151593208313,
      "step": 2834
    },
    {
      "epoch": 0.3709822523922467,
      "grad_norm": 6.54358434677124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3840124011039734,
      "logits/rejected": 0.01090635173022747,
      "logps/chosen": -401.2416076660156,
      "logps/rejected": -313.0800476074219,
      "loss": 0.5453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39894962310791016,
      "rewards/margins": 1.1234140396118164,
      "rewards/rejected": -1.5223636627197266,
      "step": 2835
    },
    {
      "epoch": 0.37111311032959843,
      "grad_norm": 9.108981132507324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03683297708630562,
      "logits/rejected": -0.011901358142495155,
      "logps/chosen": -491.95550537109375,
      "logps/rejected": -501.556640625,
      "loss": 0.9741,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7812383770942688,
      "rewards/margins": 0.21974492073059082,
      "rewards/rejected": -1.0009832382202148,
      "step": 2836
    },
    {
      "epoch": 0.3712439682669502,
      "grad_norm": 10.397104263305664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09684589505195618,
      "logits/rejected": 0.10226129740476608,
      "logps/chosen": -551.5985107421875,
      "logps/rejected": -338.7903747558594,
      "loss": 0.4494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13888761401176453,
      "rewards/margins": 1.6989752054214478,
      "rewards/rejected": -1.8378628492355347,
      "step": 2837
    },
    {
      "epoch": 0.37137482620430196,
      "grad_norm": 7.7346649169921875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3547331690788269,
      "logits/rejected": 0.21005140244960785,
      "logps/chosen": -528.0811157226562,
      "logps/rejected": -479.893310546875,
      "loss": 0.6522,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37938231229782104,
      "rewards/margins": 0.2774650752544403,
      "rewards/rejected": -0.6568474173545837,
      "step": 2838
    },
    {
      "epoch": 0.3715056841416537,
      "grad_norm": 6.253483295440674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10292291641235352,
      "logits/rejected": -0.32764965295791626,
      "logps/chosen": -599.6160888671875,
      "logps/rejected": -462.0500793457031,
      "loss": 0.4127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12395554780960083,
      "rewards/margins": 1.280821681022644,
      "rewards/rejected": -1.4047774076461792,
      "step": 2839
    },
    {
      "epoch": 0.3716365420790055,
      "grad_norm": 6.396252632141113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18847551941871643,
      "logits/rejected": -0.07068206369876862,
      "logps/chosen": -744.3529052734375,
      "logps/rejected": -674.8712158203125,
      "loss": 0.4004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.39929869771003723,
      "rewards/margins": 1.4620819091796875,
      "rewards/rejected": -1.0627832412719727,
      "step": 2840
    },
    {
      "epoch": 0.3717674000163572,
      "grad_norm": 5.7438812255859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14097817242145538,
      "logits/rejected": 0.1502118706703186,
      "logps/chosen": -440.6957702636719,
      "logps/rejected": -336.5052490234375,
      "loss": 0.355,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13602885603904724,
      "rewards/margins": 1.7566969394683838,
      "rewards/rejected": -1.6206681728363037,
      "step": 2841
    },
    {
      "epoch": 0.371898257953709,
      "grad_norm": 7.947694778442383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0130616445094347,
      "logits/rejected": 0.14502224326133728,
      "logps/chosen": -476.1595153808594,
      "logps/rejected": -391.7720947265625,
      "loss": 0.492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2965275049209595,
      "rewards/margins": 0.980955958366394,
      "rewards/rejected": -1.2774834632873535,
      "step": 2842
    },
    {
      "epoch": 0.37202911589106075,
      "grad_norm": 6.598227500915527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12986397743225098,
      "logits/rejected": -0.2388172745704651,
      "logps/chosen": -539.6934814453125,
      "logps/rejected": -371.83782958984375,
      "loss": 0.4081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43273431062698364,
      "rewards/margins": 1.2815827131271362,
      "rewards/rejected": -1.7143170833587646,
      "step": 2843
    },
    {
      "epoch": 0.37215997382841254,
      "grad_norm": 6.417436122894287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07510866224765778,
      "logits/rejected": 0.31137678027153015,
      "logps/chosen": -514.329345703125,
      "logps/rejected": -511.9766845703125,
      "loss": 0.4697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43440815806388855,
      "rewards/margins": 1.3287417888641357,
      "rewards/rejected": -0.8943336009979248,
      "step": 2844
    },
    {
      "epoch": 0.3722908317657643,
      "grad_norm": 8.435535430908203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15078583359718323,
      "logits/rejected": -0.12193290144205093,
      "logps/chosen": -487.8502197265625,
      "logps/rejected": -620.9282836914062,
      "loss": 0.6947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8384555578231812,
      "rewards/margins": 0.4280359745025635,
      "rewards/rejected": -1.2664916515350342,
      "step": 2845
    },
    {
      "epoch": 0.37242168970311607,
      "grad_norm": 7.806707859039307,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15113931894302368,
      "logits/rejected": -0.0578526109457016,
      "logps/chosen": -336.6904602050781,
      "logps/rejected": -586.2992553710938,
      "loss": 0.6073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4583193063735962,
      "rewards/margins": 0.32114851474761963,
      "rewards/rejected": -1.7794677019119263,
      "step": 2846
    },
    {
      "epoch": 0.3725525476404678,
      "grad_norm": 7.635231971740723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051829516887664795,
      "logits/rejected": -0.050182051956653595,
      "logps/chosen": -441.7646484375,
      "logps/rejected": -434.0840148925781,
      "loss": 0.5464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25677239894866943,
      "rewards/margins": 1.4214649200439453,
      "rewards/rejected": -1.6782373189926147,
      "step": 2847
    },
    {
      "epoch": 0.3726834055778196,
      "grad_norm": 7.949067115783691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16078931093215942,
      "logits/rejected": 0.2783742845058441,
      "logps/chosen": -637.6695556640625,
      "logps/rejected": -515.163330078125,
      "loss": 0.5875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0411682054400444,
      "rewards/margins": 0.5322855114936829,
      "rewards/rejected": -0.5734537243843079,
      "step": 2848
    },
    {
      "epoch": 0.37281426351517133,
      "grad_norm": 5.1010894775390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009934991598129272,
      "logits/rejected": 0.07324986904859543,
      "logps/chosen": -356.93865966796875,
      "logps/rejected": -446.5198974609375,
      "loss": 0.3175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.549975574016571,
      "rewards/margins": 1.3297210931777954,
      "rewards/rejected": -1.8796966075897217,
      "step": 2849
    },
    {
      "epoch": 0.3729451214525231,
      "grad_norm": 6.562838077545166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10607245564460754,
      "logits/rejected": -0.20014406740665436,
      "logps/chosen": -530.192138671875,
      "logps/rejected": -448.815185546875,
      "loss": 0.4947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3978479504585266,
      "rewards/margins": 0.6069305539131165,
      "rewards/rejected": -1.004778504371643,
      "step": 2850
    },
    {
      "epoch": 0.37307597938987486,
      "grad_norm": 6.792250633239746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15186141431331635,
      "logits/rejected": 0.08590463548898697,
      "logps/chosen": -269.82708740234375,
      "logps/rejected": -412.1189880371094,
      "loss": 0.5688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.82799232006073,
      "rewards/margins": 0.7842803597450256,
      "rewards/rejected": -1.6122725009918213,
      "step": 2851
    },
    {
      "epoch": 0.37320683732722665,
      "grad_norm": 7.099241733551025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0549737811088562,
      "logits/rejected": 0.05106398090720177,
      "logps/chosen": -374.7717590332031,
      "logps/rejected": -311.67059326171875,
      "loss": 0.5788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8022398352622986,
      "rewards/margins": 0.6346828937530518,
      "rewards/rejected": -1.436922550201416,
      "step": 2852
    },
    {
      "epoch": 0.3733376952645784,
      "grad_norm": 8.300169944763184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06741825491189957,
      "logits/rejected": 0.053185973316431046,
      "logps/chosen": -340.60552978515625,
      "logps/rejected": -276.10736083984375,
      "loss": 0.7542,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9661710262298584,
      "rewards/margins": 0.1799013763666153,
      "rewards/rejected": -1.1460723876953125,
      "step": 2853
    },
    {
      "epoch": 0.3734685532019302,
      "grad_norm": 7.595611095428467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09015022218227386,
      "logits/rejected": 0.16042566299438477,
      "logps/chosen": -528.7994384765625,
      "logps/rejected": -546.0444946289062,
      "loss": 0.568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6181526780128479,
      "rewards/margins": 0.7529522776603699,
      "rewards/rejected": -1.3711049556732178,
      "step": 2854
    },
    {
      "epoch": 0.3735994111392819,
      "grad_norm": 8.890103340148926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10599145293235779,
      "logits/rejected": -0.01204981841146946,
      "logps/chosen": -560.0189208984375,
      "logps/rejected": -534.1996459960938,
      "loss": 0.7725,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5058879256248474,
      "rewards/margins": 0.04507102072238922,
      "rewards/rejected": -0.5509588718414307,
      "step": 2855
    },
    {
      "epoch": 0.3737302690766337,
      "grad_norm": 5.900684356689453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02437753789126873,
      "logits/rejected": -0.09947390109300613,
      "logps/chosen": -497.3362121582031,
      "logps/rejected": -451.54827880859375,
      "loss": 0.4364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2626488208770752,
      "rewards/margins": 1.2334693670272827,
      "rewards/rejected": -0.9708206653594971,
      "step": 2856
    },
    {
      "epoch": 0.37386112701398544,
      "grad_norm": 7.816668510437012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030124159529805183,
      "logits/rejected": 0.03896946460008621,
      "logps/chosen": -342.1370544433594,
      "logps/rejected": -455.55596923828125,
      "loss": 0.5893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34129011631011963,
      "rewards/margins": 0.7611578106880188,
      "rewards/rejected": -1.102447748184204,
      "step": 2857
    },
    {
      "epoch": 0.37399198495133723,
      "grad_norm": 6.723361015319824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1957397162914276,
      "logits/rejected": 0.1932809054851532,
      "logps/chosen": -537.3280029296875,
      "logps/rejected": -511.7489013671875,
      "loss": 0.4613,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6729170083999634,
      "rewards/margins": 0.690432608127594,
      "rewards/rejected": -1.3633495569229126,
      "step": 2858
    },
    {
      "epoch": 0.37412284288868897,
      "grad_norm": 4.931149482727051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37561285495758057,
      "logits/rejected": 0.3525428771972656,
      "logps/chosen": -345.8197937011719,
      "logps/rejected": -370.4402770996094,
      "loss": 0.3218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10390491783618927,
      "rewards/margins": 1.464855432510376,
      "rewards/rejected": -1.5687603950500488,
      "step": 2859
    },
    {
      "epoch": 0.3742537008260407,
      "grad_norm": 8.784085273742676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32832300662994385,
      "logits/rejected": 0.07414398342370987,
      "logps/chosen": -397.4836730957031,
      "logps/rejected": -407.78741455078125,
      "loss": 0.874,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.549966812133789,
      "rewards/margins": -0.018218427896499634,
      "rewards/rejected": -1.5317484140396118,
      "step": 2860
    },
    {
      "epoch": 0.3743845587633925,
      "grad_norm": 6.035005569458008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030313357710838318,
      "logits/rejected": -0.05979149043560028,
      "logps/chosen": -503.163818359375,
      "logps/rejected": -467.6134033203125,
      "loss": 0.4583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12143231183290482,
      "rewards/margins": 0.8012130856513977,
      "rewards/rejected": -0.6797807216644287,
      "step": 2861
    },
    {
      "epoch": 0.37451541670074423,
      "grad_norm": 4.87947416305542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26952484250068665,
      "logits/rejected": 0.11862379312515259,
      "logps/chosen": -523.8446044921875,
      "logps/rejected": -460.1820983886719,
      "loss": 0.4277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.332492470741272,
      "rewards/margins": 1.8625950813293457,
      "rewards/rejected": -2.195087432861328,
      "step": 2862
    },
    {
      "epoch": 0.374646274638096,
      "grad_norm": 6.567107200622559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06728324294090271,
      "logits/rejected": 0.19874155521392822,
      "logps/chosen": -367.6244812011719,
      "logps/rejected": -375.2845458984375,
      "loss": 0.5561,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5691208243370056,
      "rewards/margins": 0.5171657800674438,
      "rewards/rejected": -1.0862865447998047,
      "step": 2863
    },
    {
      "epoch": 0.37477713257544776,
      "grad_norm": 6.152017593383789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17373031377792358,
      "logits/rejected": -0.045565150678157806,
      "logps/chosen": -473.31134033203125,
      "logps/rejected": -647.1530151367188,
      "loss": 0.3766,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07267531752586365,
      "rewards/margins": 1.1619230508804321,
      "rewards/rejected": -1.2345983982086182,
      "step": 2864
    },
    {
      "epoch": 0.37490799051279955,
      "grad_norm": 6.295802116394043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07236519455909729,
      "logits/rejected": -0.016650963574647903,
      "logps/chosen": -445.8671875,
      "logps/rejected": -344.12945556640625,
      "loss": 0.5112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.193668931722641,
      "rewards/margins": 0.5656871795654297,
      "rewards/rejected": -0.3720182180404663,
      "step": 2865
    },
    {
      "epoch": 0.3750388484501513,
      "grad_norm": 9.997142791748047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23967865109443665,
      "logits/rejected": 0.07642267644405365,
      "logps/chosen": -647.587646484375,
      "logps/rejected": -479.593505859375,
      "loss": 0.6737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.046552807092666626,
      "rewards/margins": 0.43994948267936707,
      "rewards/rejected": -0.39339664578437805,
      "step": 2866
    },
    {
      "epoch": 0.3751697063875031,
      "grad_norm": 4.8115458488464355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12334368377923965,
      "logits/rejected": 0.06339919567108154,
      "logps/chosen": -370.9555969238281,
      "logps/rejected": -305.1003723144531,
      "loss": 0.4281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1812047958374023,
      "rewards/margins": 0.8603629469871521,
      "rewards/rejected": -2.041567802429199,
      "step": 2867
    },
    {
      "epoch": 0.3753005643248548,
      "grad_norm": 6.4723591804504395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2371094673871994,
      "logits/rejected": 0.0875588208436966,
      "logps/chosen": -364.7614440917969,
      "logps/rejected": -316.4531555175781,
      "loss": 0.5165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34665223956108093,
      "rewards/margins": 0.7771782279014587,
      "rewards/rejected": -1.1238304376602173,
      "step": 2868
    },
    {
      "epoch": 0.3754314222622066,
      "grad_norm": 5.462518692016602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27347081899642944,
      "logits/rejected": 0.34549808502197266,
      "logps/chosen": -491.761474609375,
      "logps/rejected": -465.01617431640625,
      "loss": 0.3475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34483984112739563,
      "rewards/margins": 1.0335853099822998,
      "rewards/rejected": -0.6887454986572266,
      "step": 2869
    },
    {
      "epoch": 0.37556228019955834,
      "grad_norm": 7.140872955322266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14809927344322205,
      "logits/rejected": -0.19502629339694977,
      "logps/chosen": -619.8724365234375,
      "logps/rejected": -519.003662109375,
      "loss": 0.5806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19245031476020813,
      "rewards/margins": 0.35844576358795166,
      "rewards/rejected": -0.5508961081504822,
      "step": 2870
    },
    {
      "epoch": 0.37569313813691013,
      "grad_norm": 6.1232829093933105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12754134833812714,
      "logits/rejected": -0.09201063215732574,
      "logps/chosen": -394.9102783203125,
      "logps/rejected": -410.0122375488281,
      "loss": 0.4659,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05327397957444191,
      "rewards/margins": 0.9694254398345947,
      "rewards/rejected": -0.9161514639854431,
      "step": 2871
    },
    {
      "epoch": 0.37582399607426187,
      "grad_norm": 5.861234188079834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12188165634870529,
      "logits/rejected": -0.20791636407375336,
      "logps/chosen": -303.35247802734375,
      "logps/rejected": -420.0021057128906,
      "loss": 0.4015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5735341310501099,
      "rewards/margins": 1.3604674339294434,
      "rewards/rejected": -1.9340015649795532,
      "step": 2872
    },
    {
      "epoch": 0.37595485401161366,
      "grad_norm": 5.133844375610352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23246335983276367,
      "logits/rejected": 0.17564061284065247,
      "logps/chosen": -511.6780700683594,
      "logps/rejected": -547.0960083007812,
      "loss": 0.3397,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06568917632102966,
      "rewards/margins": 1.2315322160720825,
      "rewards/rejected": -1.2972214221954346,
      "step": 2873
    },
    {
      "epoch": 0.3760857119489654,
      "grad_norm": 5.981614112854004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1299182027578354,
      "logits/rejected": 0.18385888636112213,
      "logps/chosen": -516.1021728515625,
      "logps/rejected": -515.8076171875,
      "loss": 0.5667,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2723926603794098,
      "rewards/margins": 0.42964115738868713,
      "rewards/rejected": -0.15724849700927734,
      "step": 2874
    },
    {
      "epoch": 0.3762165698863172,
      "grad_norm": 8.632181167602539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4799325466156006,
      "logits/rejected": 0.25334179401397705,
      "logps/chosen": -486.4272766113281,
      "logps/rejected": -644.400146484375,
      "loss": 0.6031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3197881877422333,
      "rewards/margins": 0.5284717679023743,
      "rewards/rejected": -0.8482599854469299,
      "step": 2875
    },
    {
      "epoch": 0.3763474278236689,
      "grad_norm": 7.490997791290283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09385380148887634,
      "logits/rejected": 0.22343133389949799,
      "logps/chosen": -315.4398193359375,
      "logps/rejected": -359.98223876953125,
      "loss": 0.5413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8314454555511475,
      "rewards/margins": 0.9688161611557007,
      "rewards/rejected": -1.8002618551254272,
      "step": 2876
    },
    {
      "epoch": 0.3764782857610207,
      "grad_norm": 10.566007614135742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17903205752372742,
      "logits/rejected": -0.07984189689159393,
      "logps/chosen": -572.6820678710938,
      "logps/rejected": -418.83966064453125,
      "loss": 0.6523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0912179946899414,
      "rewards/margins": 0.5200706124305725,
      "rewards/rejected": -1.6112886667251587,
      "step": 2877
    },
    {
      "epoch": 0.37660914369837245,
      "grad_norm": 4.780886650085449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04515467584133148,
      "logits/rejected": -0.10969239473342896,
      "logps/chosen": -695.760009765625,
      "logps/rejected": -458.58349609375,
      "loss": 0.4173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11959612369537354,
      "rewards/margins": 0.8887302279472351,
      "rewards/rejected": -1.0083262920379639,
      "step": 2878
    },
    {
      "epoch": 0.37674000163572424,
      "grad_norm": 6.220503330230713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04074053838849068,
      "logits/rejected": -0.00437651202082634,
      "logps/chosen": -355.6978759765625,
      "logps/rejected": -356.399658203125,
      "loss": 0.5188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08824120461940765,
      "rewards/margins": 0.5441291928291321,
      "rewards/rejected": -0.45588797330856323,
      "step": 2879
    },
    {
      "epoch": 0.376870859573076,
      "grad_norm": 8.441527366638184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26299014687538147,
      "logits/rejected": 0.21492312848567963,
      "logps/chosen": -559.9921875,
      "logps/rejected": -458.4113464355469,
      "loss": 0.5841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3208602964878082,
      "rewards/margins": 0.674504280090332,
      "rewards/rejected": -0.9953645467758179,
      "step": 2880
    },
    {
      "epoch": 0.37700171751042777,
      "grad_norm": 7.0255022048950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0796283632516861,
      "logits/rejected": 0.046749912202358246,
      "logps/chosen": -489.17645263671875,
      "logps/rejected": -476.3546142578125,
      "loss": 0.5087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07084237784147263,
      "rewards/margins": 0.9481010437011719,
      "rewards/rejected": -0.8772586584091187,
      "step": 2881
    },
    {
      "epoch": 0.3771325754477795,
      "grad_norm": 7.363198280334473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21494562923908234,
      "logits/rejected": 0.17579150199890137,
      "logps/chosen": -428.4854431152344,
      "logps/rejected": -434.11102294921875,
      "loss": 0.6993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47350823879241943,
      "rewards/margins": 0.8848112225532532,
      "rewards/rejected": -1.3583195209503174,
      "step": 2882
    },
    {
      "epoch": 0.37726343338513124,
      "grad_norm": 5.044201374053955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4439886212348938,
      "logits/rejected": 0.41657689213752747,
      "logps/chosen": -418.7019348144531,
      "logps/rejected": -291.6707458496094,
      "loss": 0.3983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42344245314598083,
      "rewards/margins": 0.9299304485321045,
      "rewards/rejected": -1.3533728122711182,
      "step": 2883
    },
    {
      "epoch": 0.37739429132248303,
      "grad_norm": 8.397954940795898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2580263614654541,
      "logits/rejected": -0.2070661187171936,
      "logps/chosen": -367.54010009765625,
      "logps/rejected": -243.89291381835938,
      "loss": 0.7809,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.36702194809913635,
      "rewards/margins": 0.0846935510635376,
      "rewards/rejected": -0.45171546936035156,
      "step": 2884
    },
    {
      "epoch": 0.37752514925983477,
      "grad_norm": 9.38841724395752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10658101737499237,
      "logits/rejected": -0.14175602793693542,
      "logps/chosen": -512.51904296875,
      "logps/rejected": -468.9411315917969,
      "loss": 0.6238,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08012282848358154,
      "rewards/margins": 0.7064410448074341,
      "rewards/rejected": -0.7865638732910156,
      "step": 2885
    },
    {
      "epoch": 0.37765600719718656,
      "grad_norm": 6.1583476066589355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047635745257139206,
      "logits/rejected": -0.1735660284757614,
      "logps/chosen": -263.4488525390625,
      "logps/rejected": -406.1928405761719,
      "loss": 0.5501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9232379198074341,
      "rewards/margins": 0.7285960912704468,
      "rewards/rejected": -1.6518341302871704,
      "step": 2886
    },
    {
      "epoch": 0.3777868651345383,
      "grad_norm": 7.007874488830566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2051624059677124,
      "logits/rejected": 0.3483826518058777,
      "logps/chosen": -450.9605712890625,
      "logps/rejected": -455.289794921875,
      "loss": 0.5446,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19478075206279755,
      "rewards/margins": 0.9602113366127014,
      "rewards/rejected": -1.1549921035766602,
      "step": 2887
    },
    {
      "epoch": 0.3779177230718901,
      "grad_norm": 4.022132396697998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3016871213912964,
      "logits/rejected": 0.19062002003192902,
      "logps/chosen": -664.5096435546875,
      "logps/rejected": -392.46990966796875,
      "loss": 0.2726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2971729636192322,
      "rewards/margins": 1.6159663200378418,
      "rewards/rejected": -1.3187932968139648,
      "step": 2888
    },
    {
      "epoch": 0.3780485810092418,
      "grad_norm": 8.068770408630371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23445652425289154,
      "logits/rejected": 0.11178787052631378,
      "logps/chosen": -485.85040283203125,
      "logps/rejected": -687.5116577148438,
      "loss": 0.5088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12018128484487534,
      "rewards/margins": 1.1473958492279053,
      "rewards/rejected": -1.0272146463394165,
      "step": 2889
    },
    {
      "epoch": 0.3781794389465936,
      "grad_norm": 9.28727912902832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19439031183719635,
      "logits/rejected": 0.3298262357711792,
      "logps/chosen": -425.968994140625,
      "logps/rejected": -371.78118896484375,
      "loss": 0.5111,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1484876424074173,
      "rewards/margins": 0.745496392250061,
      "rewards/rejected": -0.8939841389656067,
      "step": 2890
    },
    {
      "epoch": 0.37831029688394535,
      "grad_norm": 8.626760482788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10036050528287888,
      "logits/rejected": -0.03420349210500717,
      "logps/chosen": -274.5323181152344,
      "logps/rejected": -317.76995849609375,
      "loss": 0.8719,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0726749897003174,
      "rewards/margins": 0.16866682469844818,
      "rewards/rejected": -1.2413417100906372,
      "step": 2891
    },
    {
      "epoch": 0.37844115482129714,
      "grad_norm": 6.805417060852051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20116040110588074,
      "logits/rejected": 0.18711423873901367,
      "logps/chosen": -493.33306884765625,
      "logps/rejected": -542.9876708984375,
      "loss": 0.4687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.01718691736459732,
      "rewards/margins": 1.5101518630981445,
      "rewards/rejected": -1.4929649829864502,
      "step": 2892
    },
    {
      "epoch": 0.3785720127586489,
      "grad_norm": 3.914541482925415,
      "learning_rate": 5e-06,
      "logits/chosen": 0.255296915769577,
      "logits/rejected": -0.19374831020832062,
      "logps/chosen": -423.12847900390625,
      "logps/rejected": -361.6654357910156,
      "loss": 0.2358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3036220669746399,
      "rewards/margins": 2.0124363899230957,
      "rewards/rejected": -2.316058397293091,
      "step": 2893
    },
    {
      "epoch": 0.37870287069600067,
      "grad_norm": 6.237781524658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2028530389070511,
      "logits/rejected": -0.002243969589471817,
      "logps/chosen": -501.8275146484375,
      "logps/rejected": -537.96484375,
      "loss": 0.2746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44613116979599,
      "rewards/margins": 1.6152287721633911,
      "rewards/rejected": -2.0613598823547363,
      "step": 2894
    },
    {
      "epoch": 0.3788337286333524,
      "grad_norm": 7.87632942199707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0934324562549591,
      "logits/rejected": -0.05363383889198303,
      "logps/chosen": -481.1853332519531,
      "logps/rejected": -517.64111328125,
      "loss": 0.5855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5579080581665039,
      "rewards/margins": 1.437997579574585,
      "rewards/rejected": -1.9959057569503784,
      "step": 2895
    },
    {
      "epoch": 0.3789645865707042,
      "grad_norm": 12.999794960021973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20036581158638,
      "logits/rejected": 0.2385556548833847,
      "logps/chosen": -386.32293701171875,
      "logps/rejected": -316.78521728515625,
      "loss": 0.8803,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9440807700157166,
      "rewards/margins": 0.2193080484867096,
      "rewards/rejected": -1.1633888483047485,
      "step": 2896
    },
    {
      "epoch": 0.37909544450805593,
      "grad_norm": 6.874507904052734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053137827664613724,
      "logits/rejected": 0.10807054489850998,
      "logps/chosen": -453.556884765625,
      "logps/rejected": -478.16815185546875,
      "loss": 0.564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2542131543159485,
      "rewards/margins": 0.77135169506073,
      "rewards/rejected": -1.0255647897720337,
      "step": 2897
    },
    {
      "epoch": 0.3792263024454077,
      "grad_norm": 7.992913722991943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20653747022151947,
      "logits/rejected": -0.17001058161258698,
      "logps/chosen": -384.1521911621094,
      "logps/rejected": -380.76348876953125,
      "loss": 0.6194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41299745440483093,
      "rewards/margins": 0.6488761305809021,
      "rewards/rejected": -1.0618735551834106,
      "step": 2898
    },
    {
      "epoch": 0.37935716038275946,
      "grad_norm": 4.8391594886779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15261876583099365,
      "logits/rejected": 0.12585270404815674,
      "logps/chosen": -735.3142700195312,
      "logps/rejected": -571.899169921875,
      "loss": 0.3076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02526262402534485,
      "rewards/margins": 1.2692161798477173,
      "rewards/rejected": -1.2439535856246948,
      "step": 2899
    },
    {
      "epoch": 0.37948801832011125,
      "grad_norm": 5.76834774017334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1306241750717163,
      "logits/rejected": -0.3732107877731323,
      "logps/chosen": -395.86627197265625,
      "logps/rejected": -306.3056945800781,
      "loss": 0.4519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3417080044746399,
      "rewards/margins": 0.7221195697784424,
      "rewards/rejected": -1.0638275146484375,
      "step": 2900
    },
    {
      "epoch": 0.379618876257463,
      "grad_norm": 9.280370712280273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.253698468208313,
      "logits/rejected": 0.21767346560955048,
      "logps/chosen": -463.8096923828125,
      "logps/rejected": -560.258056640625,
      "loss": 0.7866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0095913410186768,
      "rewards/margins": 0.36383360624313354,
      "rewards/rejected": -1.373424768447876,
      "step": 2901
    },
    {
      "epoch": 0.3797497341948148,
      "grad_norm": 4.585239410400391,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09310224652290344,
      "logits/rejected": 0.08482711017131805,
      "logps/chosen": -664.2037353515625,
      "logps/rejected": -542.7097778320312,
      "loss": 0.3057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02636907994747162,
      "rewards/margins": 1.6039319038391113,
      "rewards/rejected": -1.577562689781189,
      "step": 2902
    },
    {
      "epoch": 0.3798805921321665,
      "grad_norm": 5.6864800453186035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3199158310890198,
      "logits/rejected": 0.16710320115089417,
      "logps/chosen": -622.677490234375,
      "logps/rejected": -551.7659912109375,
      "loss": 0.319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35403600335121155,
      "rewards/margins": 1.7916103601455688,
      "rewards/rejected": -1.4375743865966797,
      "step": 2903
    },
    {
      "epoch": 0.38001145006951825,
      "grad_norm": 8.077466011047363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031833402812480927,
      "logits/rejected": 0.1580182909965515,
      "logps/chosen": -459.9924011230469,
      "logps/rejected": -438.4200744628906,
      "loss": 0.5805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6258456707000732,
      "rewards/margins": 0.40773487091064453,
      "rewards/rejected": -1.0335806608200073,
      "step": 2904
    },
    {
      "epoch": 0.38014230800687004,
      "grad_norm": 5.0666046142578125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20504158735275269,
      "logits/rejected": 0.17922550439834595,
      "logps/chosen": -480.9868469238281,
      "logps/rejected": -429.02606201171875,
      "loss": 0.3822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13344764709472656,
      "rewards/margins": 1.3522658348083496,
      "rewards/rejected": -1.4857136011123657,
      "step": 2905
    },
    {
      "epoch": 0.3802731659442218,
      "grad_norm": 5.414409637451172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24651944637298584,
      "logits/rejected": -0.1155933365225792,
      "logps/chosen": -400.43743896484375,
      "logps/rejected": -365.471923828125,
      "loss": 0.4162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23985141515731812,
      "rewards/margins": 0.8054881691932678,
      "rewards/rejected": -0.5656368732452393,
      "step": 2906
    },
    {
      "epoch": 0.38040402388157357,
      "grad_norm": 6.878169536590576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13924925029277802,
      "logits/rejected": 0.08676125109195709,
      "logps/chosen": -505.77972412109375,
      "logps/rejected": -387.34490966796875,
      "loss": 0.4243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5883259773254395,
      "rewards/margins": 1.7249914407730103,
      "rewards/rejected": -2.3133177757263184,
      "step": 2907
    },
    {
      "epoch": 0.3805348818189253,
      "grad_norm": 7.517840385437012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33186018466949463,
      "logits/rejected": -0.11615046858787537,
      "logps/chosen": -592.8051147460938,
      "logps/rejected": -462.70123291015625,
      "loss": 0.4505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13079489767551422,
      "rewards/margins": 0.9909904599189758,
      "rewards/rejected": -1.1217854022979736,
      "step": 2908
    },
    {
      "epoch": 0.3806657397562771,
      "grad_norm": 6.012912273406982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08107013255357742,
      "logits/rejected": -0.2315695434808731,
      "logps/chosen": -405.5555114746094,
      "logps/rejected": -478.0839538574219,
      "loss": 0.3993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3682689666748047,
      "rewards/margins": 1.1915779113769531,
      "rewards/rejected": -1.5598468780517578,
      "step": 2909
    },
    {
      "epoch": 0.38079659769362884,
      "grad_norm": 10.050398826599121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05515773594379425,
      "logits/rejected": 0.0665026381611824,
      "logps/chosen": -573.9597778320312,
      "logps/rejected": -522.37841796875,
      "loss": 1.0591,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8545072078704834,
      "rewards/margins": 0.3398318886756897,
      "rewards/rejected": -1.1943391561508179,
      "step": 2910
    },
    {
      "epoch": 0.3809274556309806,
      "grad_norm": 9.670367240905762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1988464742898941,
      "logits/rejected": 0.14794497191905975,
      "logps/chosen": -443.27703857421875,
      "logps/rejected": -473.89520263671875,
      "loss": 0.9511,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.22957688570022583,
      "rewards/margins": 0.12930795550346375,
      "rewards/rejected": -0.35888487100601196,
      "step": 2911
    },
    {
      "epoch": 0.38105831356833236,
      "grad_norm": 10.7623872756958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0069437697529792786,
      "logits/rejected": -0.09111332893371582,
      "logps/chosen": -394.6915588378906,
      "logps/rejected": -430.0338134765625,
      "loss": 0.9272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5359935760498047,
      "rewards/margins": 0.7059968113899231,
      "rewards/rejected": -2.241990327835083,
      "step": 2912
    },
    {
      "epoch": 0.38118917150568415,
      "grad_norm": 7.905346393585205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0780876874923706,
      "logits/rejected": 0.038731761276721954,
      "logps/chosen": -420.7666320800781,
      "logps/rejected": -571.193115234375,
      "loss": 0.5547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5497791767120361,
      "rewards/margins": 1.2188878059387207,
      "rewards/rejected": -1.7686669826507568,
      "step": 2913
    },
    {
      "epoch": 0.3813200294430359,
      "grad_norm": 4.507174491882324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3431210517883301,
      "logits/rejected": 0.1390710175037384,
      "logps/chosen": -529.34521484375,
      "logps/rejected": -456.1551208496094,
      "loss": 0.3351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0033910274505615,
      "rewards/margins": 1.1717027425765991,
      "rewards/rejected": -0.1683116853237152,
      "step": 2914
    },
    {
      "epoch": 0.3814508873803877,
      "grad_norm": 9.28369140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3089163303375244,
      "logits/rejected": 0.24083465337753296,
      "logps/chosen": -461.6734619140625,
      "logps/rejected": -549.433349609375,
      "loss": 0.6526,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2125728577375412,
      "rewards/margins": 0.5853224396705627,
      "rewards/rejected": -0.7978951930999756,
      "step": 2915
    },
    {
      "epoch": 0.3815817453177394,
      "grad_norm": 5.761511325836182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23566275835037231,
      "logits/rejected": 0.021245799958705902,
      "logps/chosen": -420.48248291015625,
      "logps/rejected": -388.7232360839844,
      "loss": 0.3211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.049274250864982605,
      "rewards/margins": 1.9112441539764404,
      "rewards/rejected": -1.960518479347229,
      "step": 2916
    },
    {
      "epoch": 0.3817126032550912,
      "grad_norm": 7.610236167907715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11906720697879791,
      "logits/rejected": 0.1037488728761673,
      "logps/chosen": -650.773681640625,
      "logps/rejected": -703.973388671875,
      "loss": 0.5318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06217728927731514,
      "rewards/margins": 0.8616266250610352,
      "rewards/rejected": -0.7994493842124939,
      "step": 2917
    },
    {
      "epoch": 0.38184346119244295,
      "grad_norm": 7.036723613739014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.332307368516922,
      "logits/rejected": 0.1967780739068985,
      "logps/chosen": -644.6655883789062,
      "logps/rejected": -601.8272705078125,
      "loss": 0.4046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07346534729003906,
      "rewards/margins": 1.0729151964187622,
      "rewards/rejected": -0.9994499087333679,
      "step": 2918
    },
    {
      "epoch": 0.38197431912979474,
      "grad_norm": 8.730850219726562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07050305604934692,
      "logits/rejected": -0.0024562180042266846,
      "logps/chosen": -295.8260803222656,
      "logps/rejected": -352.0939025878906,
      "loss": 0.6481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9961879849433899,
      "rewards/margins": 0.48028069734573364,
      "rewards/rejected": -1.4764686822891235,
      "step": 2919
    },
    {
      "epoch": 0.3821051770671465,
      "grad_norm": 5.976886749267578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07156909257173538,
      "logits/rejected": 0.1360938400030136,
      "logps/chosen": -477.52557373046875,
      "logps/rejected": -476.8219299316406,
      "loss": 0.4492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7833630442619324,
      "rewards/margins": 0.7156076431274414,
      "rewards/rejected": -1.498970627784729,
      "step": 2920
    },
    {
      "epoch": 0.38223603500449826,
      "grad_norm": 10.738628387451172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2611326575279236,
      "logits/rejected": -0.05736137181520462,
      "logps/chosen": -462.86279296875,
      "logps/rejected": -491.4198303222656,
      "loss": 0.8207,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17944546043872833,
      "rewards/margins": -0.005703523755073547,
      "rewards/rejected": -0.1737418919801712,
      "step": 2921
    },
    {
      "epoch": 0.38236689294185,
      "grad_norm": 6.734566688537598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18664246797561646,
      "logits/rejected": 0.22194227576255798,
      "logps/chosen": -479.4076843261719,
      "logps/rejected": -433.3917541503906,
      "loss": 0.539,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14690189063549042,
      "rewards/margins": 1.013373851776123,
      "rewards/rejected": -1.1602756977081299,
      "step": 2922
    },
    {
      "epoch": 0.3824977508792018,
      "grad_norm": 5.827409744262695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026182256639003754,
      "logits/rejected": -0.047215744853019714,
      "logps/chosen": -451.62109375,
      "logps/rejected": -275.2967834472656,
      "loss": 0.5589,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4449094533920288,
      "rewards/margins": 0.8359630107879639,
      "rewards/rejected": -0.3910534679889679,
      "step": 2923
    },
    {
      "epoch": 0.38262860881655353,
      "grad_norm": 8.433539390563965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028946630656719208,
      "logits/rejected": 0.058653853833675385,
      "logps/chosen": -511.58270263671875,
      "logps/rejected": -452.8763122558594,
      "loss": 0.4613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.003284662961959839,
      "rewards/margins": 1.1542763710021973,
      "rewards/rejected": -1.157561182975769,
      "step": 2924
    },
    {
      "epoch": 0.38275946675390526,
      "grad_norm": 8.148067474365234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22678855061531067,
      "logits/rejected": -0.25717535614967346,
      "logps/chosen": -344.7593994140625,
      "logps/rejected": -526.4586791992188,
      "loss": 0.5375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7047414779663086,
      "rewards/margins": 0.8006294369697571,
      "rewards/rejected": -1.5053709745407104,
      "step": 2925
    },
    {
      "epoch": 0.38289032469125706,
      "grad_norm": 6.5656657218933105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14772167801856995,
      "logits/rejected": -0.04996047914028168,
      "logps/chosen": -514.36083984375,
      "logps/rejected": -510.4999694824219,
      "loss": 0.509,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14027997851371765,
      "rewards/margins": 1.0335309505462646,
      "rewards/rejected": -1.1738109588623047,
      "step": 2926
    },
    {
      "epoch": 0.3830211826286088,
      "grad_norm": 6.413641929626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23440353572368622,
      "logits/rejected": 0.29474058747291565,
      "logps/chosen": -381.94189453125,
      "logps/rejected": -466.450927734375,
      "loss": 0.4824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2723310589790344,
      "rewards/margins": 1.1084508895874023,
      "rewards/rejected": -1.380781888961792,
      "step": 2927
    },
    {
      "epoch": 0.3831520405659606,
      "grad_norm": 6.568641185760498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40169617533683777,
      "logits/rejected": 0.23535269498825073,
      "logps/chosen": -378.7916564941406,
      "logps/rejected": -326.8375244140625,
      "loss": 0.6471,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2542743682861328,
      "rewards/margins": 0.23066160082817078,
      "rewards/rejected": -0.4849359691143036,
      "step": 2928
    },
    {
      "epoch": 0.3832828985033123,
      "grad_norm": 8.233434677124023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05893005430698395,
      "logits/rejected": 0.1358499377965927,
      "logps/chosen": -453.1239013671875,
      "logps/rejected": -371.8790283203125,
      "loss": 0.7055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8763864040374756,
      "rewards/margins": 0.28603842854499817,
      "rewards/rejected": -1.162424921989441,
      "step": 2929
    },
    {
      "epoch": 0.3834137564406641,
      "grad_norm": 6.110736846923828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09557348489761353,
      "logits/rejected": 0.028745215386152267,
      "logps/chosen": -403.5850830078125,
      "logps/rejected": -427.0557861328125,
      "loss": 0.5793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4991867244243622,
      "rewards/margins": 0.8238153457641602,
      "rewards/rejected": -1.3230019807815552,
      "step": 2930
    },
    {
      "epoch": 0.38354461437801585,
      "grad_norm": 4.7668633460998535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0848570466041565,
      "logits/rejected": 0.21987226605415344,
      "logps/chosen": -303.577880859375,
      "logps/rejected": -358.0675354003906,
      "loss": 0.4346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20309850573539734,
      "rewards/margins": 0.7272264957427979,
      "rewards/rejected": -0.9303250312805176,
      "step": 2931
    },
    {
      "epoch": 0.38367547231536764,
      "grad_norm": 4.929851531982422,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2645789384841919,
      "logits/rejected": 0.02898292988538742,
      "logps/chosen": -454.7286376953125,
      "logps/rejected": -416.12774658203125,
      "loss": 0.4519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.559400200843811,
      "rewards/margins": 0.6816362142562866,
      "rewards/rejected": -1.2410364151000977,
      "step": 2932
    },
    {
      "epoch": 0.3838063302527194,
      "grad_norm": 6.330361843109131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11444590240716934,
      "logits/rejected": 0.227580726146698,
      "logps/chosen": -417.2696533203125,
      "logps/rejected": -537.288818359375,
      "loss": 0.6325,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10078218579292297,
      "rewards/margins": 0.7964317202568054,
      "rewards/rejected": -0.6956495046615601,
      "step": 2933
    },
    {
      "epoch": 0.38393718819007117,
      "grad_norm": 6.489099979400635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07946401834487915,
      "logits/rejected": 0.027882684022188187,
      "logps/chosen": -438.98583984375,
      "logps/rejected": -402.0350646972656,
      "loss": 0.4992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3298355042934418,
      "rewards/margins": 0.6986638903617859,
      "rewards/rejected": -1.0284993648529053,
      "step": 2934
    },
    {
      "epoch": 0.3840680461274229,
      "grad_norm": 6.298666954040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026635875925421715,
      "logits/rejected": 0.03289676457643509,
      "logps/chosen": -503.18023681640625,
      "logps/rejected": -491.5255126953125,
      "loss": 0.4008,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38955381512641907,
      "rewards/margins": 1.068095088005066,
      "rewards/rejected": -1.4576488733291626,
      "step": 2935
    },
    {
      "epoch": 0.3841989040647747,
      "grad_norm": 5.596957683563232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28477394580841064,
      "logits/rejected": 0.3722805380821228,
      "logps/chosen": -297.3897399902344,
      "logps/rejected": -404.408447265625,
      "loss": 0.4343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21306954324245453,
      "rewards/margins": 1.0783357620239258,
      "rewards/rejected": -1.2914053201675415,
      "step": 2936
    },
    {
      "epoch": 0.38432976200212643,
      "grad_norm": 5.276542663574219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034146808087825775,
      "logits/rejected": 0.24177995324134827,
      "logps/chosen": -278.24603271484375,
      "logps/rejected": -313.45758056640625,
      "loss": 0.5429,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11123047769069672,
      "rewards/margins": 0.5370283126831055,
      "rewards/rejected": -0.6482588052749634,
      "step": 2937
    },
    {
      "epoch": 0.3844606199394782,
      "grad_norm": 10.772818565368652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20476406812667847,
      "logits/rejected": 0.10501794517040253,
      "logps/chosen": -415.74981689453125,
      "logps/rejected": -403.2776184082031,
      "loss": 0.7621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5296589732170105,
      "rewards/margins": 0.2286342829465866,
      "rewards/rejected": -0.7582932710647583,
      "step": 2938
    },
    {
      "epoch": 0.38459147787682996,
      "grad_norm": 5.173336505889893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027941688895225525,
      "logits/rejected": -0.4147818088531494,
      "logps/chosen": -388.4845886230469,
      "logps/rejected": -381.45367431640625,
      "loss": 0.3655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3633257746696472,
      "rewards/margins": 1.0417338609695435,
      "rewards/rejected": -0.6784080266952515,
      "step": 2939
    },
    {
      "epoch": 0.38472233581418175,
      "grad_norm": 8.956539154052734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1676226258277893,
      "logits/rejected": -0.10402584075927734,
      "logps/chosen": -412.86260986328125,
      "logps/rejected": -469.0961608886719,
      "loss": 0.7137,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10011462867259979,
      "rewards/margins": 0.6380769610404968,
      "rewards/rejected": -0.7381916046142578,
      "step": 2940
    },
    {
      "epoch": 0.3848531937515335,
      "grad_norm": 7.584024429321289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03540065884590149,
      "logits/rejected": -0.001329369843006134,
      "logps/chosen": -431.22161865234375,
      "logps/rejected": -364.429443359375,
      "loss": 0.6507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4745742678642273,
      "rewards/margins": 0.2295711487531662,
      "rewards/rejected": -0.7041454315185547,
      "step": 2941
    },
    {
      "epoch": 0.3849840516888853,
      "grad_norm": 6.742041110992432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.253142386674881,
      "logits/rejected": 0.039400774985551834,
      "logps/chosen": -352.7943115234375,
      "logps/rejected": -351.40338134765625,
      "loss": 0.5341,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6017227172851562,
      "rewards/margins": 0.5970598459243774,
      "rewards/rejected": -1.1987825632095337,
      "step": 2942
    },
    {
      "epoch": 0.385114909626237,
      "grad_norm": 8.662999153137207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3757229745388031,
      "logits/rejected": 0.2764397859573364,
      "logps/chosen": -493.88702392578125,
      "logps/rejected": -522.022705078125,
      "loss": 0.7441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5094659924507141,
      "rewards/margins": 0.3914222717285156,
      "rewards/rejected": -0.9008883237838745,
      "step": 2943
    },
    {
      "epoch": 0.3852457675635888,
      "grad_norm": 5.068352222442627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15488632023334503,
      "logits/rejected": -0.4547041058540344,
      "logps/chosen": -386.3992614746094,
      "logps/rejected": -347.818359375,
      "loss": 0.4042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2866911292076111,
      "rewards/margins": 0.8054729700088501,
      "rewards/rejected": -1.0921640396118164,
      "step": 2944
    },
    {
      "epoch": 0.38537662550094054,
      "grad_norm": 5.568187236785889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3372148871421814,
      "logits/rejected": -0.14848639070987701,
      "logps/chosen": -489.1556701660156,
      "logps/rejected": -506.05560302734375,
      "loss": 0.3279,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7874757051467896,
      "rewards/margins": 1.3369357585906982,
      "rewards/rejected": -2.1244115829467773,
      "step": 2945
    },
    {
      "epoch": 0.38550748343829233,
      "grad_norm": 4.175276279449463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3192713260650635,
      "logits/rejected": 0.0632043331861496,
      "logps/chosen": -340.5108947753906,
      "logps/rejected": -361.0648193359375,
      "loss": 0.3169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33533209562301636,
      "rewards/margins": 1.2528671026229858,
      "rewards/rejected": -0.9175350069999695,
      "step": 2946
    },
    {
      "epoch": 0.38563834137564407,
      "grad_norm": 18.862218856811523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005728226155042648,
      "logits/rejected": -0.03197348117828369,
      "logps/chosen": -559.7246704101562,
      "logps/rejected": -413.4485778808594,
      "loss": 1.3205,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.2996373176574707,
      "rewards/margins": -0.5749732255935669,
      "rewards/rejected": -0.7246643304824829,
      "step": 2947
    },
    {
      "epoch": 0.3857691993129958,
      "grad_norm": 10.734272003173828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1283518224954605,
      "logits/rejected": 0.08871633559465408,
      "logps/chosen": -408.36456298828125,
      "logps/rejected": -303.2735900878906,
      "loss": 0.7771,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6691065430641174,
      "rewards/margins": -0.05184973403811455,
      "rewards/rejected": -0.6172568202018738,
      "step": 2948
    },
    {
      "epoch": 0.3859000572503476,
      "grad_norm": 8.733227729797363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08223540335893631,
      "logits/rejected": 0.3335334360599518,
      "logps/chosen": -420.0953063964844,
      "logps/rejected": -409.08843994140625,
      "loss": 0.8916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3339681625366211,
      "rewards/margins": 0.2937059998512268,
      "rewards/rejected": -0.6276741623878479,
      "step": 2949
    },
    {
      "epoch": 0.38603091518769933,
      "grad_norm": 6.049187183380127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24768614768981934,
      "logits/rejected": 0.08234662562608719,
      "logps/chosen": -363.18646240234375,
      "logps/rejected": -428.9998474121094,
      "loss": 0.4042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6109514236450195,
      "rewards/margins": 0.9226548671722412,
      "rewards/rejected": -1.5336064100265503,
      "step": 2950
    },
    {
      "epoch": 0.3861617731250511,
      "grad_norm": 7.0413007736206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06531704217195511,
      "logits/rejected": 0.14759565889835358,
      "logps/chosen": -387.3092346191406,
      "logps/rejected": -310.4179992675781,
      "loss": 0.4638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6215063333511353,
      "rewards/margins": 0.8711856603622437,
      "rewards/rejected": -1.492691993713379,
      "step": 2951
    },
    {
      "epoch": 0.38629263106240286,
      "grad_norm": 6.527944087982178,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31427842378616333,
      "logits/rejected": 0.16134104132652283,
      "logps/chosen": -458.372802734375,
      "logps/rejected": -441.32025146484375,
      "loss": 0.5078,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025820523500442505,
      "rewards/margins": 0.8482112884521484,
      "rewards/rejected": -0.8223907947540283,
      "step": 2952
    },
    {
      "epoch": 0.38642348899975465,
      "grad_norm": 4.102355480194092,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1586298793554306,
      "logits/rejected": 0.0194394551217556,
      "logps/chosen": -566.14990234375,
      "logps/rejected": -406.22021484375,
      "loss": 0.3125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5720003843307495,
      "rewards/margins": 1.8096983432769775,
      "rewards/rejected": -1.2376980781555176,
      "step": 2953
    },
    {
      "epoch": 0.3865543469371064,
      "grad_norm": 12.112287521362305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20315706729888916,
      "logits/rejected": 0.10650563985109329,
      "logps/chosen": -464.7076721191406,
      "logps/rejected": -497.878662109375,
      "loss": 1.0555,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.0375399589538574,
      "rewards/margins": 0.12215696275234222,
      "rewards/rejected": -1.1596969366073608,
      "step": 2954
    },
    {
      "epoch": 0.3866852048744582,
      "grad_norm": 9.452129364013672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006155427545309067,
      "logits/rejected": 0.028804916888475418,
      "logps/chosen": -528.5046997070312,
      "logps/rejected": -594.3275756835938,
      "loss": 0.9472,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010887905955314636,
      "rewards/margins": 0.2014733850955963,
      "rewards/rejected": -0.21236133575439453,
      "step": 2955
    },
    {
      "epoch": 0.3868160628118099,
      "grad_norm": 8.99165153503418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16605298221111298,
      "logits/rejected": 0.13495320081710815,
      "logps/chosen": -439.88916015625,
      "logps/rejected": -419.39715576171875,
      "loss": 0.8702,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45118939876556396,
      "rewards/margins": 0.03761502355337143,
      "rewards/rejected": -0.4888044595718384,
      "step": 2956
    },
    {
      "epoch": 0.3869469207491617,
      "grad_norm": 5.431962013244629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29161518812179565,
      "logits/rejected": -0.4684324264526367,
      "logps/chosen": -256.96844482421875,
      "logps/rejected": -423.5124816894531,
      "loss": 0.3755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3761737644672394,
      "rewards/margins": 1.0439814329147339,
      "rewards/rejected": -1.4201552867889404,
      "step": 2957
    },
    {
      "epoch": 0.38707777868651344,
      "grad_norm": 6.461670398712158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09539560973644257,
      "logits/rejected": 0.014483779668807983,
      "logps/chosen": -356.67120361328125,
      "logps/rejected": -408.0711669921875,
      "loss": 0.4733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5723748803138733,
      "rewards/margins": 0.5407031774520874,
      "rewards/rejected": -1.113078236579895,
      "step": 2958
    },
    {
      "epoch": 0.38720863662386523,
      "grad_norm": 6.106997966766357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15602965652942657,
      "logits/rejected": -0.06163986772298813,
      "logps/chosen": -508.1627197265625,
      "logps/rejected": -431.295166015625,
      "loss": 0.475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3494190573692322,
      "rewards/margins": 0.7399774789810181,
      "rewards/rejected": -1.0893964767456055,
      "step": 2959
    },
    {
      "epoch": 0.38733949456121697,
      "grad_norm": 7.358178615570068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15335410833358765,
      "logits/rejected": -0.12390094995498657,
      "logps/chosen": -433.9411315917969,
      "logps/rejected": -475.0321960449219,
      "loss": 0.4888,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5900518298149109,
      "rewards/margins": 0.645602822303772,
      "rewards/rejected": -1.2356547117233276,
      "step": 2960
    },
    {
      "epoch": 0.38747035249856876,
      "grad_norm": 6.887966632843018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22862786054611206,
      "logits/rejected": -0.2109920084476471,
      "logps/chosen": -435.77593994140625,
      "logps/rejected": -456.502685546875,
      "loss": 0.4647,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3756372928619385,
      "rewards/margins": 0.9339934587478638,
      "rewards/rejected": -1.3096307516098022,
      "step": 2961
    },
    {
      "epoch": 0.3876012104359205,
      "grad_norm": 11.319253921508789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07842610776424408,
      "logits/rejected": 0.15076854825019836,
      "logps/chosen": -570.9088134765625,
      "logps/rejected": -454.86572265625,
      "loss": 0.8013,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01877633109688759,
      "rewards/margins": 0.2958860993385315,
      "rewards/rejected": -0.27710971236228943,
      "step": 2962
    },
    {
      "epoch": 0.3877320683732723,
      "grad_norm": 7.484170913696289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1386035829782486,
      "logits/rejected": 0.25801363587379456,
      "logps/chosen": -579.6435546875,
      "logps/rejected": -503.0857238769531,
      "loss": 0.5057,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3693416714668274,
      "rewards/margins": 0.6866969466209412,
      "rewards/rejected": -0.31735533475875854,
      "step": 2963
    },
    {
      "epoch": 0.387862926310624,
      "grad_norm": 7.468449115753174,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23618420958518982,
      "logits/rejected": 0.2420286238193512,
      "logps/chosen": -474.59661865234375,
      "logps/rejected": -392.9976806640625,
      "loss": 0.6413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2804940938949585,
      "rewards/margins": 0.5879784226417542,
      "rewards/rejected": -0.3074842691421509,
      "step": 2964
    },
    {
      "epoch": 0.3879937842479758,
      "grad_norm": 8.677544593811035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1383846551179886,
      "logits/rejected": -0.11362434178590775,
      "logps/chosen": -434.179931640625,
      "logps/rejected": -392.3792419433594,
      "loss": 0.5475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04837170988321304,
      "rewards/margins": 0.4911573529243469,
      "rewards/rejected": -0.4427856504917145,
      "step": 2965
    },
    {
      "epoch": 0.38812464218532755,
      "grad_norm": 5.646742820739746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3469737470149994,
      "logits/rejected": 0.4555703401565552,
      "logps/chosen": -577.7672729492188,
      "logps/rejected": -419.8468017578125,
      "loss": 0.3731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9418898820877075,
      "rewards/margins": 1.2030212879180908,
      "rewards/rejected": -0.2611314654350281,
      "step": 2966
    },
    {
      "epoch": 0.38825550012267934,
      "grad_norm": 6.117804050445557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06341347098350525,
      "logits/rejected": -0.008134637027978897,
      "logps/chosen": -381.8639831542969,
      "logps/rejected": -380.3137512207031,
      "loss": 0.4491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5371246933937073,
      "rewards/margins": 1.0097103118896484,
      "rewards/rejected": -0.47258567810058594,
      "step": 2967
    },
    {
      "epoch": 0.3883863580600311,
      "grad_norm": 6.443618297576904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13186213374137878,
      "logits/rejected": -0.013232618570327759,
      "logps/chosen": -424.7532653808594,
      "logps/rejected": -409.8569030761719,
      "loss": 0.5852,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3694532513618469,
      "rewards/margins": 0.525027871131897,
      "rewards/rejected": -0.15557460486888885,
      "step": 2968
    },
    {
      "epoch": 0.3885172159973828,
      "grad_norm": 4.415490627288818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008505607023835182,
      "logits/rejected": 0.001287803053855896,
      "logps/chosen": -514.3441772460938,
      "logps/rejected": -403.25799560546875,
      "loss": 0.2946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9699720740318298,
      "rewards/margins": 1.4653215408325195,
      "rewards/rejected": -0.4953495264053345,
      "step": 2969
    },
    {
      "epoch": 0.3886480739347346,
      "grad_norm": 5.21290922164917,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008326857350766659,
      "logits/rejected": -0.08415921032428741,
      "logps/chosen": -320.7711181640625,
      "logps/rejected": -400.265869140625,
      "loss": 0.4056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1150054782629013,
      "rewards/margins": 0.8369840979576111,
      "rewards/rejected": -0.9519895911216736,
      "step": 2970
    },
    {
      "epoch": 0.38877893187208634,
      "grad_norm": 4.183433532714844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2001838982105255,
      "logits/rejected": 0.1301582157611847,
      "logps/chosen": -466.38714599609375,
      "logps/rejected": -415.8916015625,
      "loss": 0.31,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.478397011756897,
      "rewards/margins": 1.2655174732208252,
      "rewards/rejected": -0.7871204614639282,
      "step": 2971
    },
    {
      "epoch": 0.38890978980943813,
      "grad_norm": 6.043584823608398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06246384233236313,
      "logits/rejected": 0.04167952388525009,
      "logps/chosen": -439.7443542480469,
      "logps/rejected": -560.678955078125,
      "loss": 0.3558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31765425205230713,
      "rewards/margins": 1.8478249311447144,
      "rewards/rejected": -1.5301706790924072,
      "step": 2972
    },
    {
      "epoch": 0.38904064774678987,
      "grad_norm": 7.356922149658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3468340039253235,
      "logits/rejected": 0.3411308526992798,
      "logps/chosen": -284.5814514160156,
      "logps/rejected": -414.27410888671875,
      "loss": 0.592,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10454368591308594,
      "rewards/margins": 0.8972217440605164,
      "rewards/rejected": -0.7926781177520752,
      "step": 2973
    },
    {
      "epoch": 0.38917150568414166,
      "grad_norm": 7.274130344390869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25017037987709045,
      "logits/rejected": 0.05870771408081055,
      "logps/chosen": -674.7748413085938,
      "logps/rejected": -557.7369384765625,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08667485415935516,
      "rewards/margins": 0.5544289350509644,
      "rewards/rejected": -0.6411037445068359,
      "step": 2974
    },
    {
      "epoch": 0.3893023636214934,
      "grad_norm": 4.928487777709961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051705218851566315,
      "logits/rejected": -0.13351649045944214,
      "logps/chosen": -590.3707275390625,
      "logps/rejected": -677.2429809570312,
      "loss": 0.3512,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.050946980714797974,
      "rewards/margins": 1.2031718492507935,
      "rewards/rejected": -1.2541189193725586,
      "step": 2975
    },
    {
      "epoch": 0.3894332215588452,
      "grad_norm": 8.448344230651855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12367090582847595,
      "logits/rejected": 0.15676838159561157,
      "logps/chosen": -495.79974365234375,
      "logps/rejected": -517.1683959960938,
      "loss": 0.6788,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2778453826904297,
      "rewards/margins": 0.32589930295944214,
      "rewards/rejected": -0.6037446856498718,
      "step": 2976
    },
    {
      "epoch": 0.3895640794961969,
      "grad_norm": 5.669559955596924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1801048219203949,
      "logits/rejected": -0.015310615301132202,
      "logps/chosen": -447.9387512207031,
      "logps/rejected": -398.25048828125,
      "loss": 0.4472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19644127786159515,
      "rewards/margins": 0.7166267037391663,
      "rewards/rejected": -0.9130679965019226,
      "step": 2977
    },
    {
      "epoch": 0.3896949374335487,
      "grad_norm": 5.25955867767334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014156892895698547,
      "logits/rejected": -0.06933975964784622,
      "logps/chosen": -285.547607421875,
      "logps/rejected": -377.75433349609375,
      "loss": 0.369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0866151675581932,
      "rewards/margins": 1.6791419982910156,
      "rewards/rejected": -1.592526912689209,
      "step": 2978
    },
    {
      "epoch": 0.38982579537090045,
      "grad_norm": 9.356973648071289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17585541307926178,
      "logits/rejected": 0.19000062346458435,
      "logps/chosen": -476.8039245605469,
      "logps/rejected": -612.506103515625,
      "loss": 0.6259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.34166163206100464,
      "rewards/margins": 0.8026285171508789,
      "rewards/rejected": -0.46096688508987427,
      "step": 2979
    },
    {
      "epoch": 0.38995665330825224,
      "grad_norm": 4.519033432006836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07731162756681442,
      "logits/rejected": -0.13071753084659576,
      "logps/chosen": -323.531982421875,
      "logps/rejected": -313.77685546875,
      "loss": 0.3945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40432947874069214,
      "rewards/margins": 1.0804272890090942,
      "rewards/rejected": -1.4847567081451416,
      "step": 2980
    },
    {
      "epoch": 0.390087511245604,
      "grad_norm": 8.737598419189453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053162552416324615,
      "logits/rejected": -0.07855769246816635,
      "logps/chosen": -455.36334228515625,
      "logps/rejected": -449.218994140625,
      "loss": 0.7222,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7360134720802307,
      "rewards/margins": 0.2628985643386841,
      "rewards/rejected": -0.9989120364189148,
      "step": 2981
    },
    {
      "epoch": 0.39021836918295577,
      "grad_norm": 6.576456546783447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13653787970542908,
      "logits/rejected": 0.18814092874526978,
      "logps/chosen": -619.21435546875,
      "logps/rejected": -505.9036560058594,
      "loss": 0.5181,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5020679831504822,
      "rewards/margins": 0.7771747708320618,
      "rewards/rejected": -0.275106817483902,
      "step": 2982
    },
    {
      "epoch": 0.3903492271203075,
      "grad_norm": 8.59168815612793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0266314297914505,
      "logits/rejected": 0.0008648335933685303,
      "logps/chosen": -437.373779296875,
      "logps/rejected": -390.4237976074219,
      "loss": 0.6627,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2387068122625351,
      "rewards/margins": 0.33980250358581543,
      "rewards/rejected": -0.5785093307495117,
      "step": 2983
    },
    {
      "epoch": 0.3904800850576593,
      "grad_norm": 4.099761962890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07246286422014236,
      "logits/rejected": 0.16207045316696167,
      "logps/chosen": -519.2737426757812,
      "logps/rejected": -422.06866455078125,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3352261781692505,
      "rewards/margins": 2.265096426010132,
      "rewards/rejected": -0.9298702478408813,
      "step": 2984
    },
    {
      "epoch": 0.39061094299501103,
      "grad_norm": 4.92038106918335,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24631890654563904,
      "logits/rejected": 0.1165255457162857,
      "logps/chosen": -496.951171875,
      "logps/rejected": -407.15460205078125,
      "loss": 0.3289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42596226930618286,
      "rewards/margins": 1.2060171365737915,
      "rewards/rejected": -0.7800549268722534,
      "step": 2985
    },
    {
      "epoch": 0.3907418009323628,
      "grad_norm": 2.7701900005340576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3640415370464325,
      "logits/rejected": 0.25498080253601074,
      "logps/chosen": -426.2901611328125,
      "logps/rejected": -457.6222839355469,
      "loss": 0.1849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4823518693447113,
      "rewards/margins": 2.4402618408203125,
      "rewards/rejected": -1.9579098224639893,
      "step": 2986
    },
    {
      "epoch": 0.39087265886971456,
      "grad_norm": 5.948419570922852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08772139251232147,
      "logits/rejected": 0.18054935336112976,
      "logps/chosen": -410.7646789550781,
      "logps/rejected": -398.09283447265625,
      "loss": 0.4424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19662857055664062,
      "rewards/margins": 0.8117502331733704,
      "rewards/rejected": -0.6151216626167297,
      "step": 2987
    },
    {
      "epoch": 0.39100351680706635,
      "grad_norm": 4.920621395111084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12370050698518753,
      "logits/rejected": -0.19404244422912598,
      "logps/chosen": -439.4459228515625,
      "logps/rejected": -463.6347351074219,
      "loss": 0.3951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.44410550594329834,
      "rewards/margins": 1.3236459493637085,
      "rewards/rejected": -0.8795404434204102,
      "step": 2988
    },
    {
      "epoch": 0.3911343747444181,
      "grad_norm": 6.726259708404541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23571722209453583,
      "logits/rejected": -0.004426371306180954,
      "logps/chosen": -484.30413818359375,
      "logps/rejected": -390.72991943359375,
      "loss": 0.5995,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10154649615287781,
      "rewards/margins": 0.634284257888794,
      "rewards/rejected": -0.5327377915382385,
      "step": 2989
    },
    {
      "epoch": 0.3912652326817699,
      "grad_norm": 5.982395648956299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3829282820224762,
      "logits/rejected": 0.24919193983078003,
      "logps/chosen": -421.2436218261719,
      "logps/rejected": -376.8311767578125,
      "loss": 0.3831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15181006491184235,
      "rewards/margins": 0.8844800591468811,
      "rewards/rejected": -0.7326700091362,
      "step": 2990
    },
    {
      "epoch": 0.3913960906191216,
      "grad_norm": 11.166703224182129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24307796359062195,
      "logits/rejected": 0.21117933094501495,
      "logps/chosen": -468.5820007324219,
      "logps/rejected": -505.02294921875,
      "loss": 0.5663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2660002112388611,
      "rewards/margins": 0.7260334491729736,
      "rewards/rejected": -0.46003323793411255,
      "step": 2991
    },
    {
      "epoch": 0.39152694855647335,
      "grad_norm": 7.279455184936523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1128440648317337,
      "logits/rejected": -0.004620589315891266,
      "logps/chosen": -466.63470458984375,
      "logps/rejected": -381.1782531738281,
      "loss": 0.483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07148580253124237,
      "rewards/margins": 0.9104211330413818,
      "rewards/rejected": -0.8389353156089783,
      "step": 2992
    },
    {
      "epoch": 0.39165780649382514,
      "grad_norm": 3.934478521347046,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012519221752882004,
      "logits/rejected": -0.3498799800872803,
      "logps/chosen": -509.43023681640625,
      "logps/rejected": -371.56646728515625,
      "loss": 0.2039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03162291646003723,
      "rewards/margins": 1.940471887588501,
      "rewards/rejected": -1.972095012664795,
      "step": 2993
    },
    {
      "epoch": 0.3917886644311769,
      "grad_norm": 11.152571678161621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15101750195026398,
      "logits/rejected": -0.020813584327697754,
      "logps/chosen": -390.51239013671875,
      "logps/rejected": -373.2122802734375,
      "loss": 1.0705,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7513725757598877,
      "rewards/margins": -0.2198629379272461,
      "rewards/rejected": -0.5315096378326416,
      "step": 2994
    },
    {
      "epoch": 0.39191952236852867,
      "grad_norm": 7.018592357635498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16135844588279724,
      "logits/rejected": 0.14469042420387268,
      "logps/chosen": -446.111572265625,
      "logps/rejected": -416.4078063964844,
      "loss": 0.506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9761253595352173,
      "rewards/margins": 0.5446515679359436,
      "rewards/rejected": -1.5207769870758057,
      "step": 2995
    },
    {
      "epoch": 0.3920503803058804,
      "grad_norm": 7.696230888366699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09278956055641174,
      "logits/rejected": 0.05635128170251846,
      "logps/chosen": -507.878662109375,
      "logps/rejected": -609.6177978515625,
      "loss": 0.565,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5158626437187195,
      "rewards/margins": 0.5878154635429382,
      "rewards/rejected": -0.07195280492305756,
      "step": 2996
    },
    {
      "epoch": 0.3921812382432322,
      "grad_norm": 6.62009334564209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06765661388635635,
      "logits/rejected": -0.0854136049747467,
      "logps/chosen": -401.0016784667969,
      "logps/rejected": -405.0505065917969,
      "loss": 0.541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12476159632205963,
      "rewards/margins": 0.6324453353881836,
      "rewards/rejected": -0.5076837539672852,
      "step": 2997
    },
    {
      "epoch": 0.39231209618058394,
      "grad_norm": 7.930939197540283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008603096008300781,
      "logits/rejected": 0.09665516763925552,
      "logps/chosen": -342.47406005859375,
      "logps/rejected": -331.09381103515625,
      "loss": 0.6963,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11131992936134338,
      "rewards/margins": 0.10739782452583313,
      "rewards/rejected": -0.2187177687883377,
      "step": 2998
    },
    {
      "epoch": 0.3924429541179357,
      "grad_norm": 6.44980001449585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23265844583511353,
      "logits/rejected": 0.19768556952476501,
      "logps/chosen": -557.5859985351562,
      "logps/rejected": -535.5828247070312,
      "loss": 0.321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4856729805469513,
      "rewards/margins": 1.5014656782150269,
      "rewards/rejected": -1.0157926082611084,
      "step": 2999
    },
    {
      "epoch": 0.39257381205528746,
      "grad_norm": 9.240795135498047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02925831824541092,
      "logits/rejected": -0.016031354665756226,
      "logps/chosen": -338.06243896484375,
      "logps/rejected": -470.9324645996094,
      "loss": 0.7314,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2821277976036072,
      "rewards/margins": -0.016114428639411926,
      "rewards/rejected": -0.26601338386535645,
      "step": 3000
    },
    {
      "epoch": 0.39270466999263925,
      "grad_norm": 11.009373664855957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03864918649196625,
      "logits/rejected": -0.1007227972149849,
      "logps/chosen": -412.4826354980469,
      "logps/rejected": -303.5528564453125,
      "loss": 0.9402,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2057683765888214,
      "rewards/margins": -0.20571838319301605,
      "rewards/rejected": -4.998594522476196e-05,
      "step": 3001
    },
    {
      "epoch": 0.392835527929991,
      "grad_norm": 7.3766608238220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05116913095116615,
      "logits/rejected": 0.18140801787376404,
      "logps/chosen": -404.12481689453125,
      "logps/rejected": -543.399169921875,
      "loss": 0.4485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2212410271167755,
      "rewards/margins": 0.7837772369384766,
      "rewards/rejected": -1.0050182342529297,
      "step": 3002
    },
    {
      "epoch": 0.3929663858673428,
      "grad_norm": 6.662258625030518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24863353371620178,
      "logits/rejected": -0.02524649351835251,
      "logps/chosen": -591.7325439453125,
      "logps/rejected": -494.2380065917969,
      "loss": 0.4044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22955931723117828,
      "rewards/margins": 1.3760602474212646,
      "rewards/rejected": -1.6056196689605713,
      "step": 3003
    },
    {
      "epoch": 0.3930972438046945,
      "grad_norm": 7.700562953948975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04278230667114258,
      "logits/rejected": 0.03019820526242256,
      "logps/chosen": -460.4573059082031,
      "logps/rejected": -503.515869140625,
      "loss": 0.6867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5673727989196777,
      "rewards/margins": 0.27090373635292053,
      "rewards/rejected": 0.29646894335746765,
      "step": 3004
    },
    {
      "epoch": 0.3932281017420463,
      "grad_norm": 5.208078384399414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11609814316034317,
      "logits/rejected": 0.029107028618454933,
      "logps/chosen": -347.65948486328125,
      "logps/rejected": -571.9909057617188,
      "loss": 0.4519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28826451301574707,
      "rewards/margins": 0.7953771948814392,
      "rewards/rejected": -1.083641529083252,
      "step": 3005
    },
    {
      "epoch": 0.39335895967939805,
      "grad_norm": 5.257544994354248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4102950990200043,
      "logits/rejected": 0.18609519302845,
      "logps/chosen": -260.1481628417969,
      "logps/rejected": -311.3725891113281,
      "loss": 0.3994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0016540363430976868,
      "rewards/margins": 0.9806297421455383,
      "rewards/rejected": -0.9822837710380554,
      "step": 3006
    },
    {
      "epoch": 0.39348981761674984,
      "grad_norm": 8.52885913848877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09189949184656143,
      "logits/rejected": 0.0631285011768341,
      "logps/chosen": -415.6791687011719,
      "logps/rejected": -385.44268798828125,
      "loss": 0.7948,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4747268855571747,
      "rewards/margins": 0.03163890540599823,
      "rewards/rejected": -0.5063657760620117,
      "step": 3007
    },
    {
      "epoch": 0.3936206755541016,
      "grad_norm": 7.734529495239258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3162693679332733,
      "logits/rejected": 0.40463966131210327,
      "logps/chosen": -408.14910888671875,
      "logps/rejected": -321.5479736328125,
      "loss": 0.6268,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0644862949848175,
      "rewards/margins": 0.6555012464523315,
      "rewards/rejected": -0.7199876308441162,
      "step": 3008
    },
    {
      "epoch": 0.39375153349145336,
      "grad_norm": 5.647879123687744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.234459787607193,
      "logits/rejected": 0.2635367512702942,
      "logps/chosen": -301.70379638671875,
      "logps/rejected": -307.5533142089844,
      "loss": 0.5402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10658034682273865,
      "rewards/margins": 0.3962446451187134,
      "rewards/rejected": -0.28966426849365234,
      "step": 3009
    },
    {
      "epoch": 0.3938823914288051,
      "grad_norm": 7.897002696990967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10351136326789856,
      "logits/rejected": 0.06250373274087906,
      "logps/chosen": -334.9079895019531,
      "logps/rejected": -351.2049865722656,
      "loss": 0.8531,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.597698450088501,
      "rewards/margins": -0.1366979479789734,
      "rewards/rejected": -0.46100038290023804,
      "step": 3010
    },
    {
      "epoch": 0.3940132493661569,
      "grad_norm": 7.902576923370361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18310391902923584,
      "logits/rejected": 0.014927931129932404,
      "logps/chosen": -338.26751708984375,
      "logps/rejected": -746.2836303710938,
      "loss": 0.4735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3476499915122986,
      "rewards/margins": 1.1966532468795776,
      "rewards/rejected": -0.8490033745765686,
      "step": 3011
    },
    {
      "epoch": 0.39414410730350863,
      "grad_norm": 8.960214614868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01699645072221756,
      "logits/rejected": -0.09470992535352707,
      "logps/chosen": -467.42816162109375,
      "logps/rejected": -380.7436828613281,
      "loss": 0.869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004976928234100342,
      "rewards/margins": -0.012571677565574646,
      "rewards/rejected": 0.017548635601997375,
      "step": 3012
    },
    {
      "epoch": 0.39427496524086036,
      "grad_norm": 10.26351261138916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08984434604644775,
      "logits/rejected": -0.028074268251657486,
      "logps/chosen": -303.3870849609375,
      "logps/rejected": -265.17803955078125,
      "loss": 0.7601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.17145709693431854,
      "rewards/margins": 0.08379553258419037,
      "rewards/rejected": 0.08766156435012817,
      "step": 3013
    },
    {
      "epoch": 0.39440582317821216,
      "grad_norm": 8.825556755065918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25269195437431335,
      "logits/rejected": 0.07935851812362671,
      "logps/chosen": -413.801025390625,
      "logps/rejected": -312.8723449707031,
      "loss": 0.658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2669757604598999,
      "rewards/margins": 0.5473064184188843,
      "rewards/rejected": -0.814282238483429,
      "step": 3014
    },
    {
      "epoch": 0.3945366811155639,
      "grad_norm": 5.863706111907959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10541248321533203,
      "logits/rejected": 0.06719934195280075,
      "logps/chosen": -435.8989562988281,
      "logps/rejected": -404.11944580078125,
      "loss": 0.4877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8194469213485718,
      "rewards/margins": 0.8499992489814758,
      "rewards/rejected": -0.030552327632904053,
      "step": 3015
    },
    {
      "epoch": 0.3946675390529157,
      "grad_norm": 15.04797649383545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09851446747779846,
      "logits/rejected": -0.03148616850376129,
      "logps/chosen": -509.89697265625,
      "logps/rejected": -554.9888305664062,
      "loss": 0.9187,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19323387742042542,
      "rewards/margins": 0.12717397511005402,
      "rewards/rejected": -0.3204078674316406,
      "step": 3016
    },
    {
      "epoch": 0.3947983969902674,
      "grad_norm": 7.538003921508789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03743385523557663,
      "logits/rejected": -0.0057968869805336,
      "logps/chosen": -513.175048828125,
      "logps/rejected": -522.3743896484375,
      "loss": 0.7094,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2620837986469269,
      "rewards/margins": 0.3837147653102875,
      "rewards/rejected": -0.6457984447479248,
      "step": 3017
    },
    {
      "epoch": 0.3949292549276192,
      "grad_norm": 10.047222137451172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0431092195212841,
      "logits/rejected": 0.143050417304039,
      "logps/chosen": -480.1723937988281,
      "logps/rejected": -462.65777587890625,
      "loss": 1.1299,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10576094686985016,
      "rewards/margins": -0.06932985782623291,
      "rewards/rejected": 0.17509080469608307,
      "step": 3018
    },
    {
      "epoch": 0.39506011286497095,
      "grad_norm": 6.646839141845703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1674187332391739,
      "logits/rejected": 0.34065091609954834,
      "logps/chosen": -446.69268798828125,
      "logps/rejected": -576.0042114257812,
      "loss": 0.4358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.714900016784668,
      "rewards/margins": 1.2261961698532104,
      "rewards/rejected": -0.5112960934638977,
      "step": 3019
    },
    {
      "epoch": 0.39519097080232274,
      "grad_norm": 4.8101301193237305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021431025117635727,
      "logits/rejected": -0.011649064719676971,
      "logps/chosen": -475.43206787109375,
      "logps/rejected": -494.3134765625,
      "loss": 0.3212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5890529751777649,
      "rewards/margins": 1.2530609369277954,
      "rewards/rejected": -0.6640079021453857,
      "step": 3020
    },
    {
      "epoch": 0.3953218287396745,
      "grad_norm": 6.783010005950928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05121912062168121,
      "logits/rejected": 0.10365703701972961,
      "logps/chosen": -512.4153442382812,
      "logps/rejected": -333.9955749511719,
      "loss": 0.5527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007871992886066437,
      "rewards/margins": 1.1096066236495972,
      "rewards/rejected": -1.1017345190048218,
      "step": 3021
    },
    {
      "epoch": 0.39545268667702627,
      "grad_norm": 6.173239707946777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23933963477611542,
      "logits/rejected": 0.35555359721183777,
      "logps/chosen": -340.517333984375,
      "logps/rejected": -431.2472229003906,
      "loss": 0.3917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2821537256240845,
      "rewards/margins": 0.8811959028244019,
      "rewards/rejected": -0.5990421772003174,
      "step": 3022
    },
    {
      "epoch": 0.395583544614378,
      "grad_norm": 6.88185977935791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08736049383878708,
      "logits/rejected": 0.28348663449287415,
      "logps/chosen": -389.62969970703125,
      "logps/rejected": -534.3095703125,
      "loss": 0.5556,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021652192808687687,
      "rewards/margins": 0.4931771755218506,
      "rewards/rejected": -0.49101200699806213,
      "step": 3023
    },
    {
      "epoch": 0.3957144025517298,
      "grad_norm": 5.704635143280029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07422664016485214,
      "logits/rejected": -0.20012888312339783,
      "logps/chosen": -569.0787353515625,
      "logps/rejected": -390.0260925292969,
      "loss": 0.4512,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2627410888671875,
      "rewards/margins": 0.8277392983436584,
      "rewards/rejected": -1.0904803276062012,
      "step": 3024
    },
    {
      "epoch": 0.39584526048908153,
      "grad_norm": 8.07716178894043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12572932243347168,
      "logits/rejected": 0.2548033595085144,
      "logps/chosen": -362.603515625,
      "logps/rejected": -327.9425964355469,
      "loss": 0.599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04349956288933754,
      "rewards/margins": 0.4349622130393982,
      "rewards/rejected": -0.3914627134799957,
      "step": 3025
    },
    {
      "epoch": 0.3959761184264333,
      "grad_norm": 10.220168113708496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013273227959871292,
      "logits/rejected": -0.10765858739614487,
      "logps/chosen": -610.6844482421875,
      "logps/rejected": -557.3994750976562,
      "loss": 0.9071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3471233546733856,
      "rewards/margins": 0.042939335107803345,
      "rewards/rejected": -0.39006274938583374,
      "step": 3026
    },
    {
      "epoch": 0.39610697636378506,
      "grad_norm": 5.97403621673584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08817964792251587,
      "logits/rejected": -0.022428300231695175,
      "logps/chosen": -526.2736206054688,
      "logps/rejected": -447.0058288574219,
      "loss": 0.5567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06840000301599503,
      "rewards/margins": 0.424979031085968,
      "rewards/rejected": -0.49337902665138245,
      "step": 3027
    },
    {
      "epoch": 0.39623783430113685,
      "grad_norm": 4.863459587097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21726083755493164,
      "logits/rejected": 0.0756387710571289,
      "logps/chosen": -378.7777099609375,
      "logps/rejected": -439.2608947753906,
      "loss": 0.3068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5097894668579102,
      "rewards/margins": 1.2813012599945068,
      "rewards/rejected": -0.7715119123458862,
      "step": 3028
    },
    {
      "epoch": 0.3963686922384886,
      "grad_norm": 5.430037021636963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016648218035697937,
      "logits/rejected": 0.061925966292619705,
      "logps/chosen": -332.57330322265625,
      "logps/rejected": -256.18792724609375,
      "loss": 0.4885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22431965172290802,
      "rewards/margins": 0.8612978458404541,
      "rewards/rejected": -0.6369782090187073,
      "step": 3029
    },
    {
      "epoch": 0.3964995501758404,
      "grad_norm": 10.921465873718262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01559237390756607,
      "logits/rejected": 0.20905420184135437,
      "logps/chosen": -628.864990234375,
      "logps/rejected": -517.1766967773438,
      "loss": 0.7735,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05186101049184799,
      "rewards/margins": 0.14711853861808777,
      "rewards/rejected": -0.19897958636283875,
      "step": 3030
    },
    {
      "epoch": 0.3966304081131921,
      "grad_norm": 7.9006524085998535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2711849510669708,
      "logits/rejected": 0.38320013880729675,
      "logps/chosen": -413.72216796875,
      "logps/rejected": -509.96759033203125,
      "loss": 0.5963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18514326214790344,
      "rewards/margins": 1.0851861238479614,
      "rewards/rejected": -0.900042712688446,
      "step": 3031
    },
    {
      "epoch": 0.3967612660505439,
      "grad_norm": 11.929176330566406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47401317954063416,
      "logits/rejected": 0.33881282806396484,
      "logps/chosen": -447.5972595214844,
      "logps/rejected": -452.34112548828125,
      "loss": 0.9782,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.032487474381923676,
      "rewards/margins": 0.40203744173049927,
      "rewards/rejected": -0.43452489376068115,
      "step": 3032
    },
    {
      "epoch": 0.39689212398789564,
      "grad_norm": 6.573888778686523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3321934640407562,
      "logits/rejected": -0.016299812123179436,
      "logps/chosen": -542.3673095703125,
      "logps/rejected": -425.5302734375,
      "loss": 0.357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3128635287284851,
      "rewards/margins": 1.3976112604141235,
      "rewards/rejected": -1.0847476720809937,
      "step": 3033
    },
    {
      "epoch": 0.3970229819252474,
      "grad_norm": 7.139347553253174,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26529619097709656,
      "logits/rejected": 0.24634738266468048,
      "logps/chosen": -767.30224609375,
      "logps/rejected": -664.6607666015625,
      "loss": 0.6046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07428018748760223,
      "rewards/margins": 0.7977432608604431,
      "rewards/rejected": -0.7234630584716797,
      "step": 3034
    },
    {
      "epoch": 0.39715383986259917,
      "grad_norm": 6.407281875610352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17434799671173096,
      "logits/rejected": 0.08943028002977371,
      "logps/chosen": -603.723876953125,
      "logps/rejected": -697.480712890625,
      "loss": 0.3777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.712297797203064,
      "rewards/margins": 0.929117739200592,
      "rewards/rejected": -0.21681997179985046,
      "step": 3035
    },
    {
      "epoch": 0.3972846977999509,
      "grad_norm": 8.467700958251953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13441449403762817,
      "logits/rejected": 0.07474419474601746,
      "logps/chosen": -593.1182861328125,
      "logps/rejected": -529.0287475585938,
      "loss": 0.6139,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18548297882080078,
      "rewards/margins": 0.41853198409080505,
      "rewards/rejected": -0.6040149927139282,
      "step": 3036
    },
    {
      "epoch": 0.3974155557373027,
      "grad_norm": 9.533400535583496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25386738777160645,
      "logits/rejected": -0.013605494052171707,
      "logps/chosen": -348.96380615234375,
      "logps/rejected": -371.3548889160156,
      "loss": 0.7301,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1437118500471115,
      "rewards/margins": 0.35418376326560974,
      "rewards/rejected": -0.49789559841156006,
      "step": 3037
    },
    {
      "epoch": 0.39754641367465443,
      "grad_norm": 6.798250675201416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08364575356245041,
      "logits/rejected": -0.05630367994308472,
      "logps/chosen": -601.2666015625,
      "logps/rejected": -515.132568359375,
      "loss": 0.4466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5768535733222961,
      "rewards/margins": 0.660971462726593,
      "rewards/rejected": -1.2378249168395996,
      "step": 3038
    },
    {
      "epoch": 0.3976772716120062,
      "grad_norm": 9.626830101013184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014656931161880493,
      "logits/rejected": 0.17568106949329376,
      "logps/chosen": -483.11474609375,
      "logps/rejected": -487.7542724609375,
      "loss": 0.7954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.377236932516098,
      "rewards/margins": 0.08235299587249756,
      "rewards/rejected": 0.29488393664360046,
      "step": 3039
    },
    {
      "epoch": 0.39780812954935796,
      "grad_norm": 5.39164924621582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14886793494224548,
      "logits/rejected": 0.13427242636680603,
      "logps/chosen": -452.0545654296875,
      "logps/rejected": -369.2583923339844,
      "loss": 0.3081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6851454973220825,
      "rewards/margins": 1.5177463293075562,
      "rewards/rejected": -0.8326007723808289,
      "step": 3040
    },
    {
      "epoch": 0.39793898748670975,
      "grad_norm": 6.39274787902832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22680361568927765,
      "logits/rejected": -0.07973629236221313,
      "logps/chosen": -475.5035705566406,
      "logps/rejected": -366.5989990234375,
      "loss": 0.563,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19587035477161407,
      "rewards/margins": 0.5549173355102539,
      "rewards/rejected": -0.7507876753807068,
      "step": 3041
    },
    {
      "epoch": 0.3980698454240615,
      "grad_norm": 8.840024948120117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0205699410289526,
      "logits/rejected": -0.06476577371358871,
      "logps/chosen": -297.4605407714844,
      "logps/rejected": -357.87738037109375,
      "loss": 0.792,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2414693981409073,
      "rewards/margins": 0.3511844277381897,
      "rewards/rejected": -0.5926538109779358,
      "step": 3042
    },
    {
      "epoch": 0.3982007033614133,
      "grad_norm": 4.5283355712890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.048450931906700134,
      "logits/rejected": -0.1274072825908661,
      "logps/chosen": -388.2633056640625,
      "logps/rejected": -350.659912109375,
      "loss": 0.336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25354841351509094,
      "rewards/margins": 1.2679634094238281,
      "rewards/rejected": -1.5215120315551758,
      "step": 3043
    },
    {
      "epoch": 0.398331561298765,
      "grad_norm": 8.03849983215332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2348743975162506,
      "logits/rejected": 0.00734725221991539,
      "logps/chosen": -440.5369567871094,
      "logps/rejected": -474.7143249511719,
      "loss": 0.4759,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5171346664428711,
      "rewards/margins": 0.780722975730896,
      "rewards/rejected": -1.297857642173767,
      "step": 3044
    },
    {
      "epoch": 0.3984624192361168,
      "grad_norm": 6.29823637008667,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18153896927833557,
      "logits/rejected": 0.17361582815647125,
      "logps/chosen": -517.9383544921875,
      "logps/rejected": -407.6835632324219,
      "loss": 0.4714,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6147377490997314,
      "rewards/margins": 1.0915530920028687,
      "rewards/rejected": -0.476815402507782,
      "step": 3045
    },
    {
      "epoch": 0.39859327717346854,
      "grad_norm": 5.8661580085754395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11965253204107285,
      "logits/rejected": 0.24329353868961334,
      "logps/chosen": -327.27117919921875,
      "logps/rejected": -345.4447021484375,
      "loss": 0.4566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1694362610578537,
      "rewards/margins": 0.7657020092010498,
      "rewards/rejected": -0.5962657928466797,
      "step": 3046
    },
    {
      "epoch": 0.39872413511082033,
      "grad_norm": 3.1316957473754883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21193884313106537,
      "logits/rejected": -0.0480581596493721,
      "logps/chosen": -428.8128662109375,
      "logps/rejected": -379.69384765625,
      "loss": 0.198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33616888523101807,
      "rewards/margins": 2.257528066635132,
      "rewards/rejected": -1.9213593006134033,
      "step": 3047
    },
    {
      "epoch": 0.39885499304817207,
      "grad_norm": 4.676414489746094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20718005299568176,
      "logits/rejected": 0.2345239520072937,
      "logps/chosen": -291.43115234375,
      "logps/rejected": -260.4724426269531,
      "loss": 0.4804,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11375121772289276,
      "rewards/margins": 0.6543037295341492,
      "rewards/rejected": -0.54055255651474,
      "step": 3048
    },
    {
      "epoch": 0.39898585098552386,
      "grad_norm": 6.8577561378479,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1218726634979248,
      "logits/rejected": -0.06211891770362854,
      "logps/chosen": -443.0775146484375,
      "logps/rejected": -502.8145751953125,
      "loss": 0.6417,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1374206691980362,
      "rewards/margins": 0.5810535550117493,
      "rewards/rejected": -0.44363290071487427,
      "step": 3049
    },
    {
      "epoch": 0.3991167089228756,
      "grad_norm": 6.867203712463379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05149661749601364,
      "logits/rejected": -0.07504585385322571,
      "logps/chosen": -533.0015869140625,
      "logps/rejected": -527.8096923828125,
      "loss": 0.4034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2837983965873718,
      "rewards/margins": 1.0902780294418335,
      "rewards/rejected": -0.8064796924591064,
      "step": 3050
    },
    {
      "epoch": 0.3992475668602274,
      "grad_norm": 11.047473907470703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08403107523918152,
      "logits/rejected": 0.13619525730609894,
      "logps/chosen": -418.493896484375,
      "logps/rejected": -449.2982482910156,
      "loss": 0.5965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2300170660018921,
      "rewards/margins": 0.7994648218154907,
      "rewards/rejected": -1.0294820070266724,
      "step": 3051
    },
    {
      "epoch": 0.3993784247975791,
      "grad_norm": 7.255786895751953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08353902399539948,
      "logits/rejected": 0.04788031056523323,
      "logps/chosen": -300.3598937988281,
      "logps/rejected": -373.4116516113281,
      "loss": 0.6554,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44404566287994385,
      "rewards/margins": 0.35294049978256226,
      "rewards/rejected": -0.7969861626625061,
      "step": 3052
    },
    {
      "epoch": 0.3995092827349309,
      "grad_norm": 7.321617603302002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10481473803520203,
      "logits/rejected": -0.023642122745513916,
      "logps/chosen": -575.4267578125,
      "logps/rejected": -494.590087890625,
      "loss": 0.5017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18648947775363922,
      "rewards/margins": 0.9721061587333679,
      "rewards/rejected": -1.1585956811904907,
      "step": 3053
    },
    {
      "epoch": 0.39964014067228265,
      "grad_norm": 10.783211708068848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27406927943229675,
      "logits/rejected": 0.3347322642803192,
      "logps/chosen": -529.264892578125,
      "logps/rejected": -512.7304077148438,
      "loss": 0.6369,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.30467912554740906,
      "rewards/margins": 0.25856441259384155,
      "rewards/rejected": 0.046114712953567505,
      "step": 3054
    },
    {
      "epoch": 0.39977099860963444,
      "grad_norm": 5.123461723327637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1282528191804886,
      "logits/rejected": 0.1495899111032486,
      "logps/chosen": -307.13909912109375,
      "logps/rejected": -308.5092468261719,
      "loss": 0.6552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28771018981933594,
      "rewards/margins": 0.3986692726612091,
      "rewards/rejected": -0.6863794326782227,
      "step": 3055
    },
    {
      "epoch": 0.3999018565469862,
      "grad_norm": 6.424281120300293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009223612025380135,
      "logits/rejected": 0.08665795624256134,
      "logps/chosen": -372.4748229980469,
      "logps/rejected": -492.352294921875,
      "loss": 0.4821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.020070448517799377,
      "rewards/margins": 1.1496038436889648,
      "rewards/rejected": -1.1696743965148926,
      "step": 3056
    },
    {
      "epoch": 0.4000327144843379,
      "grad_norm": 8.7846040725708,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04709856957197189,
      "logits/rejected": 0.017503637820482254,
      "logps/chosen": -440.50732421875,
      "logps/rejected": -367.1931457519531,
      "loss": 0.7265,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06074202060699463,
      "rewards/margins": 0.2114511877298355,
      "rewards/rejected": -0.15070916712284088,
      "step": 3057
    },
    {
      "epoch": 0.4001635724216897,
      "grad_norm": 12.2597017288208,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11574137210845947,
      "logits/rejected": 0.053740911185741425,
      "logps/chosen": -429.3750305175781,
      "logps/rejected": -562.7786865234375,
      "loss": 1.0698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8568515777587891,
      "rewards/margins": -0.172489196062088,
      "rewards/rejected": -0.6843624114990234,
      "step": 3058
    },
    {
      "epoch": 0.40029443035904144,
      "grad_norm": 6.299788951873779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08850109577178955,
      "logits/rejected": 0.09422355890274048,
      "logps/chosen": -395.47393798828125,
      "logps/rejected": -353.1568908691406,
      "loss": 0.4265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7278763055801392,
      "rewards/margins": 1.0281084775924683,
      "rewards/rejected": -0.3002321422100067,
      "step": 3059
    },
    {
      "epoch": 0.40042528829639323,
      "grad_norm": 12.58056640625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07025806605815887,
      "logits/rejected": -0.08331745862960815,
      "logps/chosen": -670.4840087890625,
      "logps/rejected": -337.64697265625,
      "loss": 0.9526,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8357763290405273,
      "rewards/margins": -0.144974485039711,
      "rewards/rejected": -0.6908018589019775,
      "step": 3060
    },
    {
      "epoch": 0.40055614623374497,
      "grad_norm": 4.690723896026611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30466705560684204,
      "logits/rejected": 0.15280301868915558,
      "logps/chosen": -262.10400390625,
      "logps/rejected": -355.4829406738281,
      "loss": 0.3174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1755020022392273,
      "rewards/margins": 1.3443548679351807,
      "rewards/rejected": -1.1688528060913086,
      "step": 3061
    },
    {
      "epoch": 0.40068700417109676,
      "grad_norm": 5.8814921379089355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4884336292743683,
      "logits/rejected": 0.4509350061416626,
      "logps/chosen": -501.599365234375,
      "logps/rejected": -388.9664001464844,
      "loss": 0.4845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5561839938163757,
      "rewards/margins": 0.797825813293457,
      "rewards/rejected": -0.2416418045759201,
      "step": 3062
    },
    {
      "epoch": 0.4008178621084485,
      "grad_norm": 6.070905685424805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3511100113391876,
      "logits/rejected": 0.056970320641994476,
      "logps/chosen": -371.49078369140625,
      "logps/rejected": -293.74688720703125,
      "loss": 0.5208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.031296730041503906,
      "rewards/margins": 0.5812219977378845,
      "rewards/rejected": -0.5499252080917358,
      "step": 3063
    },
    {
      "epoch": 0.4009487200458003,
      "grad_norm": 7.118423938751221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16338279843330383,
      "logits/rejected": -0.10418623685836792,
      "logps/chosen": -487.57086181640625,
      "logps/rejected": -517.0696411132812,
      "loss": 0.4262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5901539921760559,
      "rewards/margins": 1.401000738143921,
      "rewards/rejected": -0.8108468055725098,
      "step": 3064
    },
    {
      "epoch": 0.401079577983152,
      "grad_norm": 6.932277679443359,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2255708873271942,
      "logits/rejected": 0.23259659111499786,
      "logps/chosen": -334.29461669921875,
      "logps/rejected": -324.01788330078125,
      "loss": 0.4248,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17554055154323578,
      "rewards/margins": 0.9221591949462891,
      "rewards/rejected": -1.097699761390686,
      "step": 3065
    },
    {
      "epoch": 0.4012104359205038,
      "grad_norm": 8.950695037841797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1468210071325302,
      "logits/rejected": 0.21971893310546875,
      "logps/chosen": -327.6667785644531,
      "logps/rejected": -365.8623962402344,
      "loss": 0.7723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2946975529193878,
      "rewards/margins": 0.027501679956912994,
      "rewards/rejected": -0.322199285030365,
      "step": 3066
    },
    {
      "epoch": 0.40134129385785555,
      "grad_norm": 6.685145378112793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024377737194299698,
      "logits/rejected": -0.327787846326828,
      "logps/chosen": -637.5233154296875,
      "logps/rejected": -653.6156616210938,
      "loss": 0.58,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1991443634033203,
      "rewards/margins": 0.6603189706802368,
      "rewards/rejected": -0.8594633936882019,
      "step": 3067
    },
    {
      "epoch": 0.40147215179520734,
      "grad_norm": 7.825528144836426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13460931181907654,
      "logits/rejected": 0.3518279790878296,
      "logps/chosen": -538.7079467773438,
      "logps/rejected": -594.1799926757812,
      "loss": 0.6327,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.268344908952713,
      "rewards/margins": 0.4934428334236145,
      "rewards/rejected": -0.22509783506393433,
      "step": 3068
    },
    {
      "epoch": 0.4016030097325591,
      "grad_norm": 9.223732948303223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23916850984096527,
      "logits/rejected": 0.37818634510040283,
      "logps/chosen": -581.7857055664062,
      "logps/rejected": -579.9689331054688,
      "loss": 0.55,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2246776670217514,
      "rewards/margins": 1.2814743518829346,
      "rewards/rejected": -1.056796669960022,
      "step": 3069
    },
    {
      "epoch": 0.40173386766991087,
      "grad_norm": 4.329435348510742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07724285125732422,
      "logits/rejected": -0.09251749515533447,
      "logps/chosen": -299.5339660644531,
      "logps/rejected": -370.7127380371094,
      "loss": 0.3334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07070942223072052,
      "rewards/margins": 1.2464405298233032,
      "rewards/rejected": -1.1757310628890991,
      "step": 3070
    },
    {
      "epoch": 0.4018647256072626,
      "grad_norm": 7.1934661865234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0691031739115715,
      "logits/rejected": 0.023157479241490364,
      "logps/chosen": -445.0057067871094,
      "logps/rejected": -528.40673828125,
      "loss": 0.4896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13783109188079834,
      "rewards/margins": 0.7470936179161072,
      "rewards/rejected": -0.8849247694015503,
      "step": 3071
    },
    {
      "epoch": 0.4019955835446144,
      "grad_norm": 9.042387008666992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04394274950027466,
      "logits/rejected": 0.06571729481220245,
      "logps/chosen": -390.1300964355469,
      "logps/rejected": -392.8175964355469,
      "loss": 0.7251,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30158138275146484,
      "rewards/margins": 0.5558722019195557,
      "rewards/rejected": -0.8574535250663757,
      "step": 3072
    },
    {
      "epoch": 0.40212644148196613,
      "grad_norm": 5.640019416809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12337881326675415,
      "logits/rejected": -0.18669936060905457,
      "logps/chosen": -442.2486877441406,
      "logps/rejected": -395.62139892578125,
      "loss": 0.3957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34807708859443665,
      "rewards/margins": 1.225746750831604,
      "rewards/rejected": -1.5738236904144287,
      "step": 3073
    },
    {
      "epoch": 0.4022572994193179,
      "grad_norm": 5.556097984313965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11862503737211227,
      "logits/rejected": 0.19530393183231354,
      "logps/chosen": -368.22784423828125,
      "logps/rejected": -348.9012451171875,
      "loss": 0.5445,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16048505902290344,
      "rewards/margins": 0.5514101386070251,
      "rewards/rejected": -0.7118951678276062,
      "step": 3074
    },
    {
      "epoch": 0.40238815735666966,
      "grad_norm": 5.906919002532959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2950748801231384,
      "logits/rejected": -0.09700228273868561,
      "logps/chosen": -255.43344116210938,
      "logps/rejected": -390.98223876953125,
      "loss": 0.4457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5979462265968323,
      "rewards/margins": 0.7970836758613586,
      "rewards/rejected": -1.3950297832489014,
      "step": 3075
    },
    {
      "epoch": 0.40251901529402145,
      "grad_norm": 5.675334930419922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08856060355901718,
      "logits/rejected": -0.041619330644607544,
      "logps/chosen": -359.0693664550781,
      "logps/rejected": -434.14617919921875,
      "loss": 0.4357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24010160565376282,
      "rewards/margins": 0.7418357729911804,
      "rewards/rejected": -0.50173419713974,
      "step": 3076
    },
    {
      "epoch": 0.4026498732313732,
      "grad_norm": 5.576182842254639,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09633299708366394,
      "logits/rejected": -0.16541779041290283,
      "logps/chosen": -379.0213928222656,
      "logps/rejected": -427.96954345703125,
      "loss": 0.4112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3691883087158203,
      "rewards/margins": 1.0156739950180054,
      "rewards/rejected": -1.3848624229431152,
      "step": 3077
    },
    {
      "epoch": 0.4027807311687249,
      "grad_norm": 7.466772079467773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13990239799022675,
      "logits/rejected": 0.0969802662730217,
      "logps/chosen": -402.7763366699219,
      "logps/rejected": -628.9140014648438,
      "loss": 0.4459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4418569803237915,
      "rewards/margins": 1.0217559337615967,
      "rewards/rejected": -1.4636129140853882,
      "step": 3078
    },
    {
      "epoch": 0.4029115891060767,
      "grad_norm": 13.328071594238281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2042338252067566,
      "logits/rejected": 0.1438070833683014,
      "logps/chosen": -650.090576171875,
      "logps/rejected": -552.5745849609375,
      "loss": 1.1891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5845500826835632,
      "rewards/margins": -0.4855802059173584,
      "rewards/rejected": -0.09896983951330185,
      "step": 3079
    },
    {
      "epoch": 0.40304244704342845,
      "grad_norm": 9.399911880493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11019881069660187,
      "logits/rejected": 0.1153591126203537,
      "logps/chosen": -319.8307800292969,
      "logps/rejected": -367.0140686035156,
      "loss": 0.8613,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.648518443107605,
      "rewards/margins": 0.0037892162799835205,
      "rewards/rejected": -0.6523076891899109,
      "step": 3080
    },
    {
      "epoch": 0.40317330498078024,
      "grad_norm": 4.60029935836792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16182634234428406,
      "logits/rejected": 0.11747847497463226,
      "logps/chosen": -488.8291931152344,
      "logps/rejected": -557.4826049804688,
      "loss": 0.2348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7620843648910522,
      "rewards/margins": 1.985186219215393,
      "rewards/rejected": -1.2231019735336304,
      "step": 3081
    },
    {
      "epoch": 0.403304162918132,
      "grad_norm": 10.982223510742188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13601762056350708,
      "logits/rejected": -0.3405779004096985,
      "logps/chosen": -326.025146484375,
      "logps/rejected": -314.7627258300781,
      "loss": 0.9269,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.66259765625,
      "rewards/margins": -0.3146803379058838,
      "rewards/rejected": -0.347917377948761,
      "step": 3082
    },
    {
      "epoch": 0.4034350208554838,
      "grad_norm": 7.680248737335205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31483328342437744,
      "logits/rejected": 0.3228033185005188,
      "logps/chosen": -488.3758850097656,
      "logps/rejected": -373.2441101074219,
      "loss": 0.6032,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6164864301681519,
      "rewards/margins": 0.38439446687698364,
      "rewards/rejected": -1.0008808374404907,
      "step": 3083
    },
    {
      "epoch": 0.4035658787928355,
      "grad_norm": 9.715668678283691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15753072500228882,
      "logits/rejected": 0.02250683307647705,
      "logps/chosen": -382.2381896972656,
      "logps/rejected": -370.898193359375,
      "loss": 0.5412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44800475239753723,
      "rewards/margins": 0.7097283601760864,
      "rewards/rejected": -1.1577332019805908,
      "step": 3084
    },
    {
      "epoch": 0.4036967367301873,
      "grad_norm": 7.91621208190918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032499559223651886,
      "logits/rejected": -0.04093474894762039,
      "logps/chosen": -298.7803955078125,
      "logps/rejected": -379.7119140625,
      "loss": 0.8596,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7863819599151611,
      "rewards/margins": 0.16052722930908203,
      "rewards/rejected": -0.9469091296195984,
      "step": 3085
    },
    {
      "epoch": 0.40382759466753904,
      "grad_norm": 11.10334587097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20557770133018494,
      "logits/rejected": 0.06394688785076141,
      "logps/chosen": -370.7341003417969,
      "logps/rejected": -507.9759521484375,
      "loss": 0.7697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8599034547805786,
      "rewards/margins": 0.01458456739783287,
      "rewards/rejected": -0.8744879961013794,
      "step": 3086
    },
    {
      "epoch": 0.4039584526048908,
      "grad_norm": 10.29881763458252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26279520988464355,
      "logits/rejected": 0.09434224665164948,
      "logps/chosen": -391.7845458984375,
      "logps/rejected": -327.3070983886719,
      "loss": 0.9944,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04534127935767174,
      "rewards/margins": -0.33412623405456543,
      "rewards/rejected": 0.28878501057624817,
      "step": 3087
    },
    {
      "epoch": 0.40408931054224256,
      "grad_norm": 7.067723274230957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07194165140390396,
      "logits/rejected": 0.12721151113510132,
      "logps/chosen": -458.3282165527344,
      "logps/rejected": -442.0726318359375,
      "loss": 0.4742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6090335845947266,
      "rewards/margins": 0.9308841824531555,
      "rewards/rejected": -0.32185062766075134,
      "step": 3088
    },
    {
      "epoch": 0.40422016847959436,
      "grad_norm": 6.948576927185059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12126833200454712,
      "logits/rejected": 0.10022779554128647,
      "logps/chosen": -433.17742919921875,
      "logps/rejected": -463.1789855957031,
      "loss": 0.6256,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08399391174316406,
      "rewards/margins": 0.297284871339798,
      "rewards/rejected": -0.2132909595966339,
      "step": 3089
    },
    {
      "epoch": 0.4043510264169461,
      "grad_norm": 6.765669345855713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06505763530731201,
      "logits/rejected": -0.15437448024749756,
      "logps/chosen": -424.38116455078125,
      "logps/rejected": -419.75347900390625,
      "loss": 0.6706,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19068469107151031,
      "rewards/margins": 0.39843446016311646,
      "rewards/rejected": -0.5891191959381104,
      "step": 3090
    },
    {
      "epoch": 0.4044818843542979,
      "grad_norm": 5.962151527404785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4111369848251343,
      "logits/rejected": -0.009559817612171173,
      "logps/chosen": -427.4696350097656,
      "logps/rejected": -405.253173828125,
      "loss": 0.3565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23057882487773895,
      "rewards/margins": 1.2654153108596802,
      "rewards/rejected": -1.0348365306854248,
      "step": 3091
    },
    {
      "epoch": 0.4046127422916496,
      "grad_norm": 6.755467891693115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22324077785015106,
      "logits/rejected": -0.36385875940322876,
      "logps/chosen": -376.423828125,
      "logps/rejected": -340.13433837890625,
      "loss": 0.5292,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0772154331207275,
      "rewards/margins": 0.4443582594394684,
      "rewards/rejected": -1.5215736627578735,
      "step": 3092
    },
    {
      "epoch": 0.4047436002290014,
      "grad_norm": 6.261269569396973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00353066623210907,
      "logits/rejected": -0.09292827546596527,
      "logps/chosen": -509.91827392578125,
      "logps/rejected": -345.8126525878906,
      "loss": 0.4922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004314044490456581,
      "rewards/margins": 0.6425890326499939,
      "rewards/rejected": -0.6382750272750854,
      "step": 3093
    },
    {
      "epoch": 0.40487445816635315,
      "grad_norm": 6.659707546234131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10040336847305298,
      "logits/rejected": -0.19650623202323914,
      "logps/chosen": -392.29168701171875,
      "logps/rejected": -546.57958984375,
      "loss": 0.474,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18809816241264343,
      "rewards/margins": 1.5135349035263062,
      "rewards/rejected": -1.3254367113113403,
      "step": 3094
    },
    {
      "epoch": 0.40500531610370494,
      "grad_norm": 8.431777954101562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27515748143196106,
      "logits/rejected": 0.2228717803955078,
      "logps/chosen": -418.2166748046875,
      "logps/rejected": -353.7315673828125,
      "loss": 0.7151,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9309369325637817,
      "rewards/margins": 0.17654502391815186,
      "rewards/rejected": -1.1074819564819336,
      "step": 3095
    },
    {
      "epoch": 0.4051361740410567,
      "grad_norm": 6.869602203369141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05241845175623894,
      "logits/rejected": -0.18099062144756317,
      "logps/chosen": -465.29168701171875,
      "logps/rejected": -491.1624450683594,
      "loss": 0.3547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7329893112182617,
      "rewards/margins": 1.3177976608276367,
      "rewards/rejected": -0.584808349609375,
      "step": 3096
    },
    {
      "epoch": 0.40526703197840847,
      "grad_norm": 4.711813449859619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15612472593784332,
      "logits/rejected": -0.0678018257021904,
      "logps/chosen": -445.3114929199219,
      "logps/rejected": -493.5946960449219,
      "loss": 0.4382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.30459046363830566,
      "rewards/margins": 0.7809815406799316,
      "rewards/rejected": -0.4763910472393036,
      "step": 3097
    },
    {
      "epoch": 0.4053978899157602,
      "grad_norm": 4.913710117340088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08734728395938873,
      "logits/rejected": -0.16804169118404388,
      "logps/chosen": -329.53460693359375,
      "logps/rejected": -496.42926025390625,
      "loss": 0.3196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15766221284866333,
      "rewards/margins": 1.3184552192687988,
      "rewards/rejected": -1.4761172533035278,
      "step": 3098
    },
    {
      "epoch": 0.405528747853112,
      "grad_norm": 8.112130165100098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1679721474647522,
      "logits/rejected": 8.344650268554688e-05,
      "logps/chosen": -511.6783447265625,
      "logps/rejected": -468.95098876953125,
      "loss": 0.6653,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20716381072998047,
      "rewards/margins": 0.6556735038757324,
      "rewards/rejected": -0.4485096335411072,
      "step": 3099
    },
    {
      "epoch": 0.40565960579046373,
      "grad_norm": 4.403753757476807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.328002393245697,
      "logits/rejected": -0.28454968333244324,
      "logps/chosen": -411.9668884277344,
      "logps/rejected": -404.1482238769531,
      "loss": 0.378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6701920032501221,
      "rewards/margins": 1.1026887893676758,
      "rewards/rejected": -1.7728809118270874,
      "step": 3100
    },
    {
      "epoch": 0.40579046372781546,
      "grad_norm": 6.514216423034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13931912183761597,
      "logits/rejected": -0.06676406413316727,
      "logps/chosen": -429.8907470703125,
      "logps/rejected": -430.6733703613281,
      "loss": 0.4539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18336737155914307,
      "rewards/margins": 0.7787070870399475,
      "rewards/rejected": -0.5953397750854492,
      "step": 3101
    },
    {
      "epoch": 0.40592132166516726,
      "grad_norm": 5.1548752784729,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07669742405414581,
      "logits/rejected": 0.014036450535058975,
      "logps/chosen": -398.2020263671875,
      "logps/rejected": -454.2166748046875,
      "loss": 0.4014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2819167971611023,
      "rewards/margins": 1.3496662378311157,
      "rewards/rejected": -1.0677493810653687,
      "step": 3102
    },
    {
      "epoch": 0.406052179602519,
      "grad_norm": 8.445584297180176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17654015123844147,
      "logits/rejected": -0.42304080724716187,
      "logps/chosen": -539.875732421875,
      "logps/rejected": -420.07171630859375,
      "loss": 0.5617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16527310013771057,
      "rewards/margins": 0.982806384563446,
      "rewards/rejected": -0.8175333142280579,
      "step": 3103
    },
    {
      "epoch": 0.4061830375398708,
      "grad_norm": 8.05920696258545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19505125284194946,
      "logits/rejected": 0.14198525249958038,
      "logps/chosen": -375.1852722167969,
      "logps/rejected": -378.1476745605469,
      "loss": 0.5505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3235718011856079,
      "rewards/margins": 1.108438491821289,
      "rewards/rejected": -1.432010293006897,
      "step": 3104
    },
    {
      "epoch": 0.4063138954772225,
      "grad_norm": 13.088916778564453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3980597257614136,
      "logits/rejected": -0.4350152611732483,
      "logps/chosen": -549.201416015625,
      "logps/rejected": -425.681884765625,
      "loss": 1.2465,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.2340646982192993,
      "rewards/margins": -0.5464624166488647,
      "rewards/rejected": -0.6876022219657898,
      "step": 3105
    },
    {
      "epoch": 0.4064447534145743,
      "grad_norm": 4.811102867126465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0040262620896101,
      "logits/rejected": 0.07311782985925674,
      "logps/chosen": -327.3968200683594,
      "logps/rejected": -287.9056091308594,
      "loss": 0.3556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3542726933956146,
      "rewards/margins": 0.9887357354164124,
      "rewards/rejected": -0.6344631314277649,
      "step": 3106
    },
    {
      "epoch": 0.40657561135192605,
      "grad_norm": 6.412411689758301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06848659366369247,
      "logits/rejected": 0.0623391792178154,
      "logps/chosen": -543.91064453125,
      "logps/rejected": -487.2916564941406,
      "loss": 0.4266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6418390274047852,
      "rewards/margins": 0.9065589904785156,
      "rewards/rejected": -0.26471999287605286,
      "step": 3107
    },
    {
      "epoch": 0.40670646928927784,
      "grad_norm": 4.904505729675293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12685410678386688,
      "logits/rejected": 0.34198808670043945,
      "logps/chosen": -505.1456604003906,
      "logps/rejected": -478.73822021484375,
      "loss": 0.3449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10694122314453125,
      "rewards/margins": 1.1708874702453613,
      "rewards/rejected": -1.0639461278915405,
      "step": 3108
    },
    {
      "epoch": 0.4068373272266296,
      "grad_norm": 7.926639556884766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3766297996044159,
      "logits/rejected": 0.6565239429473877,
      "logps/chosen": -536.681396484375,
      "logps/rejected": -500.9192199707031,
      "loss": 0.6454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09705352783203125,
      "rewards/margins": 0.38505667448043823,
      "rewards/rejected": -0.288003146648407,
      "step": 3109
    },
    {
      "epoch": 0.40696818516398137,
      "grad_norm": 6.0215888023376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27564141154289246,
      "logits/rejected": 0.30709752440452576,
      "logps/chosen": -401.77984619140625,
      "logps/rejected": -489.78216552734375,
      "loss": 0.4327,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08060243725776672,
      "rewards/margins": 1.0159826278686523,
      "rewards/rejected": -1.0965851545333862,
      "step": 3110
    },
    {
      "epoch": 0.4070990431013331,
      "grad_norm": 6.389066219329834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3532402217388153,
      "logits/rejected": 0.43826672434806824,
      "logps/chosen": -448.69097900390625,
      "logps/rejected": -445.3725891113281,
      "loss": 0.4155,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.40553802251815796,
      "rewards/margins": 1.0407882928848267,
      "rewards/rejected": -0.6352502703666687,
      "step": 3111
    },
    {
      "epoch": 0.4072299010386849,
      "grad_norm": 5.626826286315918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40600964426994324,
      "logits/rejected": 0.16074013710021973,
      "logps/chosen": -488.89630126953125,
      "logps/rejected": -462.3720703125,
      "loss": 0.4777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24823206663131714,
      "rewards/margins": 1.098841667175293,
      "rewards/rejected": -0.8506096005439758,
      "step": 3112
    },
    {
      "epoch": 0.40736075897603663,
      "grad_norm": 5.282608509063721,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018935158848762512,
      "logits/rejected": -0.037165317684412,
      "logps/chosen": -337.3885498046875,
      "logps/rejected": -440.9960021972656,
      "loss": 0.4227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4299337565898895,
      "rewards/margins": 0.7294431924819946,
      "rewards/rejected": -0.2995094656944275,
      "step": 3113
    },
    {
      "epoch": 0.4074916169133884,
      "grad_norm": 12.412226676940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022582441568374634,
      "logits/rejected": 0.2995947003364563,
      "logps/chosen": -476.65057373046875,
      "logps/rejected": -608.98681640625,
      "loss": 0.437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.343170166015625,
      "rewards/margins": 0.9184737205505371,
      "rewards/rejected": -1.2616437673568726,
      "step": 3114
    },
    {
      "epoch": 0.40762247485074016,
      "grad_norm": 6.6900224685668945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23544728755950928,
      "logits/rejected": -0.17612236738204956,
      "logps/chosen": -619.2334594726562,
      "logps/rejected": -496.71875,
      "loss": 0.418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.500270664691925,
      "rewards/margins": 1.1668146848678589,
      "rewards/rejected": -0.6665439605712891,
      "step": 3115
    },
    {
      "epoch": 0.40775333278809195,
      "grad_norm": 7.723912239074707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12026797980070114,
      "logits/rejected": 0.2063610702753067,
      "logps/chosen": -489.39117431640625,
      "logps/rejected": -617.8568115234375,
      "loss": 0.6662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5139938592910767,
      "rewards/margins": 0.45114099979400635,
      "rewards/rejected": -0.965134859085083,
      "step": 3116
    },
    {
      "epoch": 0.4078841907254437,
      "grad_norm": 8.510641098022461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3429211974143982,
      "logits/rejected": 0.2736341953277588,
      "logps/chosen": -517.0195922851562,
      "logps/rejected": -400.0214538574219,
      "loss": 0.7149,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15810392796993256,
      "rewards/margins": 0.2777872085571289,
      "rewards/rejected": -0.11968326568603516,
      "step": 3117
    },
    {
      "epoch": 0.4080150486627955,
      "grad_norm": 2.853809118270874,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28157275915145874,
      "logits/rejected": 0.06455706804990768,
      "logps/chosen": -399.3087463378906,
      "logps/rejected": -349.91302490234375,
      "loss": 0.2052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7119877338409424,
      "rewards/margins": 1.6899501085281372,
      "rewards/rejected": -0.9779623746871948,
      "step": 3118
    },
    {
      "epoch": 0.4081459066001472,
      "grad_norm": 7.2031073570251465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29895928502082825,
      "logits/rejected": 0.2631451189517975,
      "logps/chosen": -504.9177551269531,
      "logps/rejected": -585.7015380859375,
      "loss": 0.5971,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002185821533203125,
      "rewards/margins": 0.5766785740852356,
      "rewards/rejected": -0.5788644552230835,
      "step": 3119
    },
    {
      "epoch": 0.408276764537499,
      "grad_norm": 8.159869194030762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16627466678619385,
      "logits/rejected": 0.08171810209751129,
      "logps/chosen": -255.95089721679688,
      "logps/rejected": -248.90786743164062,
      "loss": 0.9148,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9535771012306213,
      "rewards/margins": -0.11426573991775513,
      "rewards/rejected": -0.8393113613128662,
      "step": 3120
    },
    {
      "epoch": 0.40840762247485074,
      "grad_norm": 7.7652587890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13964539766311646,
      "logits/rejected": 0.2010388970375061,
      "logps/chosen": -379.05230712890625,
      "logps/rejected": -467.3240051269531,
      "loss": 0.6057,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.358803927898407,
      "rewards/margins": 0.6335521340370178,
      "rewards/rejected": -0.27474820613861084,
      "step": 3121
    },
    {
      "epoch": 0.4085384804122025,
      "grad_norm": 5.25404167175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.403969407081604,
      "logits/rejected": -0.4948444366455078,
      "logps/chosen": -348.08135986328125,
      "logps/rejected": -306.255615234375,
      "loss": 0.4758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2763441205024719,
      "rewards/margins": 0.7789273262023926,
      "rewards/rejected": -1.0552713871002197,
      "step": 3122
    },
    {
      "epoch": 0.40866933834955427,
      "grad_norm": 5.248465538024902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013494174927473068,
      "logits/rejected": 0.20803195238113403,
      "logps/chosen": -602.5241088867188,
      "logps/rejected": -543.8965454101562,
      "loss": 0.3057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1436021775007248,
      "rewards/margins": 1.3770761489868164,
      "rewards/rejected": -1.520678162574768,
      "step": 3123
    },
    {
      "epoch": 0.408800196286906,
      "grad_norm": 13.999773025512695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31262582540512085,
      "logits/rejected": 0.20049695670604706,
      "logps/chosen": -625.5565185546875,
      "logps/rejected": -450.2573547363281,
      "loss": 0.5573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1068125069141388,
      "rewards/margins": 0.9626007080078125,
      "rewards/rejected": -0.8557882308959961,
      "step": 3124
    },
    {
      "epoch": 0.4089310542242578,
      "grad_norm": 6.959291458129883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3803255259990692,
      "logits/rejected": 0.19119007885456085,
      "logps/chosen": -457.6494140625,
      "logps/rejected": -344.49200439453125,
      "loss": 0.5851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.048670947551727295,
      "rewards/margins": 0.48888978362083435,
      "rewards/rejected": -0.4402187764644623,
      "step": 3125
    },
    {
      "epoch": 0.40906191216160953,
      "grad_norm": 5.277037620544434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008286073803901672,
      "logits/rejected": -0.06976056098937988,
      "logps/chosen": -429.0348815917969,
      "logps/rejected": -437.1148376464844,
      "loss": 0.3802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2880842387676239,
      "rewards/margins": 1.1613104343414307,
      "rewards/rejected": -0.8732261657714844,
      "step": 3126
    },
    {
      "epoch": 0.4091927700989613,
      "grad_norm": 3.286132574081421,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12774738669395447,
      "logits/rejected": 0.16484250128269196,
      "logps/chosen": -463.75372314453125,
      "logps/rejected": -447.0313415527344,
      "loss": 0.18,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5956882238388062,
      "rewards/margins": 2.8156445026397705,
      "rewards/rejected": -2.219956159591675,
      "step": 3127
    },
    {
      "epoch": 0.40932362803631306,
      "grad_norm": 6.644717693328857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18363845348358154,
      "logits/rejected": 0.03986669331789017,
      "logps/chosen": -701.0383911132812,
      "logps/rejected": -516.7503051757812,
      "loss": 0.3783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29396480321884155,
      "rewards/margins": 1.0505503416061401,
      "rewards/rejected": -0.7565855979919434,
      "step": 3128
    },
    {
      "epoch": 0.40945448597366485,
      "grad_norm": 9.13740062713623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03729957342147827,
      "logits/rejected": 0.23046278953552246,
      "logps/chosen": -502.62359619140625,
      "logps/rejected": -300.3050842285156,
      "loss": 0.817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5819726586341858,
      "rewards/margins": 0.4713088870048523,
      "rewards/rejected": -1.0532816648483276,
      "step": 3129
    },
    {
      "epoch": 0.4095853439110166,
      "grad_norm": 3.4181196689605713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02967546321451664,
      "logits/rejected": -0.09515458345413208,
      "logps/chosen": -536.2725219726562,
      "logps/rejected": -458.913330078125,
      "loss": 0.2403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46622219681739807,
      "rewards/margins": 1.96013343334198,
      "rewards/rejected": -1.4939110279083252,
      "step": 3130
    },
    {
      "epoch": 0.4097162018483684,
      "grad_norm": 7.718165397644043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02321309596300125,
      "logits/rejected": 0.1349002867937088,
      "logps/chosen": -508.2270812988281,
      "logps/rejected": -556.6121215820312,
      "loss": 0.4559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.63015216588974,
      "rewards/margins": 1.1283107995986938,
      "rewards/rejected": -1.7584630250930786,
      "step": 3131
    },
    {
      "epoch": 0.4098470597857201,
      "grad_norm": 10.954889297485352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1864784061908722,
      "logits/rejected": 0.10213479399681091,
      "logps/chosen": -402.3744201660156,
      "logps/rejected": -357.6960144042969,
      "loss": 0.9549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6234356164932251,
      "rewards/margins": 0.07401435822248459,
      "rewards/rejected": -0.6974498629570007,
      "step": 3132
    },
    {
      "epoch": 0.4099779177230719,
      "grad_norm": 4.998575687408447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1624232530593872,
      "logits/rejected": 0.001446438953280449,
      "logps/chosen": -351.7728271484375,
      "logps/rejected": -358.3999938964844,
      "loss": 0.2871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07247466593980789,
      "rewards/margins": 1.5436897277832031,
      "rewards/rejected": -1.4712151288986206,
      "step": 3133
    },
    {
      "epoch": 0.41010877566042364,
      "grad_norm": 6.433107376098633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016357876360416412,
      "logits/rejected": 0.05227728188037872,
      "logps/chosen": -310.09149169921875,
      "logps/rejected": -347.78973388671875,
      "loss": 0.6411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6132000088691711,
      "rewards/margins": 0.13283884525299072,
      "rewards/rejected": -0.7460388541221619,
      "step": 3134
    },
    {
      "epoch": 0.41023963359777543,
      "grad_norm": 10.61417293548584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11361487209796906,
      "logits/rejected": 0.17925229668617249,
      "logps/chosen": -434.6683654785156,
      "logps/rejected": -532.1293334960938,
      "loss": 0.7538,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7311621308326721,
      "rewards/margins": 0.11240222305059433,
      "rewards/rejected": -0.8435642719268799,
      "step": 3135
    },
    {
      "epoch": 0.41037049153512717,
      "grad_norm": 5.625344276428223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01629827916622162,
      "logits/rejected": -0.23557832837104797,
      "logps/chosen": -378.96923828125,
      "logps/rejected": -411.226318359375,
      "loss": 0.3073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.249712735414505,
      "rewards/margins": 1.1720365285873413,
      "rewards/rejected": -1.4217493534088135,
      "step": 3136
    },
    {
      "epoch": 0.41050134947247896,
      "grad_norm": 9.025646209716797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07378458231687546,
      "logits/rejected": 0.13408485054969788,
      "logps/chosen": -445.5666198730469,
      "logps/rejected": -453.54876708984375,
      "loss": 0.7575,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16620387136936188,
      "rewards/margins": 0.5613514184951782,
      "rewards/rejected": -0.7275552749633789,
      "step": 3137
    },
    {
      "epoch": 0.4106322074098307,
      "grad_norm": 8.759160041809082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015955738723278046,
      "logits/rejected": 0.12183878570795059,
      "logps/chosen": -266.3664855957031,
      "logps/rejected": -351.6243896484375,
      "loss": 0.8002,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.001333236694336,
      "rewards/margins": 0.38405609130859375,
      "rewards/rejected": -1.3853893280029297,
      "step": 3138
    },
    {
      "epoch": 0.4107630653471825,
      "grad_norm": 8.54309368133545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10379847139120102,
      "logits/rejected": -0.08860068023204803,
      "logps/chosen": -541.7811279296875,
      "logps/rejected": -761.1246337890625,
      "loss": 0.589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5573858618736267,
      "rewards/margins": 0.4971103072166443,
      "rewards/rejected": -1.0544962882995605,
      "step": 3139
    },
    {
      "epoch": 0.4108939232845342,
      "grad_norm": 9.039139747619629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18758633732795715,
      "logits/rejected": -0.10330255329608917,
      "logps/chosen": -482.18255615234375,
      "logps/rejected": -467.6607360839844,
      "loss": 0.6946,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7060229182243347,
      "rewards/margins": 0.384327232837677,
      "rewards/rejected": -1.0903502702713013,
      "step": 3140
    },
    {
      "epoch": 0.411024781221886,
      "grad_norm": 11.536593437194824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3306991159915924,
      "logits/rejected": -0.27863332629203796,
      "logps/chosen": -415.1461486816406,
      "logps/rejected": -399.45977783203125,
      "loss": 0.7964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7775234580039978,
      "rewards/margins": 0.7382717132568359,
      "rewards/rejected": -1.5157952308654785,
      "step": 3141
    },
    {
      "epoch": 0.41115563915923775,
      "grad_norm": 5.012216091156006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1351386457681656,
      "logits/rejected": -0.22126404941082,
      "logps/chosen": -277.7993469238281,
      "logps/rejected": -296.7856750488281,
      "loss": 0.3792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30209046602249146,
      "rewards/margins": 1.1698182821273804,
      "rewards/rejected": -1.4719088077545166,
      "step": 3142
    },
    {
      "epoch": 0.4112864970965895,
      "grad_norm": 6.520636081695557,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1866350919008255,
      "logits/rejected": -0.11407731473445892,
      "logps/chosen": -485.090087890625,
      "logps/rejected": -468.42340087890625,
      "loss": 0.5489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19444063305854797,
      "rewards/margins": 0.8982462286949158,
      "rewards/rejected": -1.0926868915557861,
      "step": 3143
    },
    {
      "epoch": 0.4114173550339413,
      "grad_norm": 11.186304092407227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0006411708891391754,
      "logits/rejected": 0.09487172961235046,
      "logps/chosen": -538.0095825195312,
      "logps/rejected": -442.9157409667969,
      "loss": 0.5817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20038911700248718,
      "rewards/margins": 0.9782558679580688,
      "rewards/rejected": -1.1786448955535889,
      "step": 3144
    },
    {
      "epoch": 0.411548212971293,
      "grad_norm": 5.567758083343506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1384129375219345,
      "logits/rejected": -0.07704327255487442,
      "logps/chosen": -501.0607604980469,
      "logps/rejected": -338.7513122558594,
      "loss": 0.442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3935199975967407,
      "rewards/margins": 1.241539478302002,
      "rewards/rejected": -0.8480193614959717,
      "step": 3145
    },
    {
      "epoch": 0.4116790709086448,
      "grad_norm": 6.058106899261475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16124656796455383,
      "logits/rejected": 0.05588994175195694,
      "logps/chosen": -509.09722900390625,
      "logps/rejected": -475.1608581542969,
      "loss": 0.4172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17979204654693604,
      "rewards/margins": 1.1026414632797241,
      "rewards/rejected": -0.9228495359420776,
      "step": 3146
    },
    {
      "epoch": 0.41180992884599654,
      "grad_norm": 9.624638557434082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09136337041854858,
      "logits/rejected": 0.07914957404136658,
      "logps/chosen": -375.09100341796875,
      "logps/rejected": -449.16754150390625,
      "loss": 0.5871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.026050478219985962,
      "rewards/margins": 0.8547317981719971,
      "rewards/rejected": -0.8807823657989502,
      "step": 3147
    },
    {
      "epoch": 0.41194078678334833,
      "grad_norm": 6.298135757446289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2648053765296936,
      "logits/rejected": 0.2637348175048828,
      "logps/chosen": -372.8288879394531,
      "logps/rejected": -293.18505859375,
      "loss": 0.3628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5971990823745728,
      "rewards/margins": 1.417767882347107,
      "rewards/rejected": -0.8205686807632446,
      "step": 3148
    },
    {
      "epoch": 0.41207164472070007,
      "grad_norm": 7.1739115715026855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05817725136876106,
      "logits/rejected": 0.08184105902910233,
      "logps/chosen": -297.9808349609375,
      "logps/rejected": -259.45599365234375,
      "loss": 0.6422,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.29947298765182495,
      "rewards/margins": 0.4725662171840668,
      "rewards/rejected": -0.7720392346382141,
      "step": 3149
    },
    {
      "epoch": 0.41220250265805186,
      "grad_norm": 4.722691535949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009341253899037838,
      "logits/rejected": -0.1302761733531952,
      "logps/chosen": -494.9100341796875,
      "logps/rejected": -373.1924133300781,
      "loss": 0.2757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7597233057022095,
      "rewards/margins": 1.6085988283157349,
      "rewards/rejected": -0.8488755822181702,
      "step": 3150
    },
    {
      "epoch": 0.4123333605954036,
      "grad_norm": 4.835753917694092,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11655199527740479,
      "logits/rejected": 0.05196909233927727,
      "logps/chosen": -470.5604248046875,
      "logps/rejected": -481.70367431640625,
      "loss": 0.2849,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.32530444860458374,
      "rewards/margins": 1.6365480422973633,
      "rewards/rejected": -1.3112434148788452,
      "step": 3151
    },
    {
      "epoch": 0.4124642185327554,
      "grad_norm": 4.55446195602417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05202746018767357,
      "logits/rejected": -0.02326459437608719,
      "logps/chosen": -450.7324523925781,
      "logps/rejected": -455.7050476074219,
      "loss": 0.3757,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4008532166481018,
      "rewards/margins": 1.5351985692977905,
      "rewards/rejected": -1.134345293045044,
      "step": 3152
    },
    {
      "epoch": 0.4125950764701071,
      "grad_norm": 5.850094318389893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16644342243671417,
      "logits/rejected": 0.14309339225292206,
      "logps/chosen": -377.11077880859375,
      "logps/rejected": -411.9199523925781,
      "loss": 0.3731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2669523358345032,
      "rewards/margins": 1.3419950008392334,
      "rewards/rejected": -1.0750428438186646,
      "step": 3153
    },
    {
      "epoch": 0.4127259344074589,
      "grad_norm": 10.290587425231934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03596052899956703,
      "logits/rejected": -0.1692763715982437,
      "logps/chosen": -482.9290466308594,
      "logps/rejected": -491.3252868652344,
      "loss": 1.0111,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24948003888130188,
      "rewards/margins": -0.1987510621547699,
      "rewards/rejected": -0.050728991627693176,
      "step": 3154
    },
    {
      "epoch": 0.41285679234481065,
      "grad_norm": 5.055084228515625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16439403593540192,
      "logits/rejected": -0.09362468868494034,
      "logps/chosen": -492.7795104980469,
      "logps/rejected": -538.0195922851562,
      "loss": 0.3402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5651514530181885,
      "rewards/margins": 1.0211763381958008,
      "rewards/rejected": -0.4560249149799347,
      "step": 3155
    },
    {
      "epoch": 0.41298765028216244,
      "grad_norm": 3.5760228633880615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06254041939973831,
      "logits/rejected": -0.06666209548711777,
      "logps/chosen": -553.2913818359375,
      "logps/rejected": -416.9221496582031,
      "loss": 0.2856,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9412296414375305,
      "rewards/margins": 1.819422721862793,
      "rewards/rejected": -0.8781930804252625,
      "step": 3156
    },
    {
      "epoch": 0.4131185082195142,
      "grad_norm": 12.767125129699707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18671518564224243,
      "logits/rejected": 0.15693840384483337,
      "logps/chosen": -369.93511962890625,
      "logps/rejected": -367.8780822753906,
      "loss": 0.9213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3578792214393616,
      "rewards/margins": 0.36694955825805664,
      "rewards/rejected": -0.724828839302063,
      "step": 3157
    },
    {
      "epoch": 0.41324936615686597,
      "grad_norm": 5.744237422943115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08333180844783783,
      "logits/rejected": 0.20945340394973755,
      "logps/chosen": -503.189453125,
      "logps/rejected": -503.4391784667969,
      "loss": 0.3919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3200662434101105,
      "rewards/margins": 1.0661835670471191,
      "rewards/rejected": -1.3862498998641968,
      "step": 3158
    },
    {
      "epoch": 0.4133802240942177,
      "grad_norm": 9.135340690612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13991962373256683,
      "logits/rejected": 0.022015612572431564,
      "logps/chosen": -459.184814453125,
      "logps/rejected": -419.88360595703125,
      "loss": 0.6775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44384461641311646,
      "rewards/margins": 0.5286407470703125,
      "rewards/rejected": -0.9724854230880737,
      "step": 3159
    },
    {
      "epoch": 0.4135110820315695,
      "grad_norm": 9.71886920928955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1673484891653061,
      "logits/rejected": -0.16162261366844177,
      "logps/chosen": -302.7934875488281,
      "logps/rejected": -346.5307312011719,
      "loss": 1.149,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1292792558670044,
      "rewards/margins": -0.3227540850639343,
      "rewards/rejected": -0.8065251111984253,
      "step": 3160
    },
    {
      "epoch": 0.41364193996892124,
      "grad_norm": 7.663440227508545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1566484421491623,
      "logits/rejected": 0.17916417121887207,
      "logps/chosen": -294.52069091796875,
      "logps/rejected": -411.4878234863281,
      "loss": 0.8215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2718774676322937,
      "rewards/margins": 1.1234029531478882,
      "rewards/rejected": -1.3952804803848267,
      "step": 3161
    },
    {
      "epoch": 0.413772797906273,
      "grad_norm": 6.174917221069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0967286229133606,
      "logits/rejected": -0.18457360565662384,
      "logps/chosen": -423.6571044921875,
      "logps/rejected": -414.4960021972656,
      "loss": 0.376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3564714789390564,
      "rewards/margins": 1.439773440361023,
      "rewards/rejected": -1.0833020210266113,
      "step": 3162
    },
    {
      "epoch": 0.41390365584362476,
      "grad_norm": 5.559939861297607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03817521035671234,
      "logits/rejected": 0.18193204700946808,
      "logps/chosen": -492.42987060546875,
      "logps/rejected": -491.0621337890625,
      "loss": 0.3852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13728316128253937,
      "rewards/margins": 1.1761207580566406,
      "rewards/rejected": -1.313403844833374,
      "step": 3163
    },
    {
      "epoch": 0.41403451378097655,
      "grad_norm": 6.818889141082764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06747588515281677,
      "logits/rejected": -0.07651864737272263,
      "logps/chosen": -458.33270263671875,
      "logps/rejected": -376.9881896972656,
      "loss": 0.5272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1423753798007965,
      "rewards/margins": 0.6384323835372925,
      "rewards/rejected": -0.4960569143295288,
      "step": 3164
    },
    {
      "epoch": 0.4141653717183283,
      "grad_norm": 5.424581527709961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030056452378630638,
      "logits/rejected": -0.08643145114183426,
      "logps/chosen": -449.7184143066406,
      "logps/rejected": -446.9866638183594,
      "loss": 0.2938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35746026039123535,
      "rewards/margins": 1.3261806964874268,
      "rewards/rejected": -1.683640956878662,
      "step": 3165
    },
    {
      "epoch": 0.41429622965568,
      "grad_norm": 7.010410308837891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05148763954639435,
      "logits/rejected": 0.0646599531173706,
      "logps/chosen": -501.3701171875,
      "logps/rejected": -608.6483764648438,
      "loss": 0.5218,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5304247140884399,
      "rewards/margins": 1.1997044086456299,
      "rewards/rejected": -0.6692796945571899,
      "step": 3166
    },
    {
      "epoch": 0.4144270875930318,
      "grad_norm": 9.28195571899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0538254976272583,
      "logits/rejected": 0.03837715461850166,
      "logps/chosen": -443.4571228027344,
      "logps/rejected": -438.3258972167969,
      "loss": 0.5357,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5880455374717712,
      "rewards/margins": 0.8366447687149048,
      "rewards/rejected": -1.4246902465820312,
      "step": 3167
    },
    {
      "epoch": 0.41455794553038355,
      "grad_norm": 8.010848045349121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20144912600517273,
      "logits/rejected": 0.20615461468696594,
      "logps/chosen": -801.2142333984375,
      "logps/rejected": -416.2420654296875,
      "loss": 0.5983,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19659100472927094,
      "rewards/margins": 0.719753623008728,
      "rewards/rejected": -0.9163446426391602,
      "step": 3168
    },
    {
      "epoch": 0.41468880346773535,
      "grad_norm": 6.861983299255371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26928746700286865,
      "logits/rejected": 0.2526978850364685,
      "logps/chosen": -548.95849609375,
      "logps/rejected": -409.6680603027344,
      "loss": 0.6327,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.19690951704978943,
      "rewards/margins": 0.7086236476898193,
      "rewards/rejected": -0.5117141604423523,
      "step": 3169
    },
    {
      "epoch": 0.4148196614050871,
      "grad_norm": 14.487859725952148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21181945502758026,
      "logits/rejected": -0.19781792163848877,
      "logps/chosen": -509.5242614746094,
      "logps/rejected": -428.47613525390625,
      "loss": 0.6227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4166585206985474,
      "rewards/margins": 0.7079431414604187,
      "rewards/rejected": -2.1246016025543213,
      "step": 3170
    },
    {
      "epoch": 0.4149505193424389,
      "grad_norm": 6.009631156921387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2694244384765625,
      "logits/rejected": 0.09930656105279922,
      "logps/chosen": -497.3956298828125,
      "logps/rejected": -424.8789978027344,
      "loss": 0.3798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.41985055804252625,
      "rewards/margins": 1.2508161067962646,
      "rewards/rejected": -0.8309654593467712,
      "step": 3171
    },
    {
      "epoch": 0.4150813772797906,
      "grad_norm": 8.330102920532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10047618299722672,
      "logits/rejected": -0.04560364782810211,
      "logps/chosen": -470.3789367675781,
      "logps/rejected": -436.1410827636719,
      "loss": 0.8051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7621210217475891,
      "rewards/margins": 0.04024313762784004,
      "rewards/rejected": -0.8023641705513,
      "step": 3172
    },
    {
      "epoch": 0.4152122352171424,
      "grad_norm": 5.01138973236084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08592045307159424,
      "logits/rejected": -0.024698561057448387,
      "logps/chosen": -412.8450927734375,
      "logps/rejected": -443.7431640625,
      "loss": 0.3461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5500719547271729,
      "rewards/margins": 1.6283841133117676,
      "rewards/rejected": -1.0783121585845947,
      "step": 3173
    },
    {
      "epoch": 0.41534309315449414,
      "grad_norm": 3.7328848838806152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09786112606525421,
      "logits/rejected": -0.02674916386604309,
      "logps/chosen": -340.28570556640625,
      "logps/rejected": -275.97412109375,
      "loss": 0.4448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19603881239891052,
      "rewards/margins": 0.7195051312446594,
      "rewards/rejected": -0.9155439138412476,
      "step": 3174
    },
    {
      "epoch": 0.41547395109184593,
      "grad_norm": 7.127553462982178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15792642533779144,
      "logits/rejected": -0.11032892018556595,
      "logps/chosen": -447.4134216308594,
      "logps/rejected": -383.69342041015625,
      "loss": 0.6223,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9356914162635803,
      "rewards/margins": 0.2653219401836395,
      "rewards/rejected": -1.2010133266448975,
      "step": 3175
    },
    {
      "epoch": 0.41560480902919766,
      "grad_norm": 7.7332234382629395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1448848992586136,
      "logits/rejected": 0.03240814059972763,
      "logps/chosen": -455.48223876953125,
      "logps/rejected": -490.5292053222656,
      "loss": 0.5385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7530497908592224,
      "rewards/margins": 1.0200916528701782,
      "rewards/rejected": -1.773141622543335,
      "step": 3176
    },
    {
      "epoch": 0.41573566696654946,
      "grad_norm": 5.851843357086182,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25817862153053284,
      "logits/rejected": -0.06187453866004944,
      "logps/chosen": -533.3822021484375,
      "logps/rejected": -374.4745788574219,
      "loss": 0.4239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.41109657287597656,
      "rewards/margins": 0.9991678595542908,
      "rewards/rejected": -0.5880712270736694,
      "step": 3177
    },
    {
      "epoch": 0.4158665249039012,
      "grad_norm": 7.461389541625977,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2553773522377014,
      "logits/rejected": -0.18036164343357086,
      "logps/chosen": -640.4107666015625,
      "logps/rejected": -405.5396728515625,
      "loss": 0.4833,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23929710686206818,
      "rewards/margins": 1.326440453529358,
      "rewards/rejected": -1.0871433019638062,
      "step": 3178
    },
    {
      "epoch": 0.415997382841253,
      "grad_norm": 4.7711358070373535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08599153906106949,
      "logits/rejected": 0.1683880239725113,
      "logps/chosen": -715.4877319335938,
      "logps/rejected": -497.4799499511719,
      "loss": 0.3117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42540159821510315,
      "rewards/margins": 1.4871920347213745,
      "rewards/rejected": -1.0617904663085938,
      "step": 3179
    },
    {
      "epoch": 0.4161282407786047,
      "grad_norm": 5.352427959442139,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1448233723640442,
      "logits/rejected": 0.1646910160779953,
      "logps/chosen": -415.3024597167969,
      "logps/rejected": -443.79437255859375,
      "loss": 0.4488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6294345259666443,
      "rewards/margins": 1.0614227056503296,
      "rewards/rejected": -0.43198826909065247,
      "step": 3180
    },
    {
      "epoch": 0.4162590987159565,
      "grad_norm": 6.527711391448975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2654329538345337,
      "logits/rejected": -0.024831699207425117,
      "logps/chosen": -413.48309326171875,
      "logps/rejected": -307.11236572265625,
      "loss": 0.4447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20218105614185333,
      "rewards/margins": 1.3138502836227417,
      "rewards/rejected": -1.1116691827774048,
      "step": 3181
    },
    {
      "epoch": 0.41638995665330825,
      "grad_norm": 4.934072494506836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.054197873920202255,
      "logits/rejected": -0.029002554714679718,
      "logps/chosen": -465.47760009765625,
      "logps/rejected": -368.50848388671875,
      "loss": 0.2763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19448091089725494,
      "rewards/margins": 1.4373185634613037,
      "rewards/rejected": -1.631799340248108,
      "step": 3182
    },
    {
      "epoch": 0.41652081459066004,
      "grad_norm": 7.367011547088623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23935453593730927,
      "logits/rejected": -0.21493463218212128,
      "logps/chosen": -254.4317626953125,
      "logps/rejected": -306.3756103515625,
      "loss": 0.7581,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6580308675765991,
      "rewards/margins": 0.2135048806667328,
      "rewards/rejected": -0.8715357780456543,
      "step": 3183
    },
    {
      "epoch": 0.4166516725280118,
      "grad_norm": 10.753327369689941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1272190660238266,
      "logits/rejected": -0.30854466557502747,
      "logps/chosen": -461.1568298339844,
      "logps/rejected": -379.9476318359375,
      "loss": 0.8302,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6950713396072388,
      "rewards/margins": 0.25317448377609253,
      "rewards/rejected": -0.9482458233833313,
      "step": 3184
    },
    {
      "epoch": 0.41678253046536357,
      "grad_norm": 9.050809860229492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14841917157173157,
      "logits/rejected": 0.10531506687402725,
      "logps/chosen": -354.774169921875,
      "logps/rejected": -416.90716552734375,
      "loss": 0.8511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0763460248708725,
      "rewards/margins": 0.08458080142736435,
      "rewards/rejected": -0.008234771899878979,
      "step": 3185
    },
    {
      "epoch": 0.4169133884027153,
      "grad_norm": 7.053791046142578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17014634609222412,
      "logits/rejected": 0.20146462321281433,
      "logps/chosen": -752.691650390625,
      "logps/rejected": -696.4532470703125,
      "loss": 0.5391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3902992606163025,
      "rewards/margins": 0.701118528842926,
      "rewards/rejected": -0.31081920862197876,
      "step": 3186
    },
    {
      "epoch": 0.41704424634006704,
      "grad_norm": 8.98698902130127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08615411818027496,
      "logits/rejected": 0.033125318586826324,
      "logps/chosen": -632.5635375976562,
      "logps/rejected": -410.1694030761719,
      "loss": 0.5825,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.092058777809143,
      "rewards/margins": 0.6829021573066711,
      "rewards/rejected": -1.774960994720459,
      "step": 3187
    },
    {
      "epoch": 0.41717510427741883,
      "grad_norm": 7.749507427215576,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06416843831539154,
      "logits/rejected": 0.17140714824199677,
      "logps/chosen": -435.7442626953125,
      "logps/rejected": -380.739013671875,
      "loss": 0.6054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08363684266805649,
      "rewards/margins": 0.4443691372871399,
      "rewards/rejected": -0.5280059576034546,
      "step": 3188
    },
    {
      "epoch": 0.41730596221477056,
      "grad_norm": 4.91145133972168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4445333778858185,
      "logits/rejected": 0.2865239381790161,
      "logps/chosen": -510.6302795410156,
      "logps/rejected": -625.1661376953125,
      "loss": 0.2843,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09378661215305328,
      "rewards/margins": 2.0057969093322754,
      "rewards/rejected": -1.9120104312896729,
      "step": 3189
    },
    {
      "epoch": 0.41743682015212236,
      "grad_norm": 6.884687900543213,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14489835500717163,
      "logits/rejected": 0.12359629571437836,
      "logps/chosen": -686.3173217773438,
      "logps/rejected": -698.701171875,
      "loss": 0.4808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6271640658378601,
      "rewards/margins": 1.050420880317688,
      "rewards/rejected": -0.42325684428215027,
      "step": 3190
    },
    {
      "epoch": 0.4175676780894741,
      "grad_norm": 5.800294876098633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1896919310092926,
      "logits/rejected": 0.25713396072387695,
      "logps/chosen": -561.2318725585938,
      "logps/rejected": -504.3206787109375,
      "loss": 0.4378,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9693251252174377,
      "rewards/margins": 1.676071047782898,
      "rewards/rejected": -0.7067459225654602,
      "step": 3191
    },
    {
      "epoch": 0.4176985360268259,
      "grad_norm": 6.8753437995910645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1440768837928772,
      "logits/rejected": -0.017675355076789856,
      "logps/chosen": -463.00274658203125,
      "logps/rejected": -397.9108581542969,
      "loss": 0.4999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5115882754325867,
      "rewards/margins": 0.899500846862793,
      "rewards/rejected": -0.3879125714302063,
      "step": 3192
    },
    {
      "epoch": 0.4178293939641776,
      "grad_norm": 5.7036871910095215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0627334862947464,
      "logits/rejected": -0.16035789251327515,
      "logps/chosen": -450.7323913574219,
      "logps/rejected": -408.9765625,
      "loss": 0.5095,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.611380934715271,
      "rewards/margins": 0.9328523874282837,
      "rewards/rejected": -1.5442333221435547,
      "step": 3193
    },
    {
      "epoch": 0.4179602519015294,
      "grad_norm": 2.7161238193511963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05123334750533104,
      "logits/rejected": -0.20479780435562134,
      "logps/chosen": -480.9820251464844,
      "logps/rejected": -423.1581115722656,
      "loss": 0.1828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5192234516143799,
      "rewards/margins": 1.9016880989074707,
      "rewards/rejected": -1.3824646472930908,
      "step": 3194
    },
    {
      "epoch": 0.41809110983888115,
      "grad_norm": 8.012208938598633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15176154673099518,
      "logits/rejected": -0.17542657256126404,
      "logps/chosen": -431.0983581542969,
      "logps/rejected": -462.3016357421875,
      "loss": 0.4482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08587265014648438,
      "rewards/margins": 1.0486036539077759,
      "rewards/rejected": -0.9627309441566467,
      "step": 3195
    },
    {
      "epoch": 0.41822196777623294,
      "grad_norm": 5.425885200500488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3138459324836731,
      "logits/rejected": 0.33298635482788086,
      "logps/chosen": -406.86883544921875,
      "logps/rejected": -389.586669921875,
      "loss": 0.4619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21808432042598724,
      "rewards/margins": 0.7928665280342102,
      "rewards/rejected": -0.5747821927070618,
      "step": 3196
    },
    {
      "epoch": 0.4183528257135847,
      "grad_norm": 7.085920333862305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026159102097153664,
      "logits/rejected": 0.029533157125115395,
      "logps/chosen": -478.9877014160156,
      "logps/rejected": -549.3331909179688,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.758697509765625,
      "rewards/margins": 0.6849134564399719,
      "rewards/rejected": 0.07378409057855606,
      "step": 3197
    },
    {
      "epoch": 0.41848368365093647,
      "grad_norm": 9.771587371826172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0890614315867424,
      "logits/rejected": -0.08495430648326874,
      "logps/chosen": -469.88232421875,
      "logps/rejected": -658.6600341796875,
      "loss": 0.3162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1632072627544403,
      "rewards/margins": 1.8063156604766846,
      "rewards/rejected": -1.9695229530334473,
      "step": 3198
    },
    {
      "epoch": 0.4186145415882882,
      "grad_norm": 9.29471206665039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14305689930915833,
      "logits/rejected": -0.10614882409572601,
      "logps/chosen": -409.6324157714844,
      "logps/rejected": -429.1226806640625,
      "loss": 0.7394,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.23428094387054443,
      "rewards/rejected": -0.29900264739990234,
      "step": 3199
    },
    {
      "epoch": 0.41874539952564,
      "grad_norm": 10.057168006896973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0018549710512161255,
      "logits/rejected": 0.2525882124900818,
      "logps/chosen": -537.0853271484375,
      "logps/rejected": -501.5331726074219,
      "loss": 0.8063,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09619777649641037,
      "rewards/margins": 0.2984004318714142,
      "rewards/rejected": -0.39459818601608276,
      "step": 3200
    },
    {
      "epoch": 0.41887625746299173,
      "grad_norm": 5.679637908935547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10072900354862213,
      "logits/rejected": -0.02705548331141472,
      "logps/chosen": -370.7014465332031,
      "logps/rejected": -653.6432495117188,
      "loss": 0.2976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9600234031677246,
      "rewards/margins": 1.3481173515319824,
      "rewards/rejected": -2.308140516281128,
      "step": 3201
    },
    {
      "epoch": 0.4190071154003435,
      "grad_norm": 7.272526264190674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18447870016098022,
      "logits/rejected": 0.16364732384681702,
      "logps/chosen": -371.8384704589844,
      "logps/rejected": -513.2196044921875,
      "loss": 0.5466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8241980075836182,
      "rewards/margins": 0.6203117966651917,
      "rewards/rejected": -1.444509744644165,
      "step": 3202
    },
    {
      "epoch": 0.41913797333769526,
      "grad_norm": 3.5998778343200684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23485863208770752,
      "logits/rejected": 0.3208996653556824,
      "logps/chosen": -385.8546447753906,
      "logps/rejected": -502.1180419921875,
      "loss": 0.3335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5835509896278381,
      "rewards/margins": 1.3935072422027588,
      "rewards/rejected": -0.8099561929702759,
      "step": 3203
    },
    {
      "epoch": 0.41926883127504705,
      "grad_norm": 9.155332565307617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3130454123020172,
      "logits/rejected": 0.3535311222076416,
      "logps/chosen": -477.2173156738281,
      "logps/rejected": -485.6532897949219,
      "loss": 0.6565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27794820070266724,
      "rewards/margins": 0.9292826056480408,
      "rewards/rejected": -1.2072306871414185,
      "step": 3204
    },
    {
      "epoch": 0.4193996892123988,
      "grad_norm": 9.341755867004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10146720707416534,
      "logits/rejected": -0.043171048164367676,
      "logps/chosen": -406.58685302734375,
      "logps/rejected": -563.025634765625,
      "loss": 0.8456,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32838118076324463,
      "rewards/margins": 0.25774502754211426,
      "rewards/rejected": -0.5861260890960693,
      "step": 3205
    },
    {
      "epoch": 0.4195305471497506,
      "grad_norm": 7.1193695068359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03351515904068947,
      "logits/rejected": 0.02342173084616661,
      "logps/chosen": -425.1763000488281,
      "logps/rejected": -497.9607849121094,
      "loss": 0.5641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5667300224304199,
      "rewards/margins": 0.9836493730545044,
      "rewards/rejected": -0.41691938042640686,
      "step": 3206
    },
    {
      "epoch": 0.4196614050871023,
      "grad_norm": 5.1245808601379395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09988653659820557,
      "logits/rejected": 0.056172993034124374,
      "logps/chosen": -312.6209716796875,
      "logps/rejected": -353.8775939941406,
      "loss": 0.3555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20366419851779938,
      "rewards/margins": 0.9934623837471008,
      "rewards/rejected": -0.7897982001304626,
      "step": 3207
    },
    {
      "epoch": 0.4197922630244541,
      "grad_norm": 6.218186378479004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08680329471826553,
      "logits/rejected": 0.24394722282886505,
      "logps/chosen": -407.0408935546875,
      "logps/rejected": -427.0705261230469,
      "loss": 0.566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2589109539985657,
      "rewards/margins": 0.6385878920555115,
      "rewards/rejected": -0.8974989056587219,
      "step": 3208
    },
    {
      "epoch": 0.41992312096180584,
      "grad_norm": 5.362277030944824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12511606514453888,
      "logits/rejected": 0.11504101753234863,
      "logps/chosen": -684.3535766601562,
      "logps/rejected": -655.5361938476562,
      "loss": 0.3989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3558501899242401,
      "rewards/margins": 1.517262578010559,
      "rewards/rejected": -1.8731127977371216,
      "step": 3209
    },
    {
      "epoch": 0.4200539788991576,
      "grad_norm": 4.312793731689453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1438412368297577,
      "logits/rejected": -0.2343997210264206,
      "logps/chosen": -506.4960632324219,
      "logps/rejected": -424.90777587890625,
      "loss": 0.27,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.020039765164256096,
      "rewards/margins": 1.5924501419067383,
      "rewards/rejected": -1.572410225868225,
      "step": 3210
    },
    {
      "epoch": 0.42018483683650937,
      "grad_norm": 5.9454240798950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11341366916894913,
      "logits/rejected": 0.10040857642889023,
      "logps/chosen": -470.09674072265625,
      "logps/rejected": -463.0347595214844,
      "loss": 0.3676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13191911578178406,
      "rewards/margins": 1.230898380279541,
      "rewards/rejected": -1.0989792346954346,
      "step": 3211
    },
    {
      "epoch": 0.4203156947738611,
      "grad_norm": 6.861956596374512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1471196711063385,
      "logits/rejected": 0.023633792996406555,
      "logps/chosen": -358.49664306640625,
      "logps/rejected": -364.7154235839844,
      "loss": 0.3975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08528216183185577,
      "rewards/margins": 0.9768930673599243,
      "rewards/rejected": -0.8916109204292297,
      "step": 3212
    },
    {
      "epoch": 0.4204465527112129,
      "grad_norm": 6.824094772338867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27486544847488403,
      "logits/rejected": 0.14393998682498932,
      "logps/chosen": -651.7308349609375,
      "logps/rejected": -565.7982177734375,
      "loss": 0.4539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9096189737319946,
      "rewards/margins": 1.3163948059082031,
      "rewards/rejected": -2.226013660430908,
      "step": 3213
    },
    {
      "epoch": 0.42057741064856463,
      "grad_norm": 7.724137783050537,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0896252691745758,
      "logits/rejected": 0.05647147819399834,
      "logps/chosen": -346.9552001953125,
      "logps/rejected": -287.7359313964844,
      "loss": 0.6971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.35429075360298157,
      "rewards/margins": 0.375301718711853,
      "rewards/rejected": -0.7295925617218018,
      "step": 3214
    },
    {
      "epoch": 0.4207082685859164,
      "grad_norm": 8.36942195892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4336649179458618,
      "logits/rejected": -0.3031366169452667,
      "logps/chosen": -507.2978820800781,
      "logps/rejected": -435.49395751953125,
      "loss": 0.6496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08951303362846375,
      "rewards/margins": 0.5932598114013672,
      "rewards/rejected": -0.6827728748321533,
      "step": 3215
    },
    {
      "epoch": 0.42083912652326816,
      "grad_norm": 7.345964431762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2548668682575226,
      "logits/rejected": 0.11537203192710876,
      "logps/chosen": -441.27899169921875,
      "logps/rejected": -426.9295349121094,
      "loss": 0.5295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2654802203178406,
      "rewards/margins": 0.8527662754058838,
      "rewards/rejected": -1.1182465553283691,
      "step": 3216
    },
    {
      "epoch": 0.42096998446061995,
      "grad_norm": 4.30106782913208,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16325826942920685,
      "logits/rejected": 0.21136412024497986,
      "logps/chosen": -333.21612548828125,
      "logps/rejected": -380.2314147949219,
      "loss": 0.3039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32417309284210205,
      "rewards/margins": 1.4362231492996216,
      "rewards/rejected": -1.7603962421417236,
      "step": 3217
    },
    {
      "epoch": 0.4211008423979717,
      "grad_norm": 6.599152088165283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2826728820800781,
      "logits/rejected": 0.2534458041191101,
      "logps/chosen": -313.7621154785156,
      "logps/rejected": -422.1671142578125,
      "loss": 0.6406,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13144397735595703,
      "rewards/margins": 0.4660078287124634,
      "rewards/rejected": -0.5974518060684204,
      "step": 3218
    },
    {
      "epoch": 0.4212317003353235,
      "grad_norm": 10.755022048950195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08318331837654114,
      "logits/rejected": 0.0422155037522316,
      "logps/chosen": -353.1758117675781,
      "logps/rejected": -297.8984680175781,
      "loss": 0.5514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42636266350746155,
      "rewards/margins": 0.8367900252342224,
      "rewards/rejected": -1.2631527185440063,
      "step": 3219
    },
    {
      "epoch": 0.4213625582726752,
      "grad_norm": 12.419088363647461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028373707085847855,
      "logits/rejected": 0.06851662695407867,
      "logps/chosen": -444.584228515625,
      "logps/rejected": -483.7269287109375,
      "loss": 0.9987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6573745608329773,
      "rewards/margins": 0.23189204931259155,
      "rewards/rejected": -0.8892666697502136,
      "step": 3220
    },
    {
      "epoch": 0.421493416210027,
      "grad_norm": 4.392005920410156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5404680371284485,
      "logits/rejected": 0.43165266513824463,
      "logps/chosen": -405.39056396484375,
      "logps/rejected": -549.135009765625,
      "loss": 0.2456,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19772568345069885,
      "rewards/margins": 1.895781397819519,
      "rewards/rejected": -2.0935070514678955,
      "step": 3221
    },
    {
      "epoch": 0.42162427414737874,
      "grad_norm": 9.062796592712402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1531735062599182,
      "logits/rejected": -0.2844708263874054,
      "logps/chosen": -379.3337707519531,
      "logps/rejected": -409.5015563964844,
      "loss": 0.9034,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1712031364440918,
      "rewards/margins": -0.09217682480812073,
      "rewards/rejected": -1.079026222229004,
      "step": 3222
    },
    {
      "epoch": 0.42175513208473053,
      "grad_norm": 8.18172550201416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3339671492576599,
      "logits/rejected": 0.20111781358718872,
      "logps/chosen": -518.125732421875,
      "logps/rejected": -326.6209716796875,
      "loss": 0.5867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21371807157993317,
      "rewards/margins": 0.912222683429718,
      "rewards/rejected": -1.1259406805038452,
      "step": 3223
    },
    {
      "epoch": 0.42188599002208227,
      "grad_norm": 6.841318130493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2502900958061218,
      "logits/rejected": -0.05639837682247162,
      "logps/chosen": -454.1722717285156,
      "logps/rejected": -556.3336791992188,
      "loss": 0.3975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.767565131187439,
      "rewards/margins": 1.161359429359436,
      "rewards/rejected": -1.9289246797561646,
      "step": 3224
    },
    {
      "epoch": 0.42201684795943406,
      "grad_norm": 7.080258846282959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21259447932243347,
      "logits/rejected": -0.059933386743068695,
      "logps/chosen": -384.552001953125,
      "logps/rejected": -401.4980163574219,
      "loss": 0.4924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5431776642799377,
      "rewards/margins": 0.999779224395752,
      "rewards/rejected": -1.5429567098617554,
      "step": 3225
    },
    {
      "epoch": 0.4221477058967858,
      "grad_norm": 7.7583441734313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14805176854133606,
      "logits/rejected": -0.17395809292793274,
      "logps/chosen": -440.06231689453125,
      "logps/rejected": -409.4190979003906,
      "loss": 0.4227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1341359615325928,
      "rewards/margins": 1.0258504152297974,
      "rewards/rejected": -2.1599860191345215,
      "step": 3226
    },
    {
      "epoch": 0.4222785638341376,
      "grad_norm": 7.216962814331055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3433316946029663,
      "logits/rejected": -0.214560404419899,
      "logps/chosen": -498.5684509277344,
      "logps/rejected": -485.92254638671875,
      "loss": 0.4646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03644201159477234,
      "rewards/margins": 1.0702189207077026,
      "rewards/rejected": -1.1066609621047974,
      "step": 3227
    },
    {
      "epoch": 0.4224094217714893,
      "grad_norm": 7.945133209228516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03238070756196976,
      "logits/rejected": -0.06070764362812042,
      "logps/chosen": -451.7362976074219,
      "logps/rejected": -583.5182495117188,
      "loss": 0.5612,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3784305453300476,
      "rewards/margins": 0.5311985015869141,
      "rewards/rejected": -0.9096290469169617,
      "step": 3228
    },
    {
      "epoch": 0.4225402797088411,
      "grad_norm": 7.00089168548584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23650287091732025,
      "logits/rejected": 0.13184845447540283,
      "logps/chosen": -386.53009033203125,
      "logps/rejected": -401.8972473144531,
      "loss": 0.5867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31455254554748535,
      "rewards/margins": 0.8132429122924805,
      "rewards/rejected": -1.1277954578399658,
      "step": 3229
    },
    {
      "epoch": 0.42267113764619285,
      "grad_norm": 6.73306131362915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36118489503860474,
      "logits/rejected": 0.35022562742233276,
      "logps/chosen": -492.6578063964844,
      "logps/rejected": -552.3862915039062,
      "loss": 0.3822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10517160594463348,
      "rewards/margins": 1.0314662456512451,
      "rewards/rejected": -0.9262948036193848,
      "step": 3230
    },
    {
      "epoch": 0.4228019955835446,
      "grad_norm": 3.3431167602539062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18219038844108582,
      "logits/rejected": -0.17625021934509277,
      "logps/chosen": -491.7276306152344,
      "logps/rejected": -441.0274658203125,
      "loss": 0.2946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14868107438087463,
      "rewards/margins": 1.6444218158721924,
      "rewards/rejected": -1.4957407712936401,
      "step": 3231
    },
    {
      "epoch": 0.4229328535208964,
      "grad_norm": 12.095471382141113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10183224827051163,
      "logits/rejected": -0.09920531511306763,
      "logps/chosen": -405.0130920410156,
      "logps/rejected": -305.291259765625,
      "loss": 0.9339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3708385229110718,
      "rewards/margins": -0.12049531936645508,
      "rewards/rejected": -1.2503432035446167,
      "step": 3232
    },
    {
      "epoch": 0.4230637114582481,
      "grad_norm": 4.772500991821289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3103426694869995,
      "logits/rejected": 0.23771704733371735,
      "logps/chosen": -490.3214111328125,
      "logps/rejected": -441.478759765625,
      "loss": 0.3463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41152897477149963,
      "rewards/margins": 0.9702249765396118,
      "rewards/rejected": -1.3817538022994995,
      "step": 3233
    },
    {
      "epoch": 0.4231945693955999,
      "grad_norm": 6.731635570526123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28968513011932373,
      "logits/rejected": 0.12624669075012207,
      "logps/chosen": -627.9395141601562,
      "logps/rejected": -542.6546020507812,
      "loss": 0.4267,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.37742775678634644,
      "rewards/margins": 1.125006079673767,
      "rewards/rejected": -0.747578501701355,
      "step": 3234
    },
    {
      "epoch": 0.42332542733295164,
      "grad_norm": 4.285659313201904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.202881321310997,
      "logits/rejected": 0.07906069606542587,
      "logps/chosen": -361.2875671386719,
      "logps/rejected": -354.7727355957031,
      "loss": 0.3932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0032638460397720337,
      "rewards/margins": 1.622354507446289,
      "rewards/rejected": -1.6256183385849,
      "step": 3235
    },
    {
      "epoch": 0.42345628527030343,
      "grad_norm": 6.093684196472168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09429703652858734,
      "logits/rejected": -0.04325646534562111,
      "logps/chosen": -554.1566772460938,
      "logps/rejected": -446.8705749511719,
      "loss": 0.3354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8862959146499634,
      "rewards/margins": 1.5248416662216187,
      "rewards/rejected": -2.411137580871582,
      "step": 3236
    },
    {
      "epoch": 0.42358714320765517,
      "grad_norm": 5.256718158721924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3559871315956116,
      "logits/rejected": 0.07467817515134811,
      "logps/chosen": -481.7123107910156,
      "logps/rejected": -550.915283203125,
      "loss": 0.3026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21221618354320526,
      "rewards/margins": 1.2707418203353882,
      "rewards/rejected": -1.4829580783843994,
      "step": 3237
    },
    {
      "epoch": 0.42371800114500696,
      "grad_norm": 5.512347221374512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3399696350097656,
      "logits/rejected": 0.16229532659053802,
      "logps/chosen": -408.56842041015625,
      "logps/rejected": -417.1990661621094,
      "loss": 0.3586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7286879420280457,
      "rewards/margins": 1.1488817930221558,
      "rewards/rejected": -1.8775697946548462,
      "step": 3238
    },
    {
      "epoch": 0.4238488590823587,
      "grad_norm": 6.191164016723633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26745033264160156,
      "logits/rejected": 0.092341348528862,
      "logps/chosen": -389.47991943359375,
      "logps/rejected": -472.04742431640625,
      "loss": 0.3637,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30989038944244385,
      "rewards/margins": 1.2396304607391357,
      "rewards/rejected": -1.5495208501815796,
      "step": 3239
    },
    {
      "epoch": 0.4239797170197105,
      "grad_norm": 6.325794696807861,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1690235584974289,
      "logits/rejected": 0.4408167600631714,
      "logps/chosen": -415.5906677246094,
      "logps/rejected": -507.3297424316406,
      "loss": 0.3333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1728103756904602,
      "rewards/margins": 1.2751253843307495,
      "rewards/rejected": -1.447935700416565,
      "step": 3240
    },
    {
      "epoch": 0.4241105749570622,
      "grad_norm": 7.850595951080322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15653756260871887,
      "logits/rejected": -0.17924492061138153,
      "logps/chosen": -338.8000793457031,
      "logps/rejected": -299.2917175292969,
      "loss": 0.7975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43738919496536255,
      "rewards/margins": 0.04322032630443573,
      "rewards/rejected": -0.4806095361709595,
      "step": 3241
    },
    {
      "epoch": 0.424241432894414,
      "grad_norm": 9.57726764678955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1312144547700882,
      "logits/rejected": -0.1594776213169098,
      "logps/chosen": -444.51300048828125,
      "logps/rejected": -468.68389892578125,
      "loss": 0.6981,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8257091641426086,
      "rewards/margins": 0.3170248568058014,
      "rewards/rejected": -1.1427340507507324,
      "step": 3242
    },
    {
      "epoch": 0.42437229083176575,
      "grad_norm": 5.920496463775635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03125244751572609,
      "logits/rejected": 0.2198893129825592,
      "logps/chosen": -375.39031982421875,
      "logps/rejected": -548.6724853515625,
      "loss": 0.344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22417524456977844,
      "rewards/margins": 1.407362461090088,
      "rewards/rejected": -1.631537675857544,
      "step": 3243
    },
    {
      "epoch": 0.42450314876911754,
      "grad_norm": 5.3178863525390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018901418894529343,
      "logits/rejected": 0.09542980790138245,
      "logps/chosen": -553.2567138671875,
      "logps/rejected": -545.2273559570312,
      "loss": 0.3317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.019856639206409454,
      "rewards/margins": 1.6559052467346191,
      "rewards/rejected": -1.6360487937927246,
      "step": 3244
    },
    {
      "epoch": 0.4246340067064693,
      "grad_norm": 3.8935635089874268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017563968896865845,
      "logits/rejected": 0.018169179558753967,
      "logps/chosen": -383.2623291015625,
      "logps/rejected": -318.54248046875,
      "loss": 0.2688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2500648498535156,
      "rewards/margins": 2.0330581665039062,
      "rewards/rejected": -1.7829933166503906,
      "step": 3245
    },
    {
      "epoch": 0.42476486464382107,
      "grad_norm": 8.287320137023926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17514574527740479,
      "logits/rejected": 0.29601186513900757,
      "logps/chosen": -454.7271423339844,
      "logps/rejected": -498.5940856933594,
      "loss": 0.6335,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04290273040533066,
      "rewards/margins": 1.0765479803085327,
      "rewards/rejected": -1.1194506883621216,
      "step": 3246
    },
    {
      "epoch": 0.4248957225811728,
      "grad_norm": 6.3607683181762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24398116767406464,
      "logits/rejected": -0.14698003232479095,
      "logps/chosen": -316.4073791503906,
      "logps/rejected": -403.41754150390625,
      "loss": 0.5859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07511673867702484,
      "rewards/margins": 0.38848745822906494,
      "rewards/rejected": -0.3133706748485565,
      "step": 3247
    },
    {
      "epoch": 0.4250265805185246,
      "grad_norm": 12.960923194885254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04849863052368164,
      "logits/rejected": -0.10856128484010696,
      "logps/chosen": -581.0186157226562,
      "logps/rejected": -501.36590576171875,
      "loss": 1.0947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1593904495239258,
      "rewards/margins": -0.33086758852005005,
      "rewards/rejected": -0.8285226821899414,
      "step": 3248
    },
    {
      "epoch": 0.42515743845587634,
      "grad_norm": 7.831576824188232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07019232213497162,
      "logits/rejected": -0.007461287081241608,
      "logps/chosen": -413.0198059082031,
      "logps/rejected": -440.7702941894531,
      "loss": 0.5923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3551512062549591,
      "rewards/margins": 0.3215276598930359,
      "rewards/rejected": -0.6766788363456726,
      "step": 3249
    },
    {
      "epoch": 0.4252882963932281,
      "grad_norm": 5.028213024139404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.087416872382164,
      "logits/rejected": 0.020099714398384094,
      "logps/chosen": -368.11639404296875,
      "logps/rejected": -403.8488464355469,
      "loss": 0.3972,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06464841961860657,
      "rewards/margins": 1.0034420490264893,
      "rewards/rejected": -1.0680904388427734,
      "step": 3250
    },
    {
      "epoch": 0.42541915433057986,
      "grad_norm": 11.072965621948242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2634536325931549,
      "logits/rejected": 0.29679355025291443,
      "logps/chosen": -475.610595703125,
      "logps/rejected": -524.33447265625,
      "loss": 0.8744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8380472660064697,
      "rewards/margins": 0.0738724023103714,
      "rewards/rejected": -0.9119195938110352,
      "step": 3251
    },
    {
      "epoch": 0.42555001226793165,
      "grad_norm": 4.928225040435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035274092108011246,
      "logits/rejected": 0.06655459105968475,
      "logps/chosen": -563.3487548828125,
      "logps/rejected": -500.011474609375,
      "loss": 0.2986,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6704893112182617,
      "rewards/margins": 1.513436198234558,
      "rewards/rejected": -2.1839256286621094,
      "step": 3252
    },
    {
      "epoch": 0.4256808702052834,
      "grad_norm": 7.774270534515381,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3153107464313507,
      "logits/rejected": 0.2801545262336731,
      "logps/chosen": -442.6906433105469,
      "logps/rejected": -448.4175109863281,
      "loss": 0.5528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7657936811447144,
      "rewards/margins": 0.7289584875106812,
      "rewards/rejected": -1.494752049446106,
      "step": 3253
    },
    {
      "epoch": 0.4258117281426351,
      "grad_norm": 5.928317070007324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10604750365018845,
      "logits/rejected": -0.013671786524355412,
      "logps/chosen": -336.3417663574219,
      "logps/rejected": -355.7153625488281,
      "loss": 0.4439,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2378019243478775,
      "rewards/margins": 1.6077638864517212,
      "rewards/rejected": -1.8455657958984375,
      "step": 3254
    },
    {
      "epoch": 0.4259425860799869,
      "grad_norm": 7.236236095428467,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0162336528301239,
      "logits/rejected": -0.055991798639297485,
      "logps/chosen": -468.0518493652344,
      "logps/rejected": -455.78778076171875,
      "loss": 0.4984,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6637806296348572,
      "rewards/margins": 0.9866998791694641,
      "rewards/rejected": -1.6504805088043213,
      "step": 3255
    },
    {
      "epoch": 0.42607344401733865,
      "grad_norm": 6.482762813568115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30261683464050293,
      "logits/rejected": 0.12212880700826645,
      "logps/chosen": -576.35302734375,
      "logps/rejected": -517.8016357421875,
      "loss": 0.4352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3075084686279297,
      "rewards/margins": 0.9603607654571533,
      "rewards/rejected": -1.2678691148757935,
      "step": 3256
    },
    {
      "epoch": 0.42620430195469045,
      "grad_norm": 8.542656898498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39531534910202026,
      "logits/rejected": 0.31807205080986023,
      "logps/chosen": -558.73876953125,
      "logps/rejected": -360.2425537109375,
      "loss": 0.7081,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5375477075576782,
      "rewards/margins": 0.13380470871925354,
      "rewards/rejected": -0.6713523864746094,
      "step": 3257
    },
    {
      "epoch": 0.4263351598920422,
      "grad_norm": 10.778573989868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010718036442995071,
      "logits/rejected": -0.25399839878082275,
      "logps/chosen": -635.6016235351562,
      "logps/rejected": -501.81866455078125,
      "loss": 0.7665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9542623162269592,
      "rewards/margins": 0.38949525356292725,
      "rewards/rejected": -1.3437576293945312,
      "step": 3258
    },
    {
      "epoch": 0.426466017829394,
      "grad_norm": 4.65321683883667,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17861723899841309,
      "logits/rejected": -0.15358290076255798,
      "logps/chosen": -481.53277587890625,
      "logps/rejected": -582.8185424804688,
      "loss": 0.2853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0048469603061676025,
      "rewards/margins": 1.605044960975647,
      "rewards/rejected": -1.6098917722702026,
      "step": 3259
    },
    {
      "epoch": 0.4265968757667457,
      "grad_norm": 7.668245315551758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19894835352897644,
      "logits/rejected": 0.015300264582037926,
      "logps/chosen": -432.8616943359375,
      "logps/rejected": -419.2244873046875,
      "loss": 0.398,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3093770742416382,
      "rewards/margins": 1.0966253280639648,
      "rewards/rejected": -1.4060025215148926,
      "step": 3260
    },
    {
      "epoch": 0.4267277337040975,
      "grad_norm": 5.600707054138184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1352752149105072,
      "logits/rejected": 0.09821073710918427,
      "logps/chosen": -444.2300720214844,
      "logps/rejected": -404.0605163574219,
      "loss": 0.5172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0121303796768188,
      "rewards/margins": 0.695002555847168,
      "rewards/rejected": -1.7071328163146973,
      "step": 3261
    },
    {
      "epoch": 0.42685859164144924,
      "grad_norm": 5.9094719886779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29996752738952637,
      "logits/rejected": 0.2531915307044983,
      "logps/chosen": -595.733642578125,
      "logps/rejected": -553.8705444335938,
      "loss": 0.3513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1716056913137436,
      "rewards/margins": 1.2998847961425781,
      "rewards/rejected": -1.471490502357483,
      "step": 3262
    },
    {
      "epoch": 0.42698944957880103,
      "grad_norm": 8.707521438598633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20235052704811096,
      "logits/rejected": 0.021093277260661125,
      "logps/chosen": -309.7733459472656,
      "logps/rejected": -458.9568786621094,
      "loss": 0.4719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3228718042373657,
      "rewards/margins": 0.6652700304985046,
      "rewards/rejected": -1.9881420135498047,
      "step": 3263
    },
    {
      "epoch": 0.42712030751615276,
      "grad_norm": 7.714970588684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02091088891029358,
      "logits/rejected": 0.10795725882053375,
      "logps/chosen": -532.6675415039062,
      "logps/rejected": -522.8250732421875,
      "loss": 0.5951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04632606357336044,
      "rewards/margins": 0.32651329040527344,
      "rewards/rejected": -0.2801872789859772,
      "step": 3264
    },
    {
      "epoch": 0.42725116545350456,
      "grad_norm": 5.852935791015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20777000486850739,
      "logits/rejected": -0.0719294548034668,
      "logps/chosen": -398.61029052734375,
      "logps/rejected": -399.1541748046875,
      "loss": 0.5728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0619171857833862,
      "rewards/margins": 0.619280993938446,
      "rewards/rejected": -1.6811981201171875,
      "step": 3265
    },
    {
      "epoch": 0.4273820233908563,
      "grad_norm": 5.246676921844482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07539738714694977,
      "logits/rejected": 0.13514858484268188,
      "logps/chosen": -352.31646728515625,
      "logps/rejected": -429.6228942871094,
      "loss": 0.4092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7060301303863525,
      "rewards/margins": 1.2482678890228271,
      "rewards/rejected": -1.9542981386184692,
      "step": 3266
    },
    {
      "epoch": 0.4275128813282081,
      "grad_norm": 11.52895736694336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14159150421619415,
      "logits/rejected": -0.135273277759552,
      "logps/chosen": -501.989013671875,
      "logps/rejected": -374.5255432128906,
      "loss": 0.8654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4147804379463196,
      "rewards/margins": 0.07880781590938568,
      "rewards/rejected": -0.49358832836151123,
      "step": 3267
    },
    {
      "epoch": 0.4276437392655598,
      "grad_norm": 6.815978527069092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011304712854325771,
      "logits/rejected": -0.03535648062825203,
      "logps/chosen": -492.84857177734375,
      "logps/rejected": -525.3407592773438,
      "loss": 0.478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04375801980495453,
      "rewards/margins": 1.0104118585586548,
      "rewards/rejected": -1.0541698932647705,
      "step": 3268
    },
    {
      "epoch": 0.4277745972029116,
      "grad_norm": 10.524636268615723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22493714094161987,
      "logits/rejected": 0.14370287954807281,
      "logps/chosen": -499.9249572753906,
      "logps/rejected": -473.34197998046875,
      "loss": 0.8567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.719578742980957,
      "rewards/margins": 0.22077025473117828,
      "rewards/rejected": -0.9403491020202637,
      "step": 3269
    },
    {
      "epoch": 0.42790545514026335,
      "grad_norm": 9.407196998596191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04721258953213692,
      "logits/rejected": -0.0022968053817749023,
      "logps/chosen": -524.6722412109375,
      "logps/rejected": -628.2861328125,
      "loss": 0.5833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32621288299560547,
      "rewards/margins": 1.0151169300079346,
      "rewards/rejected": -1.34132981300354,
      "step": 3270
    },
    {
      "epoch": 0.42803631307761514,
      "grad_norm": 5.132707118988037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08750028163194656,
      "logits/rejected": -0.08673438429832458,
      "logps/chosen": -548.5426025390625,
      "logps/rejected": -531.2354736328125,
      "loss": 0.2301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35878676176071167,
      "rewards/margins": 2.6848433017730713,
      "rewards/rejected": -2.326056718826294,
      "step": 3271
    },
    {
      "epoch": 0.4281671710149669,
      "grad_norm": 7.220724105834961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.258973628282547,
      "logits/rejected": 0.234795480966568,
      "logps/chosen": -490.0504150390625,
      "logps/rejected": -565.0919189453125,
      "loss": 0.5296,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43161454796791077,
      "rewards/margins": 1.1933943033218384,
      "rewards/rejected": -1.6250088214874268,
      "step": 3272
    },
    {
      "epoch": 0.42829802895231867,
      "grad_norm": 6.486116886138916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23584218323230743,
      "logits/rejected": 0.11681652069091797,
      "logps/chosen": -526.9129638671875,
      "logps/rejected": -374.0007629394531,
      "loss": 0.4783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20865842700004578,
      "rewards/margins": 0.9843845367431641,
      "rewards/rejected": -1.1930429935455322,
      "step": 3273
    },
    {
      "epoch": 0.4284288868896704,
      "grad_norm": 9.5137357711792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03784197196364403,
      "logits/rejected": 0.06240927428007126,
      "logps/chosen": -387.9811096191406,
      "logps/rejected": -324.30755615234375,
      "loss": 0.7753,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9931925535202026,
      "rewards/margins": 0.6791955828666687,
      "rewards/rejected": -1.6723880767822266,
      "step": 3274
    },
    {
      "epoch": 0.42855974482702214,
      "grad_norm": 7.202136516571045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07370580732822418,
      "logits/rejected": -0.18886354565620422,
      "logps/chosen": -470.5224609375,
      "logps/rejected": -406.6836853027344,
      "loss": 0.4037,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5194517970085144,
      "rewards/margins": 1.2622995376586914,
      "rewards/rejected": -1.7817513942718506,
      "step": 3275
    },
    {
      "epoch": 0.42869060276437393,
      "grad_norm": 7.793026924133301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1438530832529068,
      "logits/rejected": -0.030804596841335297,
      "logps/chosen": -397.8988952636719,
      "logps/rejected": -377.5854797363281,
      "loss": 0.5159,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44888681173324585,
      "rewards/margins": 0.7324217557907104,
      "rewards/rejected": -1.1813085079193115,
      "step": 3276
    },
    {
      "epoch": 0.42882146070172567,
      "grad_norm": 10.498004913330078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10378973186016083,
      "logits/rejected": -0.0419657863676548,
      "logps/chosen": -564.8129272460938,
      "logps/rejected": -406.7591552734375,
      "loss": 0.8344,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1160995960235596,
      "rewards/margins": 0.03983987122774124,
      "rewards/rejected": -1.1559393405914307,
      "step": 3277
    },
    {
      "epoch": 0.42895231863907746,
      "grad_norm": 6.0114359855651855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03131754323840141,
      "logits/rejected": 0.018939783796668053,
      "logps/chosen": -431.730712890625,
      "logps/rejected": -357.4285888671875,
      "loss": 0.476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2133285403251648,
      "rewards/margins": 0.7754493951797485,
      "rewards/rejected": -0.5621208548545837,
      "step": 3278
    },
    {
      "epoch": 0.4290831765764292,
      "grad_norm": 6.679197311401367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05282413959503174,
      "logits/rejected": 0.1202785074710846,
      "logps/chosen": -582.424560546875,
      "logps/rejected": -513.48779296875,
      "loss": 0.3619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6604011654853821,
      "rewards/margins": 1.4386926889419556,
      "rewards/rejected": -2.0990939140319824,
      "step": 3279
    },
    {
      "epoch": 0.429214034513781,
      "grad_norm": 6.105681896209717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04757670685648918,
      "logits/rejected": 0.21727672219276428,
      "logps/chosen": -535.4124755859375,
      "logps/rejected": -540.1408081054688,
      "loss": 0.5784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41050034761428833,
      "rewards/margins": 0.6559351086616516,
      "rewards/rejected": -1.0664355754852295,
      "step": 3280
    },
    {
      "epoch": 0.4293448924511327,
      "grad_norm": 9.604320526123047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13885560631752014,
      "logits/rejected": 0.056136488914489746,
      "logps/chosen": -620.7008666992188,
      "logps/rejected": -520.0620727539062,
      "loss": 0.7457,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4397909343242645,
      "rewards/margins": 0.4416409134864807,
      "rewards/rejected": -0.8814318776130676,
      "step": 3281
    },
    {
      "epoch": 0.4294757503884845,
      "grad_norm": 11.057015419006348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4580215811729431,
      "logits/rejected": 0.34035879373550415,
      "logps/chosen": -563.167236328125,
      "logps/rejected": -443.1059875488281,
      "loss": 0.892,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8318861722946167,
      "rewards/margins": 0.35901278257369995,
      "rewards/rejected": -1.1908988952636719,
      "step": 3282
    },
    {
      "epoch": 0.42960660832583625,
      "grad_norm": 8.675201416015625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4338427484035492,
      "logits/rejected": 0.24142029881477356,
      "logps/chosen": -407.4696350097656,
      "logps/rejected": -406.5118408203125,
      "loss": 0.539,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.051319677382707596,
      "rewards/margins": 0.6946735382080078,
      "rewards/rejected": -0.6433538198471069,
      "step": 3283
    },
    {
      "epoch": 0.42973746626318804,
      "grad_norm": 13.079547882080078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05075379088521004,
      "logits/rejected": 0.14172042906284332,
      "logps/chosen": -458.2389831542969,
      "logps/rejected": -521.22314453125,
      "loss": 0.9844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5717800259590149,
      "rewards/margins": 0.010601818561553955,
      "rewards/rejected": -0.5823818445205688,
      "step": 3284
    },
    {
      "epoch": 0.4298683242005398,
      "grad_norm": 4.260423183441162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22973880171775818,
      "logits/rejected": -0.2391689270734787,
      "logps/chosen": -385.9187927246094,
      "logps/rejected": -597.4596557617188,
      "loss": 0.2389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3896610736846924,
      "rewards/margins": 2.5422651767730713,
      "rewards/rejected": -2.9319262504577637,
      "step": 3285
    },
    {
      "epoch": 0.42999918213789157,
      "grad_norm": 5.729428291320801,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14621366560459137,
      "logits/rejected": 0.253553569316864,
      "logps/chosen": -529.734130859375,
      "logps/rejected": -565.1884155273438,
      "loss": 0.5147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7874858975410461,
      "rewards/margins": 1.0738123655319214,
      "rewards/rejected": -1.8612982034683228,
      "step": 3286
    },
    {
      "epoch": 0.4301300400752433,
      "grad_norm": 2.976633310317993,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21302270889282227,
      "logits/rejected": 0.2759767472743988,
      "logps/chosen": -393.41229248046875,
      "logps/rejected": -477.1153869628906,
      "loss": 0.1501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3635783791542053,
      "rewards/margins": 1.9962248802185059,
      "rewards/rejected": -1.6326464414596558,
      "step": 3287
    },
    {
      "epoch": 0.4302608980125951,
      "grad_norm": 10.681709289550781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004382051527500153,
      "logits/rejected": -0.16002844274044037,
      "logps/chosen": -737.7679443359375,
      "logps/rejected": -698.6419067382812,
      "loss": 0.7377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14738957583904266,
      "rewards/margins": 0.6564759016036987,
      "rewards/rejected": -0.8038654327392578,
      "step": 3288
    },
    {
      "epoch": 0.43039175594994683,
      "grad_norm": 6.702852725982666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5731947422027588,
      "logits/rejected": 0.4677777886390686,
      "logps/chosen": -469.4044189453125,
      "logps/rejected": -476.4374084472656,
      "loss": 0.6037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.32380256056785583,
      "rewards/margins": 0.9857348799705505,
      "rewards/rejected": -0.6619324088096619,
      "step": 3289
    },
    {
      "epoch": 0.4305226138872986,
      "grad_norm": 12.026776313781738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04303209111094475,
      "logits/rejected": -0.10132483392953873,
      "logps/chosen": -326.5986633300781,
      "logps/rejected": -345.1587829589844,
      "loss": 1.1688,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.6275663375854492,
      "rewards/margins": -0.2810370922088623,
      "rewards/rejected": -1.3465293645858765,
      "step": 3290
    },
    {
      "epoch": 0.43065347182465036,
      "grad_norm": 10.588329315185547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.185266375541687,
      "logits/rejected": 0.05617143213748932,
      "logps/chosen": -451.0917663574219,
      "logps/rejected": -534.541015625,
      "loss": 0.685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9811370968818665,
      "rewards/margins": 1.7713618278503418,
      "rewards/rejected": -2.7524991035461426,
      "step": 3291
    },
    {
      "epoch": 0.43078432976200215,
      "grad_norm": 6.4947509765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38088810443878174,
      "logits/rejected": 0.24681538343429565,
      "logps/chosen": -482.87066650390625,
      "logps/rejected": -421.6293029785156,
      "loss": 0.4524,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14375707507133484,
      "rewards/margins": 1.035205364227295,
      "rewards/rejected": -0.8914481997489929,
      "step": 3292
    },
    {
      "epoch": 0.4309151876993539,
      "grad_norm": 7.635465621948242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08870947360992432,
      "logits/rejected": -0.038174811750650406,
      "logps/chosen": -327.8749694824219,
      "logps/rejected": -345.9364929199219,
      "loss": 0.5587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0738669633865356,
      "rewards/margins": 0.8164669275283813,
      "rewards/rejected": -1.8903340101242065,
      "step": 3293
    },
    {
      "epoch": 0.4310460456367057,
      "grad_norm": 6.961939334869385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12906627357006073,
      "logits/rejected": -0.003544580191373825,
      "logps/chosen": -301.5085754394531,
      "logps/rejected": -505.0947570800781,
      "loss": 0.5018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6603529453277588,
      "rewards/margins": 1.4051518440246582,
      "rewards/rejected": -2.065505027770996,
      "step": 3294
    },
    {
      "epoch": 0.4311769035740574,
      "grad_norm": 6.2286057472229,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29469990730285645,
      "logits/rejected": -0.26876914501190186,
      "logps/chosen": -470.3544006347656,
      "logps/rejected": -641.098388671875,
      "loss": 0.4375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13663671910762787,
      "rewards/margins": 1.2681407928466797,
      "rewards/rejected": -1.1315041780471802,
      "step": 3295
    },
    {
      "epoch": 0.43130776151140915,
      "grad_norm": 6.97944450378418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26564276218414307,
      "logits/rejected": 0.25004202127456665,
      "logps/chosen": -509.8975524902344,
      "logps/rejected": -509.3990478515625,
      "loss": 0.5743,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.499114990234375,
      "rewards/margins": 0.4265987277030945,
      "rewards/rejected": -0.9257137179374695,
      "step": 3296
    },
    {
      "epoch": 0.43143861944876094,
      "grad_norm": 5.816553592681885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06355640292167664,
      "logits/rejected": -0.1563473492860794,
      "logps/chosen": -466.91455078125,
      "logps/rejected": -469.3388671875,
      "loss": 0.29,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09752769023180008,
      "rewards/margins": 1.324338436126709,
      "rewards/rejected": -1.4218662977218628,
      "step": 3297
    },
    {
      "epoch": 0.4315694773861127,
      "grad_norm": 7.236236572265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11006062477827072,
      "logits/rejected": 0.027328211814165115,
      "logps/chosen": -545.249267578125,
      "logps/rejected": -413.4808349609375,
      "loss": 0.5872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9993143081665039,
      "rewards/margins": 0.4607192873954773,
      "rewards/rejected": -1.460033655166626,
      "step": 3298
    },
    {
      "epoch": 0.43170033532346447,
      "grad_norm": 5.16211462020874,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09572970867156982,
      "logits/rejected": -0.2612159550189972,
      "logps/chosen": -509.091064453125,
      "logps/rejected": -393.6675720214844,
      "loss": 0.4011,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5057939291000366,
      "rewards/margins": 1.10020911693573,
      "rewards/rejected": -1.6060032844543457,
      "step": 3299
    },
    {
      "epoch": 0.4318311932608162,
      "grad_norm": 9.142414093017578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26193156838417053,
      "logits/rejected": 0.374822735786438,
      "logps/chosen": -502.22869873046875,
      "logps/rejected": -510.8262939453125,
      "loss": 0.6849,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2788032293319702,
      "rewards/margins": 0.231806218624115,
      "rewards/rejected": -1.51060950756073,
      "step": 3300
    },
    {
      "epoch": 0.431962051198168,
      "grad_norm": 4.665041446685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01658537983894348,
      "logits/rejected": -0.39480751752853394,
      "logps/chosen": -386.5018615722656,
      "logps/rejected": -445.70111083984375,
      "loss": 0.304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4686439633369446,
      "rewards/margins": 1.4392244815826416,
      "rewards/rejected": -1.9078686237335205,
      "step": 3301
    },
    {
      "epoch": 0.43209290913551973,
      "grad_norm": 9.149276733398438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15125928819179535,
      "logits/rejected": 0.1741611361503601,
      "logps/chosen": -657.167236328125,
      "logps/rejected": -570.7384033203125,
      "loss": 0.7149,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5918605923652649,
      "rewards/margins": 0.611548662185669,
      "rewards/rejected": -0.019688040018081665,
      "step": 3302
    },
    {
      "epoch": 0.4322237670728715,
      "grad_norm": 7.59537935256958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04896319657564163,
      "logits/rejected": 0.06051858142018318,
      "logps/chosen": -427.5442810058594,
      "logps/rejected": -387.5047912597656,
      "loss": 0.6631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2833654582500458,
      "rewards/margins": 0.35583195090293884,
      "rewards/rejected": -0.07246650010347366,
      "step": 3303
    },
    {
      "epoch": 0.43235462501022326,
      "grad_norm": 6.405568599700928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0021239258348941803,
      "logits/rejected": -0.03264366462826729,
      "logps/chosen": -464.5605773925781,
      "logps/rejected": -404.3487548828125,
      "loss": 0.3616,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0692945346236229,
      "rewards/margins": 1.3196008205413818,
      "rewards/rejected": -1.2503063678741455,
      "step": 3304
    },
    {
      "epoch": 0.43248548294757505,
      "grad_norm": 9.464029312133789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06717146188020706,
      "logits/rejected": 0.03434496745467186,
      "logps/chosen": -380.7838439941406,
      "logps/rejected": -337.5074768066406,
      "loss": 0.7166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12231484055519104,
      "rewards/margins": 0.4600335955619812,
      "rewards/rejected": -0.5823484659194946,
      "step": 3305
    },
    {
      "epoch": 0.4326163408849268,
      "grad_norm": 6.711846351623535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3573082387447357,
      "logits/rejected": 0.07589315623044968,
      "logps/chosen": -258.00531005859375,
      "logps/rejected": -366.9256286621094,
      "loss": 0.3806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5352991223335266,
      "rewards/margins": 1.762467622756958,
      "rewards/rejected": -2.297766923904419,
      "step": 3306
    },
    {
      "epoch": 0.4327471988222786,
      "grad_norm": 7.463958740234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047778185456991196,
      "logits/rejected": -0.027962086722254753,
      "logps/chosen": -437.851318359375,
      "logps/rejected": -383.5862121582031,
      "loss": 0.4185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6553892493247986,
      "rewards/margins": 0.978131115436554,
      "rewards/rejected": -1.633520245552063,
      "step": 3307
    },
    {
      "epoch": 0.4328780567596303,
      "grad_norm": 11.102225303649902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026439346373081207,
      "logits/rejected": -0.038354549556970596,
      "logps/chosen": -454.75048828125,
      "logps/rejected": -505.5987548828125,
      "loss": 1.0351,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0945724248886108,
      "rewards/margins": -0.1803409606218338,
      "rewards/rejected": -0.914231538772583,
      "step": 3308
    },
    {
      "epoch": 0.4330089146969821,
      "grad_norm": 12.01225757598877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10291951894760132,
      "logits/rejected": 0.17081531882286072,
      "logps/chosen": -902.8745727539062,
      "logps/rejected": -554.65283203125,
      "loss": 0.6806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43899157643318176,
      "rewards/margins": 0.4339672327041626,
      "rewards/rejected": -0.8729587197303772,
      "step": 3309
    },
    {
      "epoch": 0.43313977263433384,
      "grad_norm": 4.016901969909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.059246137738227844,
      "logits/rejected": -0.1362934112548828,
      "logps/chosen": -452.742919921875,
      "logps/rejected": -527.374267578125,
      "loss": 0.2621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4237644076347351,
      "rewards/margins": 1.676235556602478,
      "rewards/rejected": -2.0999999046325684,
      "step": 3310
    },
    {
      "epoch": 0.43327063057168563,
      "grad_norm": 7.914358615875244,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16659288108348846,
      "logits/rejected": -0.12949822843074799,
      "logps/chosen": -459.4951171875,
      "logps/rejected": -440.4366455078125,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25252094864845276,
      "rewards/margins": 1.0299952030181885,
      "rewards/rejected": -1.2825162410736084,
      "step": 3311
    },
    {
      "epoch": 0.43340148850903737,
      "grad_norm": 4.598884582519531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3567434847354889,
      "logits/rejected": 0.33380722999572754,
      "logps/chosen": -561.8165283203125,
      "logps/rejected": -440.9280090332031,
      "loss": 0.3187,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5146379470825195,
      "rewards/margins": 1.1096704006195068,
      "rewards/rejected": -1.624308466911316,
      "step": 3312
    },
    {
      "epoch": 0.43353234644638916,
      "grad_norm": 8.84575080871582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22619366645812988,
      "logits/rejected": -0.0804394856095314,
      "logps/chosen": -491.109375,
      "logps/rejected": -577.0382690429688,
      "loss": 0.6155,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.309912919998169,
      "rewards/margins": 0.7460705041885376,
      "rewards/rejected": -2.055983543395996,
      "step": 3313
    },
    {
      "epoch": 0.4336632043837409,
      "grad_norm": 7.6641740798950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09523628652095795,
      "logits/rejected": -0.04715092480182648,
      "logps/chosen": -351.09259033203125,
      "logps/rejected": -335.7670593261719,
      "loss": 0.7184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9623449444770813,
      "rewards/margins": 0.05054132640361786,
      "rewards/rejected": -1.01288640499115,
      "step": 3314
    },
    {
      "epoch": 0.4337940623210927,
      "grad_norm": 6.945923328399658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14251939952373505,
      "logits/rejected": 0.2337987720966339,
      "logps/chosen": -399.1842346191406,
      "logps/rejected": -508.88140869140625,
      "loss": 0.3194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11264093220233917,
      "rewards/margins": 1.8649592399597168,
      "rewards/rejected": -1.7523181438446045,
      "step": 3315
    },
    {
      "epoch": 0.4339249202584444,
      "grad_norm": 9.457279205322266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27578476071357727,
      "logits/rejected": 0.14931955933570862,
      "logps/chosen": -544.2886962890625,
      "logps/rejected": -500.47930908203125,
      "loss": 0.6602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2582511901855469,
      "rewards/margins": 1.0054197311401367,
      "rewards/rejected": -1.2636710405349731,
      "step": 3316
    },
    {
      "epoch": 0.4340557781957962,
      "grad_norm": 7.013016700744629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1848171353340149,
      "logits/rejected": 0.11476624011993408,
      "logps/chosen": -553.886962890625,
      "logps/rejected": -600.5054931640625,
      "loss": 0.62,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38720303773880005,
      "rewards/margins": 0.5449562668800354,
      "rewards/rejected": -0.9321593046188354,
      "step": 3317
    },
    {
      "epoch": 0.43418663613314795,
      "grad_norm": 6.023440361022949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06585967540740967,
      "logits/rejected": 0.19393165409564972,
      "logps/chosen": -227.09930419921875,
      "logps/rejected": -426.4982604980469,
      "loss": 0.2546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.004110321402549744,
      "rewards/margins": 2.2424397468566895,
      "rewards/rejected": -2.2383296489715576,
      "step": 3318
    },
    {
      "epoch": 0.4343174940704997,
      "grad_norm": 8.513830184936523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04141894727945328,
      "logits/rejected": 0.15987709164619446,
      "logps/chosen": -472.0138854980469,
      "logps/rejected": -417.94952392578125,
      "loss": 0.6975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4264860451221466,
      "rewards/margins": 0.9178609848022461,
      "rewards/rejected": -1.3443470001220703,
      "step": 3319
    },
    {
      "epoch": 0.4344483520078515,
      "grad_norm": 8.254018783569336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006484296638518572,
      "logits/rejected": 0.04879189282655716,
      "logps/chosen": -389.556396484375,
      "logps/rejected": -310.61444091796875,
      "loss": 0.4776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5440051555633545,
      "rewards/margins": 1.0057623386383057,
      "rewards/rejected": -1.5497673749923706,
      "step": 3320
    },
    {
      "epoch": 0.4345792099452032,
      "grad_norm": 6.6688947677612305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1993410885334015,
      "logits/rejected": 0.10257046669721603,
      "logps/chosen": -422.5643310546875,
      "logps/rejected": -394.3602600097656,
      "loss": 0.4471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14671307802200317,
      "rewards/margins": 0.7971513867378235,
      "rewards/rejected": -0.6504383683204651,
      "step": 3321
    },
    {
      "epoch": 0.434710067882555,
      "grad_norm": 8.146117210388184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1266438364982605,
      "logits/rejected": -0.09825941920280457,
      "logps/chosen": -381.10699462890625,
      "logps/rejected": -573.2304077148438,
      "loss": 0.4706,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0160579681396484,
      "rewards/margins": 0.9881105422973633,
      "rewards/rejected": -2.0041685104370117,
      "step": 3322
    },
    {
      "epoch": 0.43484092581990674,
      "grad_norm": 6.71412992477417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020456742495298386,
      "logits/rejected": -0.30950111150741577,
      "logps/chosen": -485.5067443847656,
      "logps/rejected": -421.10968017578125,
      "loss": 0.4308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06509435176849365,
      "rewards/margins": 1.299895167350769,
      "rewards/rejected": -1.2348006963729858,
      "step": 3323
    },
    {
      "epoch": 0.43497178375725853,
      "grad_norm": 5.278783798217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0338001623749733,
      "logits/rejected": -0.14922918379306793,
      "logps/chosen": -234.93704223632812,
      "logps/rejected": -282.81219482421875,
      "loss": 0.4571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45708510279655457,
      "rewards/margins": 0.8326727747917175,
      "rewards/rejected": -0.37558767199516296,
      "step": 3324
    },
    {
      "epoch": 0.43510264169461027,
      "grad_norm": 8.798276901245117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03420587629079819,
      "logits/rejected": 0.20536071062088013,
      "logps/chosen": -509.95574951171875,
      "logps/rejected": -520.53076171875,
      "loss": 0.6195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4006114602088928,
      "rewards/margins": 0.7286560535430908,
      "rewards/rejected": -1.1292674541473389,
      "step": 3325
    },
    {
      "epoch": 0.43523349963196206,
      "grad_norm": 6.414581775665283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04862378537654877,
      "logits/rejected": 0.08100810647010803,
      "logps/chosen": -318.411376953125,
      "logps/rejected": -262.08404541015625,
      "loss": 0.4648,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8142869472503662,
      "rewards/margins": 0.7116350531578064,
      "rewards/rejected": -1.5259220600128174,
      "step": 3326
    },
    {
      "epoch": 0.4353643575693138,
      "grad_norm": 7.107457160949707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14825275540351868,
      "logits/rejected": -0.22848676145076752,
      "logps/chosen": -289.0120849609375,
      "logps/rejected": -298.2313537597656,
      "loss": 0.7698,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4864482879638672,
      "rewards/margins": 0.11908091604709625,
      "rewards/rejected": -0.605529248714447,
      "step": 3327
    },
    {
      "epoch": 0.4354952155066656,
      "grad_norm": 5.592336654663086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2086510807275772,
      "logits/rejected": -0.014970950782299042,
      "logps/chosen": -709.8787231445312,
      "logps/rejected": -740.2530517578125,
      "loss": 0.3189,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6671773791313171,
      "rewards/margins": 1.4833288192749023,
      "rewards/rejected": -2.1505062580108643,
      "step": 3328
    },
    {
      "epoch": 0.4356260734440173,
      "grad_norm": 6.548085689544678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07527022808790207,
      "logits/rejected": 0.11041782796382904,
      "logps/chosen": -502.7652587890625,
      "logps/rejected": -432.2861328125,
      "loss": 0.408,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5403262376785278,
      "rewards/margins": 1.2820454835891724,
      "rewards/rejected": -1.8223718404769897,
      "step": 3329
    },
    {
      "epoch": 0.4357569313813691,
      "grad_norm": 6.374773979187012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08374397456645966,
      "logits/rejected": 0.004810631275177002,
      "logps/chosen": -606.5899658203125,
      "logps/rejected": -541.54541015625,
      "loss": 0.2937,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5749777555465698,
      "rewards/margins": 1.6645394563674927,
      "rewards/rejected": -1.0895617008209229,
      "step": 3330
    },
    {
      "epoch": 0.43588778931872085,
      "grad_norm": 4.657774925231934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13197678327560425,
      "logits/rejected": -0.1715162694454193,
      "logps/chosen": -362.7827453613281,
      "logps/rejected": -523.3477783203125,
      "loss": 0.2759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48833030462265015,
      "rewards/margins": 1.3204437494277954,
      "rewards/rejected": -1.8087741136550903,
      "step": 3331
    },
    {
      "epoch": 0.43601864725607264,
      "grad_norm": 5.7876434326171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22158966958522797,
      "logits/rejected": 0.2027381956577301,
      "logps/chosen": -553.1617431640625,
      "logps/rejected": -556.802490234375,
      "loss": 0.3867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35175588726997375,
      "rewards/margins": 1.339426040649414,
      "rewards/rejected": -1.6911817789077759,
      "step": 3332
    },
    {
      "epoch": 0.4361495051934244,
      "grad_norm": 7.15287971496582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13780027627944946,
      "logits/rejected": 0.2500413954257965,
      "logps/chosen": -608.0516357421875,
      "logps/rejected": -473.0395202636719,
      "loss": 0.3675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6018709540367126,
      "rewards/margins": 1.031802773475647,
      "rewards/rejected": -0.4299318194389343,
      "step": 3333
    },
    {
      "epoch": 0.43628036313077617,
      "grad_norm": 4.759995460510254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06946061551570892,
      "logits/rejected": -0.23099330067634583,
      "logps/chosen": -543.8180541992188,
      "logps/rejected": -533.8466186523438,
      "loss": 0.3053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.141702651977539,
      "rewards/margins": 1.4117858409881592,
      "rewards/rejected": -0.2700832486152649,
      "step": 3334
    },
    {
      "epoch": 0.4364112210681279,
      "grad_norm": 6.918516159057617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02777533046901226,
      "logits/rejected": -0.11757082492113113,
      "logps/chosen": -485.60919189453125,
      "logps/rejected": -486.79388427734375,
      "loss": 0.5243,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11006622761487961,
      "rewards/margins": 0.7451872229576111,
      "rewards/rejected": -0.8552533984184265,
      "step": 3335
    },
    {
      "epoch": 0.4365420790054797,
      "grad_norm": 7.307866096496582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07554306089878082,
      "logits/rejected": 0.3291224539279938,
      "logps/chosen": -370.1551513671875,
      "logps/rejected": -582.2656860351562,
      "loss": 0.3914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7635958194732666,
      "rewards/margins": 1.1623880863189697,
      "rewards/rejected": -1.9259837865829468,
      "step": 3336
    },
    {
      "epoch": 0.43667293694283144,
      "grad_norm": 12.298247337341309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22929859161376953,
      "logits/rejected": -0.10381126403808594,
      "logps/chosen": -485.81744384765625,
      "logps/rejected": -427.33544921875,
      "loss": 1.1017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0764949321746826,
      "rewards/margins": -0.03839746490120888,
      "rewards/rejected": -1.0380975008010864,
      "step": 3337
    },
    {
      "epoch": 0.4368037948801832,
      "grad_norm": 10.056997299194336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03414797782897949,
      "logits/rejected": -0.06777369976043701,
      "logps/chosen": -322.9199523925781,
      "logps/rejected": -375.01507568359375,
      "loss": 0.8325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6524189114570618,
      "rewards/margins": 0.255790114402771,
      "rewards/rejected": -0.9082090258598328,
      "step": 3338
    },
    {
      "epoch": 0.43693465281753496,
      "grad_norm": 9.096073150634766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3095996081829071,
      "logits/rejected": 0.40400072932243347,
      "logps/chosen": -429.23162841796875,
      "logps/rejected": -436.9649658203125,
      "loss": 0.6763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.513469934463501,
      "rewards/margins": 0.7199680805206299,
      "rewards/rejected": -1.2334380149841309,
      "step": 3339
    },
    {
      "epoch": 0.4370655107548867,
      "grad_norm": 5.853013038635254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07528539001941681,
      "logits/rejected": -0.058152396231889725,
      "logps/chosen": -353.31005859375,
      "logps/rejected": -399.58343505859375,
      "loss": 0.444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41625040769577026,
      "rewards/margins": 1.2134398221969604,
      "rewards/rejected": -1.629690170288086,
      "step": 3340
    },
    {
      "epoch": 0.4371963686922385,
      "grad_norm": 5.429232120513916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30210697650909424,
      "logits/rejected": 0.2102908492088318,
      "logps/chosen": -369.7904968261719,
      "logps/rejected": -397.2933349609375,
      "loss": 0.4188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6283970475196838,
      "rewards/margins": 0.8826482892036438,
      "rewards/rejected": -0.25425130128860474,
      "step": 3341
    },
    {
      "epoch": 0.4373272266295902,
      "grad_norm": 6.187525272369385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09561567008495331,
      "logits/rejected": -0.23353609442710876,
      "logps/chosen": -386.5779113769531,
      "logps/rejected": -375.0907897949219,
      "loss": 0.4341,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21352039277553558,
      "rewards/margins": 1.4777519702911377,
      "rewards/rejected": -1.691272258758545,
      "step": 3342
    },
    {
      "epoch": 0.437458084566942,
      "grad_norm": 3.8191518783569336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10375848412513733,
      "logits/rejected": 0.06726669520139694,
      "logps/chosen": -428.236328125,
      "logps/rejected": -289.1847229003906,
      "loss": 0.3397,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19831408560276031,
      "rewards/margins": 1.3305542469024658,
      "rewards/rejected": -1.5288684368133545,
      "step": 3343
    },
    {
      "epoch": 0.43758894250429375,
      "grad_norm": 5.942628860473633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.107656329870224,
      "logits/rejected": 0.1308320164680481,
      "logps/chosen": -463.5582275390625,
      "logps/rejected": -333.234130859375,
      "loss": 0.3633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03914393484592438,
      "rewards/margins": 1.0757337808609009,
      "rewards/rejected": -1.114877700805664,
      "step": 3344
    },
    {
      "epoch": 0.43771980044164555,
      "grad_norm": 2.7228853702545166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08347327262163162,
      "logits/rejected": 0.11329925060272217,
      "logps/chosen": -389.36346435546875,
      "logps/rejected": -532.7501831054688,
      "loss": 0.1874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3172712028026581,
      "rewards/margins": 2.512052059173584,
      "rewards/rejected": -2.8293232917785645,
      "step": 3345
    },
    {
      "epoch": 0.4378506583789973,
      "grad_norm": 4.93681001663208,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17363561689853668,
      "logits/rejected": 0.21036861836910248,
      "logps/chosen": -447.827392578125,
      "logps/rejected": -452.34979248046875,
      "loss": 0.3015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4988737106323242,
      "rewards/margins": 1.7996095418930054,
      "rewards/rejected": -2.29848313331604,
      "step": 3346
    },
    {
      "epoch": 0.4379815163163491,
      "grad_norm": 9.522101402282715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2581433355808258,
      "logits/rejected": 0.2640336751937866,
      "logps/chosen": -482.839599609375,
      "logps/rejected": -423.2860107421875,
      "loss": 0.6327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8963015079498291,
      "rewards/margins": 0.5077098608016968,
      "rewards/rejected": -1.4040113687515259,
      "step": 3347
    },
    {
      "epoch": 0.4381123742537008,
      "grad_norm": 8.52286148071289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12089598178863525,
      "logits/rejected": 0.16116350889205933,
      "logps/chosen": -379.35235595703125,
      "logps/rejected": -338.56964111328125,
      "loss": 0.4819,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7454246282577515,
      "rewards/margins": 0.880851686000824,
      "rewards/rejected": -1.6262764930725098,
      "step": 3348
    },
    {
      "epoch": 0.4382432321910526,
      "grad_norm": 9.857950210571289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13302695751190186,
      "logits/rejected": 0.03976300358772278,
      "logps/chosen": -390.1798400878906,
      "logps/rejected": -369.071044921875,
      "loss": 0.6409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2798964977264404,
      "rewards/margins": 0.6908551454544067,
      "rewards/rejected": -1.9707516431808472,
      "step": 3349
    },
    {
      "epoch": 0.43837409012840434,
      "grad_norm": 6.616501808166504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19281670451164246,
      "logits/rejected": -0.14720873534679413,
      "logps/chosen": -753.6539306640625,
      "logps/rejected": -673.6480712890625,
      "loss": 0.3335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3501833975315094,
      "rewards/margins": 2.4200596809387207,
      "rewards/rejected": -2.069876194000244,
      "step": 3350
    },
    {
      "epoch": 0.43850494806575613,
      "grad_norm": 7.739674091339111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07382439076900482,
      "logits/rejected": -0.2337205559015274,
      "logps/chosen": -494.3871154785156,
      "logps/rejected": -527.2307739257812,
      "loss": 0.3509,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5458369255065918,
      "rewards/margins": 1.4665827751159668,
      "rewards/rejected": -2.0124199390411377,
      "step": 3351
    },
    {
      "epoch": 0.43863580600310786,
      "grad_norm": 4.9734063148498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1815953552722931,
      "logits/rejected": 0.270198792219162,
      "logps/chosen": -620.2713012695312,
      "logps/rejected": -650.8853149414062,
      "loss": 0.3356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.056807711720466614,
      "rewards/margins": 1.364774227142334,
      "rewards/rejected": -1.4215819835662842,
      "step": 3352
    },
    {
      "epoch": 0.43876666394045966,
      "grad_norm": 8.757925033569336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17462855577468872,
      "logits/rejected": -0.06792724132537842,
      "logps/chosen": -410.573486328125,
      "logps/rejected": -433.8682861328125,
      "loss": 0.5982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7783606052398682,
      "rewards/margins": 0.4843643307685852,
      "rewards/rejected": -1.2627248764038086,
      "step": 3353
    },
    {
      "epoch": 0.4388975218778114,
      "grad_norm": 8.573248863220215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35597604513168335,
      "logits/rejected": 0.31256452202796936,
      "logps/chosen": -291.38470458984375,
      "logps/rejected": -350.029052734375,
      "loss": 0.9026,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029919905588030815,
      "rewards/margins": 0.05737924575805664,
      "rewards/rejected": -0.027459323406219482,
      "step": 3354
    },
    {
      "epoch": 0.4390283798151632,
      "grad_norm": 10.703320503234863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5247134566307068,
      "logits/rejected": 0.28272074460983276,
      "logps/chosen": -554.5721435546875,
      "logps/rejected": -542.5586547851562,
      "loss": 0.6407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5901501178741455,
      "rewards/margins": 1.0402981042861938,
      "rewards/rejected": -1.6304481029510498,
      "step": 3355
    },
    {
      "epoch": 0.4391592377525149,
      "grad_norm": 9.025524139404297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.220968097448349,
      "logits/rejected": 0.05281408876180649,
      "logps/chosen": -475.96087646484375,
      "logps/rejected": -479.8128967285156,
      "loss": 0.8406,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.536304771900177,
      "rewards/margins": 0.18123820424079895,
      "rewards/rejected": -0.7175429463386536,
      "step": 3356
    },
    {
      "epoch": 0.4392900956898667,
      "grad_norm": 8.179811477661133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37810492515563965,
      "logits/rejected": 0.27437302470207214,
      "logps/chosen": -674.1581420898438,
      "logps/rejected": -498.36492919921875,
      "loss": 0.5528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48904094099998474,
      "rewards/margins": 0.6661676168441772,
      "rewards/rejected": -1.1552085876464844,
      "step": 3357
    },
    {
      "epoch": 0.43942095362721845,
      "grad_norm": 8.780782699584961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01740282028913498,
      "logits/rejected": 0.013203602284193039,
      "logps/chosen": -421.5583801269531,
      "logps/rejected": -425.8786926269531,
      "loss": 0.7442,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.616715669631958,
      "rewards/margins": 0.13217636942863464,
      "rewards/rejected": -0.7488920092582703,
      "step": 3358
    },
    {
      "epoch": 0.43955181156457024,
      "grad_norm": 8.347519874572754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12842750549316406,
      "logits/rejected": -0.13381065428256989,
      "logps/chosen": -492.4960021972656,
      "logps/rejected": -538.6915283203125,
      "loss": 0.5581,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.448739230632782,
      "rewards/margins": 0.8027275800704956,
      "rewards/rejected": -1.2514667510986328,
      "step": 3359
    },
    {
      "epoch": 0.439682669501922,
      "grad_norm": 8.744401931762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24579086899757385,
      "logits/rejected": 0.05310777947306633,
      "logps/chosen": -462.1074523925781,
      "logps/rejected": -399.82281494140625,
      "loss": 0.7249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7251573801040649,
      "rewards/margins": 0.5651878714561462,
      "rewards/rejected": -1.2903451919555664,
      "step": 3360
    },
    {
      "epoch": 0.43981352743927377,
      "grad_norm": 6.48077917098999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2509380280971527,
      "logits/rejected": 0.27693668007850647,
      "logps/chosen": -406.338134765625,
      "logps/rejected": -383.9999694824219,
      "loss": 0.4127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.221238911151886,
      "rewards/margins": 0.9875195026397705,
      "rewards/rejected": -1.2087583541870117,
      "step": 3361
    },
    {
      "epoch": 0.4399443853766255,
      "grad_norm": 6.434291362762451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17240643501281738,
      "logits/rejected": -0.08849292993545532,
      "logps/chosen": -642.234130859375,
      "logps/rejected": -508.802490234375,
      "loss": 0.4907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5135631561279297,
      "rewards/margins": 0.6132164597511292,
      "rewards/rejected": -0.09965325891971588,
      "step": 3362
    },
    {
      "epoch": 0.44007524331397724,
      "grad_norm": 10.179062843322754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21510271728038788,
      "logits/rejected": 0.0807509571313858,
      "logps/chosen": -452.53216552734375,
      "logps/rejected": -441.018310546875,
      "loss": 0.8385,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.46114271879196167,
      "rewards/margins": 0.6524757146835327,
      "rewards/rejected": -1.1136186122894287,
      "step": 3363
    },
    {
      "epoch": 0.44020610125132903,
      "grad_norm": 5.881530284881592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17081278562545776,
      "logits/rejected": 0.13769777119159698,
      "logps/chosen": -489.6479187011719,
      "logps/rejected": -546.3497314453125,
      "loss": 0.3987,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4117368757724762,
      "rewards/margins": 1.404273509979248,
      "rewards/rejected": -1.8160102367401123,
      "step": 3364
    },
    {
      "epoch": 0.44033695918868077,
      "grad_norm": 8.084052085876465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09155724942684174,
      "logits/rejected": 0.12102588266134262,
      "logps/chosen": -437.9140625,
      "logps/rejected": -532.5750732421875,
      "loss": 0.5652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6445785760879517,
      "rewards/margins": 0.7082813382148743,
      "rewards/rejected": -0.06370276212692261,
      "step": 3365
    },
    {
      "epoch": 0.44046781712603256,
      "grad_norm": 9.69800090789795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07482466846704483,
      "logits/rejected": 0.11448323726654053,
      "logps/chosen": -431.1667785644531,
      "logps/rejected": -427.91619873046875,
      "loss": 0.8282,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39550894498825073,
      "rewards/margins": 0.14425203204154968,
      "rewards/rejected": -0.5397610068321228,
      "step": 3366
    },
    {
      "epoch": 0.4405986750633843,
      "grad_norm": 6.865969181060791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3701985776424408,
      "logits/rejected": 0.12693996727466583,
      "logps/chosen": -453.88482666015625,
      "logps/rejected": -380.8857116699219,
      "loss": 0.6077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18923383951187134,
      "rewards/margins": 0.5776524543762207,
      "rewards/rejected": -0.3884184658527374,
      "step": 3367
    },
    {
      "epoch": 0.4407295330007361,
      "grad_norm": 6.9908576011657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005583629012107849,
      "logits/rejected": -0.34568366408348083,
      "logps/chosen": -440.7255859375,
      "logps/rejected": -391.93804931640625,
      "loss": 0.3797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04112684726715088,
      "rewards/margins": 1.9007787704467773,
      "rewards/rejected": -1.9419057369232178,
      "step": 3368
    },
    {
      "epoch": 0.4408603909380878,
      "grad_norm": 10.399452209472656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08040137588977814,
      "logits/rejected": -0.24134542047977448,
      "logps/chosen": -524.0715942382812,
      "logps/rejected": -523.8595581054688,
      "loss": 0.6159,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7076729536056519,
      "rewards/margins": 0.7358783483505249,
      "rewards/rejected": -1.4435514211654663,
      "step": 3369
    },
    {
      "epoch": 0.4409912488754396,
      "grad_norm": 8.66429615020752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005157398991286755,
      "logits/rejected": -0.1377929002046585,
      "logps/chosen": -460.0751647949219,
      "logps/rejected": -369.0673828125,
      "loss": 0.6048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6927043199539185,
      "rewards/margins": 0.3946285843849182,
      "rewards/rejected": -1.0873329639434814,
      "step": 3370
    },
    {
      "epoch": 0.44112210681279135,
      "grad_norm": 9.363465309143066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28401264548301697,
      "logits/rejected": 0.3384712338447571,
      "logps/chosen": -421.1315612792969,
      "logps/rejected": -433.5522766113281,
      "loss": 0.7132,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5389031767845154,
      "rewards/margins": 0.24429801106452942,
      "rewards/rejected": -0.7832012176513672,
      "step": 3371
    },
    {
      "epoch": 0.44125296475014314,
      "grad_norm": 8.620800971984863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17406556010246277,
      "logits/rejected": 0.1338222175836563,
      "logps/chosen": -436.5840148925781,
      "logps/rejected": -434.9239501953125,
      "loss": 0.6856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1115058958530426,
      "rewards/margins": 0.416253924369812,
      "rewards/rejected": -0.5277597904205322,
      "step": 3372
    },
    {
      "epoch": 0.4413838226874949,
      "grad_norm": 7.002343654632568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019596584141254425,
      "logits/rejected": 0.2220839560031891,
      "logps/chosen": -430.4404296875,
      "logps/rejected": -339.73211669921875,
      "loss": 0.5388,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.544977605342865,
      "rewards/margins": 1.0437191724777222,
      "rewards/rejected": -0.4987415075302124,
      "step": 3373
    },
    {
      "epoch": 0.44151468062484667,
      "grad_norm": 9.702377319335938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13865457475185394,
      "logits/rejected": -0.024348020553588867,
      "logps/chosen": -503.2018737792969,
      "logps/rejected": -290.005859375,
      "loss": 0.7001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29414597153663635,
      "rewards/margins": 0.48226428031921387,
      "rewards/rejected": -0.7764103412628174,
      "step": 3374
    },
    {
      "epoch": 0.4416455385621984,
      "grad_norm": 6.36137056350708,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05033652484416962,
      "logits/rejected": -0.15790638327598572,
      "logps/chosen": -416.1562805175781,
      "logps/rejected": -534.5301513671875,
      "loss": 0.4194,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2858714759349823,
      "rewards/margins": 1.3184058666229248,
      "rewards/rejected": -1.0325343608856201,
      "step": 3375
    },
    {
      "epoch": 0.4417763964995502,
      "grad_norm": 7.263011932373047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17351621389389038,
      "logits/rejected": 0.29557350277900696,
      "logps/chosen": -421.263427734375,
      "logps/rejected": -481.95794677734375,
      "loss": 0.6216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2880970239639282,
      "rewards/margins": 0.7341526746749878,
      "rewards/rejected": -0.4460555911064148,
      "step": 3376
    },
    {
      "epoch": 0.44190725443690193,
      "grad_norm": 2.6000847816467285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09519116580486298,
      "logits/rejected": 0.2510550022125244,
      "logps/chosen": -337.050048828125,
      "logps/rejected": -382.9562683105469,
      "loss": 0.2333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16712933778762817,
      "rewards/margins": 2.091318130493164,
      "rewards/rejected": -1.9241889715194702,
      "step": 3377
    },
    {
      "epoch": 0.4420381123742537,
      "grad_norm": 5.8205037117004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16975048184394836,
      "logits/rejected": 0.02442942187190056,
      "logps/chosen": -429.33447265625,
      "logps/rejected": -502.7115173339844,
      "loss": 0.3299,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26609134674072266,
      "rewards/margins": 1.3202769756317139,
      "rewards/rejected": -1.0541855096817017,
      "step": 3378
    },
    {
      "epoch": 0.44216897031160546,
      "grad_norm": 10.454103469848633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12922343611717224,
      "logits/rejected": 0.1101040467619896,
      "logps/chosen": -420.9770812988281,
      "logps/rejected": -510.3237609863281,
      "loss": 0.5963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6322370171546936,
      "rewards/margins": 0.8921416997909546,
      "rewards/rejected": -1.5243786573410034,
      "step": 3379
    },
    {
      "epoch": 0.44229982824895725,
      "grad_norm": 7.262473106384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2738570272922516,
      "logits/rejected": -0.16085869073867798,
      "logps/chosen": -351.6607666015625,
      "logps/rejected": -399.94903564453125,
      "loss": 0.5642,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6252369284629822,
      "rewards/margins": 0.8556827306747437,
      "rewards/rejected": -1.4809197187423706,
      "step": 3380
    },
    {
      "epoch": 0.442430686186309,
      "grad_norm": 6.642032146453857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29302725195884705,
      "logits/rejected": 0.29226192831993103,
      "logps/chosen": -380.7981872558594,
      "logps/rejected": -390.127197265625,
      "loss": 0.5034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.33035808801651,
      "rewards/margins": 0.7739921808242798,
      "rewards/rejected": -0.4436340928077698,
      "step": 3381
    },
    {
      "epoch": 0.4425615441236608,
      "grad_norm": 10.066058158874512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4438914656639099,
      "logits/rejected": 0.31379422545433044,
      "logps/chosen": -304.34832763671875,
      "logps/rejected": -382.2604064941406,
      "loss": 1.0016,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.7510091662406921,
      "rewards/margins": -0.2839809060096741,
      "rewards/rejected": -0.46702826023101807,
      "step": 3382
    },
    {
      "epoch": 0.4426924020610125,
      "grad_norm": 8.714081764221191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11534962058067322,
      "logits/rejected": -0.08972832560539246,
      "logps/chosen": -540.1488647460938,
      "logps/rejected": -568.5733032226562,
      "loss": 0.7994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3593021631240845,
      "rewards/margins": 0.37173157930374146,
      "rewards/rejected": -0.7310336828231812,
      "step": 3383
    },
    {
      "epoch": 0.44282325999836425,
      "grad_norm": 7.751836776733398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3964442014694214,
      "logits/rejected": 0.369563490152359,
      "logps/chosen": -407.2325439453125,
      "logps/rejected": -456.6864929199219,
      "loss": 0.5932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.037754449993371964,
      "rewards/margins": 0.33918267488479614,
      "rewards/rejected": -0.3769370913505554,
      "step": 3384
    },
    {
      "epoch": 0.44295411793571604,
      "grad_norm": 8.307283401489258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2517963647842407,
      "logits/rejected": -0.3333472013473511,
      "logps/chosen": -367.2412414550781,
      "logps/rejected": -439.5108947753906,
      "loss": 0.6828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5737242698669434,
      "rewards/margins": 0.3326117992401123,
      "rewards/rejected": -1.9063360691070557,
      "step": 3385
    },
    {
      "epoch": 0.4430849758730678,
      "grad_norm": 7.722166538238525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16265961527824402,
      "logits/rejected": 0.18997780978679657,
      "logps/chosen": -473.621826171875,
      "logps/rejected": -582.0790405273438,
      "loss": 0.5355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5270770788192749,
      "rewards/margins": 0.7514297962188721,
      "rewards/rejected": -0.2243526577949524,
      "step": 3386
    },
    {
      "epoch": 0.44321583381041957,
      "grad_norm": 4.19259786605835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054916396737098694,
      "logits/rejected": -0.289011687040329,
      "logps/chosen": -378.4860534667969,
      "logps/rejected": -370.62518310546875,
      "loss": 0.3765,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9596811532974243,
      "rewards/margins": 1.5324280261993408,
      "rewards/rejected": -2.4921090602874756,
      "step": 3387
    },
    {
      "epoch": 0.4433466917477713,
      "grad_norm": 4.9522175788879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28051865100860596,
      "logits/rejected": 0.15938133001327515,
      "logps/chosen": -449.8626403808594,
      "logps/rejected": -750.18896484375,
      "loss": 0.4037,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8887510895729065,
      "rewards/margins": 0.9666403532028198,
      "rewards/rejected": -0.07788926362991333,
      "step": 3388
    },
    {
      "epoch": 0.4434775496851231,
      "grad_norm": 3.2801549434661865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06415559351444244,
      "logits/rejected": -0.015208769589662552,
      "logps/chosen": -337.071044921875,
      "logps/rejected": -328.89678955078125,
      "loss": 0.2356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34630873799324036,
      "rewards/margins": 1.479215145111084,
      "rewards/rejected": -1.132906436920166,
      "step": 3389
    },
    {
      "epoch": 0.44360840762247483,
      "grad_norm": 9.285623550415039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21348628401756287,
      "logits/rejected": 0.2972698509693146,
      "logps/chosen": -464.7706298828125,
      "logps/rejected": -500.33502197265625,
      "loss": 0.7744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5451481342315674,
      "rewards/margins": 0.1736290156841278,
      "rewards/rejected": -0.718777060508728,
      "step": 3390
    },
    {
      "epoch": 0.4437392655598266,
      "grad_norm": 11.306746482849121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13451433181762695,
      "logits/rejected": -0.10493644326925278,
      "logps/chosen": -614.733154296875,
      "logps/rejected": -527.690673828125,
      "loss": 0.5512,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16085119545459747,
      "rewards/margins": 0.784254789352417,
      "rewards/rejected": -0.6234035491943359,
      "step": 3391
    },
    {
      "epoch": 0.44387012349717836,
      "grad_norm": 9.629056930541992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05992061644792557,
      "logits/rejected": 0.1238628625869751,
      "logps/chosen": -494.8819274902344,
      "logps/rejected": -505.275634765625,
      "loss": 0.6145,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3224165439605713,
      "rewards/margins": 0.34731215238571167,
      "rewards/rejected": -0.6697286367416382,
      "step": 3392
    },
    {
      "epoch": 0.44400098143453015,
      "grad_norm": 6.422051906585693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15333467721939087,
      "logits/rejected": 0.18802231550216675,
      "logps/chosen": -364.0469665527344,
      "logps/rejected": -320.7397766113281,
      "loss": 0.4572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10661261528730392,
      "rewards/margins": 0.9710023999214172,
      "rewards/rejected": -1.0776150226593018,
      "step": 3393
    },
    {
      "epoch": 0.4441318393718819,
      "grad_norm": 4.647985935211182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03454762324690819,
      "logits/rejected": 0.0868842676281929,
      "logps/chosen": -349.45281982421875,
      "logps/rejected": -513.7862548828125,
      "loss": 0.3554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49681970477104187,
      "rewards/margins": 1.1410903930664062,
      "rewards/rejected": -0.6442707180976868,
      "step": 3394
    },
    {
      "epoch": 0.4442626973092337,
      "grad_norm": 7.424475193023682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14146697521209717,
      "logits/rejected": 0.09595263004302979,
      "logps/chosen": -496.2708740234375,
      "logps/rejected": -447.3145751953125,
      "loss": 0.3885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1816398650407791,
      "rewards/margins": 1.415193796157837,
      "rewards/rejected": -1.59683358669281,
      "step": 3395
    },
    {
      "epoch": 0.4443935552465854,
      "grad_norm": 9.414769172668457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10062175989151001,
      "logits/rejected": -0.12678350508213043,
      "logps/chosen": -405.771240234375,
      "logps/rejected": -473.1485595703125,
      "loss": 0.8964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.3692466616630554,
      "rewards/margins": 0.2941405475139618,
      "rewards/rejected": 0.07510611414909363,
      "step": 3396
    },
    {
      "epoch": 0.4445244131839372,
      "grad_norm": 9.072442054748535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23414181172847748,
      "logits/rejected": -0.24670718610286713,
      "logps/chosen": -294.5906982421875,
      "logps/rejected": -321.4820861816406,
      "loss": 0.612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0143080949783325,
      "rewards/margins": 0.5935701131820679,
      "rewards/rejected": -1.60787832736969,
      "step": 3397
    },
    {
      "epoch": 0.44465527112128894,
      "grad_norm": 8.528257369995117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13092371821403503,
      "logits/rejected": 0.08999307453632355,
      "logps/chosen": -507.2481994628906,
      "logps/rejected": -574.2146606445312,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6316531896591187,
      "rewards/margins": 1.7041873931884766,
      "rewards/rejected": -1.0725340843200684,
      "step": 3398
    },
    {
      "epoch": 0.44478612905864073,
      "grad_norm": 7.424252510070801,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05908362194895744,
      "logits/rejected": 0.22921255230903625,
      "logps/chosen": -261.2000427246094,
      "logps/rejected": -301.06610107421875,
      "loss": 0.521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06438273936510086,
      "rewards/margins": 1.1843923330307007,
      "rewards/rejected": -1.1200097799301147,
      "step": 3399
    },
    {
      "epoch": 0.44491698699599247,
      "grad_norm": 7.865713119506836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.038939327001571655,
      "logits/rejected": 0.07562947273254395,
      "logps/chosen": -356.6321105957031,
      "logps/rejected": -335.4582214355469,
      "loss": 0.6054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09374197572469711,
      "rewards/margins": 0.5630297660827637,
      "rewards/rejected": -0.46928781270980835,
      "step": 3400
    },
    {
      "epoch": 0.44504784493334426,
      "grad_norm": 7.987987995147705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10401800274848938,
      "logits/rejected": 0.20617780089378357,
      "logps/chosen": -514.2216796875,
      "logps/rejected": -532.1547241210938,
      "loss": 0.5244,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25748157501220703,
      "rewards/margins": 1.0140416622161865,
      "rewards/rejected": -1.2715229988098145,
      "step": 3401
    },
    {
      "epoch": 0.445178702870696,
      "grad_norm": 6.002937316894531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03599634766578674,
      "logits/rejected": 0.06413327902555466,
      "logps/chosen": -375.6700744628906,
      "logps/rejected": -464.16387939453125,
      "loss": 0.4123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7897060513496399,
      "rewards/margins": 1.3773419857025146,
      "rewards/rejected": -2.167048215866089,
      "step": 3402
    },
    {
      "epoch": 0.4453095608080478,
      "grad_norm": 15.328454971313477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1239730566740036,
      "logits/rejected": -0.08065919578075409,
      "logps/chosen": -539.714111328125,
      "logps/rejected": -495.9305114746094,
      "loss": 0.8797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37983131408691406,
      "rewards/margins": 0.4510783851146698,
      "rewards/rejected": -0.8309097290039062,
      "step": 3403
    },
    {
      "epoch": 0.4454404187453995,
      "grad_norm": 4.579298496246338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20125234127044678,
      "logits/rejected": 0.10857369750738144,
      "logps/chosen": -319.7685852050781,
      "logps/rejected": -321.4775085449219,
      "loss": 0.35,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7394751310348511,
      "rewards/margins": 1.3006525039672852,
      "rewards/rejected": -2.0401275157928467,
      "step": 3404
    },
    {
      "epoch": 0.44557127668275126,
      "grad_norm": 6.210498332977295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.062122222036123276,
      "logits/rejected": -0.034002870321273804,
      "logps/chosen": -249.33975219726562,
      "logps/rejected": -341.46673583984375,
      "loss": 0.499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7755066156387329,
      "rewards/margins": 0.8229013681411743,
      "rewards/rejected": -1.5984079837799072,
      "step": 3405
    },
    {
      "epoch": 0.44570213462010305,
      "grad_norm": 7.02575159072876,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1748981475830078,
      "logits/rejected": 0.005350150167942047,
      "logps/chosen": -411.7735900878906,
      "logps/rejected": -295.9138488769531,
      "loss": 0.4401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5189289450645447,
      "rewards/margins": 1.0403642654418945,
      "rewards/rejected": -1.5592931509017944,
      "step": 3406
    },
    {
      "epoch": 0.4458329925574548,
      "grad_norm": 9.867073059082031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.055315203964710236,
      "logits/rejected": -0.0385582372546196,
      "logps/chosen": -439.89642333984375,
      "logps/rejected": -438.47674560546875,
      "loss": 1.0559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0036962032318115,
      "rewards/margins": -0.12215560674667358,
      "rewards/rejected": -0.8815406560897827,
      "step": 3407
    },
    {
      "epoch": 0.4459638504948066,
      "grad_norm": 9.557064056396484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19720160961151123,
      "logits/rejected": -0.13743993639945984,
      "logps/chosen": -461.1870422363281,
      "logps/rejected": -392.4922180175781,
      "loss": 0.5426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5999590158462524,
      "rewards/margins": 0.7062839865684509,
      "rewards/rejected": -1.3062430620193481,
      "step": 3408
    },
    {
      "epoch": 0.4460947084321583,
      "grad_norm": 6.032750606536865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30529579520225525,
      "logits/rejected": -0.04593318700790405,
      "logps/chosen": -584.8980712890625,
      "logps/rejected": -440.75006103515625,
      "loss": 0.3567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5677403211593628,
      "rewards/margins": 1.6775259971618652,
      "rewards/rejected": -1.1097856760025024,
      "step": 3409
    },
    {
      "epoch": 0.4462255663695101,
      "grad_norm": 8.222356796264648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18521170318126678,
      "logits/rejected": 0.2584432363510132,
      "logps/chosen": -424.8572692871094,
      "logps/rejected": -520.4590454101562,
      "loss": 0.5538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.122441865503788,
      "rewards/margins": 0.386130154132843,
      "rewards/rejected": -0.26368826627731323,
      "step": 3410
    },
    {
      "epoch": 0.44635642430686184,
      "grad_norm": 5.737516403198242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.105890654027462,
      "logits/rejected": -0.029816947877407074,
      "logps/chosen": -444.05426025390625,
      "logps/rejected": -496.2192687988281,
      "loss": 0.3518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2827894389629364,
      "rewards/margins": 1.4916757345199585,
      "rewards/rejected": -1.7744653224945068,
      "step": 3411
    },
    {
      "epoch": 0.44648728224421363,
      "grad_norm": 7.048180103302002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28346627950668335,
      "logits/rejected": -0.2987859845161438,
      "logps/chosen": -412.6672668457031,
      "logps/rejected": -448.5394287109375,
      "loss": 0.4139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3998680114746094,
      "rewards/margins": 1.3441352844238281,
      "rewards/rejected": -1.7440035343170166,
      "step": 3412
    },
    {
      "epoch": 0.44661814018156537,
      "grad_norm": 8.123477935791016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.163655623793602,
      "logits/rejected": -0.04711263254284859,
      "logps/chosen": -408.8089904785156,
      "logps/rejected": -552.4375,
      "loss": 0.6421,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7129531502723694,
      "rewards/margins": 0.18212664127349854,
      "rewards/rejected": -0.8950797915458679,
      "step": 3413
    },
    {
      "epoch": 0.44674899811891716,
      "grad_norm": 8.852299690246582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21506062150001526,
      "logits/rejected": -0.05151711776852608,
      "logps/chosen": -297.2169189453125,
      "logps/rejected": -339.63568115234375,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08751163631677628,
      "rewards/margins": 0.5630512237548828,
      "rewards/rejected": -0.6505628824234009,
      "step": 3414
    },
    {
      "epoch": 0.4468798560562689,
      "grad_norm": 5.964916229248047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4381062090396881,
      "logits/rejected": 0.2563667297363281,
      "logps/chosen": -473.5765380859375,
      "logps/rejected": -360.41217041015625,
      "loss": 0.3221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7424415349960327,
      "rewards/margins": 1.3899343013763428,
      "rewards/rejected": -0.6474928855895996,
      "step": 3415
    },
    {
      "epoch": 0.4470107139936207,
      "grad_norm": 8.289311408996582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023651819676160812,
      "logits/rejected": -0.20863790810108185,
      "logps/chosen": -426.41259765625,
      "logps/rejected": -442.1773986816406,
      "loss": 0.4689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.860634982585907,
      "rewards/margins": 1.1585954427719116,
      "rewards/rejected": -2.019230604171753,
      "step": 3416
    },
    {
      "epoch": 0.4471415719309724,
      "grad_norm": 6.68959379196167,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3329126238822937,
      "logits/rejected": 0.2743547856807709,
      "logps/chosen": -547.845458984375,
      "logps/rejected": -495.6904296875,
      "loss": 0.3762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22387076914310455,
      "rewards/margins": 1.1906065940856934,
      "rewards/rejected": -1.4144773483276367,
      "step": 3417
    },
    {
      "epoch": 0.4472724298683242,
      "grad_norm": 3.652710437774658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012724022381007671,
      "logits/rejected": 0.0014914646744728088,
      "logps/chosen": -383.01910400390625,
      "logps/rejected": -393.2162170410156,
      "loss": 0.324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04605443775653839,
      "rewards/margins": 2.177126169204712,
      "rewards/rejected": -2.2231805324554443,
      "step": 3418
    },
    {
      "epoch": 0.44740328780567595,
      "grad_norm": 6.2457804679870605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13256600499153137,
      "logits/rejected": -0.006685703992843628,
      "logps/chosen": -498.7494201660156,
      "logps/rejected": -719.9149169921875,
      "loss": 0.3565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4985910654067993,
      "rewards/margins": 1.048288345336914,
      "rewards/rejected": -0.5496973991394043,
      "step": 3419
    },
    {
      "epoch": 0.44753414574302774,
      "grad_norm": 6.671191215515137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07233917713165283,
      "logits/rejected": -0.09298305213451385,
      "logps/chosen": -535.7860717773438,
      "logps/rejected": -442.88726806640625,
      "loss": 0.4257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0576368123292923,
      "rewards/margins": 0.8469067215919495,
      "rewards/rejected": -0.7892699241638184,
      "step": 3420
    },
    {
      "epoch": 0.4476650036803795,
      "grad_norm": 8.422860145568848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5071296095848083,
      "logits/rejected": 0.1300894170999527,
      "logps/chosen": -476.4001159667969,
      "logps/rejected": -539.1544799804688,
      "loss": 0.4994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3369274139404297,
      "rewards/margins": 0.7276420593261719,
      "rewards/rejected": -1.0645694732666016,
      "step": 3421
    },
    {
      "epoch": 0.4477958616177313,
      "grad_norm": 8.692827224731445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23711217939853668,
      "logits/rejected": 0.41145992279052734,
      "logps/chosen": -546.6148071289062,
      "logps/rejected": -481.36175537109375,
      "loss": 0.7088,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06778315454721451,
      "rewards/margins": 0.6443449258804321,
      "rewards/rejected": -0.5765616297721863,
      "step": 3422
    },
    {
      "epoch": 0.447926719555083,
      "grad_norm": 7.527719974517822,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07149076461791992,
      "logits/rejected": 0.10997961461544037,
      "logps/chosen": -348.9647521972656,
      "logps/rejected": -453.8867492675781,
      "loss": 0.708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02098315954208374,
      "rewards/margins": 0.42676258087158203,
      "rewards/rejected": -0.44774574041366577,
      "step": 3423
    },
    {
      "epoch": 0.4480575774924348,
      "grad_norm": 6.541527271270752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3613514006137848,
      "logits/rejected": -0.12200000882148743,
      "logps/chosen": -332.8027648925781,
      "logps/rejected": -391.90802001953125,
      "loss": 0.3769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.49849531054496765,
      "rewards/margins": 1.327004075050354,
      "rewards/rejected": -0.828508734703064,
      "step": 3424
    },
    {
      "epoch": 0.44818843542978654,
      "grad_norm": 3.555479049682617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17667636275291443,
      "logits/rejected": 0.1397193968296051,
      "logps/chosen": -379.852294921875,
      "logps/rejected": -384.99090576171875,
      "loss": 0.2911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2863723635673523,
      "rewards/margins": 1.416624903678894,
      "rewards/rejected": -1.130252480506897,
      "step": 3425
    },
    {
      "epoch": 0.4483192933671383,
      "grad_norm": 7.051976680755615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33867037296295166,
      "logits/rejected": 0.2677899897098541,
      "logps/chosen": -507.9377136230469,
      "logps/rejected": -477.3304138183594,
      "loss": 0.4241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5066831707954407,
      "rewards/margins": 1.009161353111267,
      "rewards/rejected": -0.5024782419204712,
      "step": 3426
    },
    {
      "epoch": 0.44845015130449006,
      "grad_norm": 5.0556864738464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08013857156038284,
      "logits/rejected": 0.0013455133885145187,
      "logps/chosen": -375.4833679199219,
      "logps/rejected": -281.5362243652344,
      "loss": 0.4854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.37947309017181396,
      "rewards/margins": 0.8745682835578918,
      "rewards/rejected": -0.4950951635837555,
      "step": 3427
    },
    {
      "epoch": 0.4485810092418418,
      "grad_norm": 15.824376106262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0006165280938148499,
      "logits/rejected": 0.0077757928520441055,
      "logps/chosen": -504.0599060058594,
      "logps/rejected": -356.4503479003906,
      "loss": 0.8616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.04830002784729,
      "rewards/margins": 0.49093034863471985,
      "rewards/rejected": -1.5392303466796875,
      "step": 3428
    },
    {
      "epoch": 0.4487118671791936,
      "grad_norm": 6.156388759613037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11346229165792465,
      "logits/rejected": 0.2508399486541748,
      "logps/chosen": -576.5057983398438,
      "logps/rejected": -511.81927490234375,
      "loss": 0.6125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30018576979637146,
      "rewards/margins": 0.5114538669586182,
      "rewards/rejected": -0.811639666557312,
      "step": 3429
    },
    {
      "epoch": 0.4488427251165453,
      "grad_norm": 5.915779113769531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03967589884996414,
      "logits/rejected": -0.03518913313746452,
      "logps/chosen": -412.1750793457031,
      "logps/rejected": -373.08966064453125,
      "loss": 0.4139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11689245700836182,
      "rewards/margins": 1.1235557794570923,
      "rewards/rejected": -1.240448236465454,
      "step": 3430
    },
    {
      "epoch": 0.4489735830538971,
      "grad_norm": 9.597101211547852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03135935217142105,
      "logits/rejected": 0.009742319583892822,
      "logps/chosen": -597.1442260742188,
      "logps/rejected": -480.2591247558594,
      "loss": 0.7744,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0821615606546402,
      "rewards/margins": 0.8088066577911377,
      "rewards/rejected": -0.8909682035446167,
      "step": 3431
    },
    {
      "epoch": 0.44910444099124885,
      "grad_norm": 9.479765892028809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12527374923229218,
      "logits/rejected": -0.10909894108772278,
      "logps/chosen": -482.01934814453125,
      "logps/rejected": -634.4005126953125,
      "loss": 0.7328,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0750928744673729,
      "rewards/margins": 0.3812589645385742,
      "rewards/rejected": -0.30616605281829834,
      "step": 3432
    },
    {
      "epoch": 0.44923529892860065,
      "grad_norm": 4.34959077835083,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16450530290603638,
      "logits/rejected": 0.17023271322250366,
      "logps/chosen": -411.4736022949219,
      "logps/rejected": -425.6653137207031,
      "loss": 0.2071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11460438370704651,
      "rewards/margins": 1.6017441749572754,
      "rewards/rejected": -1.4871399402618408,
      "step": 3433
    },
    {
      "epoch": 0.4493661568659524,
      "grad_norm": 5.67297887802124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08505714684724808,
      "logits/rejected": -0.012385427951812744,
      "logps/chosen": -404.42840576171875,
      "logps/rejected": -515.146484375,
      "loss": 0.454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6177944540977478,
      "rewards/margins": 1.308616042137146,
      "rewards/rejected": -1.9264105558395386,
      "step": 3434
    },
    {
      "epoch": 0.4494970148033042,
      "grad_norm": 3.271322011947632,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020119979977607727,
      "logits/rejected": 0.05425591766834259,
      "logps/chosen": -517.3269653320312,
      "logps/rejected": -385.3448181152344,
      "loss": 0.2314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25603577494621277,
      "rewards/margins": 1.8579472303390503,
      "rewards/rejected": -1.6019113063812256,
      "step": 3435
    },
    {
      "epoch": 0.4496278727406559,
      "grad_norm": 7.053824424743652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0033166836947202682,
      "logits/rejected": -0.136576846241951,
      "logps/chosen": -635.31982421875,
      "logps/rejected": -634.9521484375,
      "loss": 0.408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08114089071750641,
      "rewards/margins": 0.9067897796630859,
      "rewards/rejected": -0.8256489038467407,
      "step": 3436
    },
    {
      "epoch": 0.4497587306780077,
      "grad_norm": 6.2443671226501465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2044563591480255,
      "logits/rejected": 0.10383488237857819,
      "logps/chosen": -481.6739807128906,
      "logps/rejected": -448.1123046875,
      "loss": 0.4176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3682763874530792,
      "rewards/margins": 1.1990853548049927,
      "rewards/rejected": -0.8308089971542358,
      "step": 3437
    },
    {
      "epoch": 0.44988958861535944,
      "grad_norm": 13.75097942352295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.418631374835968,
      "logits/rejected": 0.19173841178417206,
      "logps/chosen": -538.5184936523438,
      "logps/rejected": -467.8621826171875,
      "loss": 0.7468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.020759940147399902,
      "rewards/margins": 0.6123096942901611,
      "rewards/rejected": -0.633069634437561,
      "step": 3438
    },
    {
      "epoch": 0.45002044655271123,
      "grad_norm": 9.286176681518555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023013530299067497,
      "logits/rejected": 0.05404849722981453,
      "logps/chosen": -378.0674743652344,
      "logps/rejected": -472.047119140625,
      "loss": 0.7778,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24195006489753723,
      "rewards/margins": 0.12777480483055115,
      "rewards/rejected": -0.3697248697280884,
      "step": 3439
    },
    {
      "epoch": 0.45015130449006296,
      "grad_norm": 12.216876983642578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08347740024328232,
      "logits/rejected": -0.1506638377904892,
      "logps/chosen": -354.2333068847656,
      "logps/rejected": -404.19305419921875,
      "loss": 0.6597,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8306106925010681,
      "rewards/margins": 0.5441625714302063,
      "rewards/rejected": -1.3747732639312744,
      "step": 3440
    },
    {
      "epoch": 0.45028216242741476,
      "grad_norm": 8.020554542541504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10740035772323608,
      "logits/rejected": 0.10612599551677704,
      "logps/chosen": -567.8516235351562,
      "logps/rejected": -340.46197509765625,
      "loss": 0.6122,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14809897541999817,
      "rewards/margins": 0.6588573455810547,
      "rewards/rejected": -0.5107583403587341,
      "step": 3441
    },
    {
      "epoch": 0.4504130203647665,
      "grad_norm": 7.070685386657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024074114859104156,
      "logits/rejected": -0.13303476572036743,
      "logps/chosen": -546.2030029296875,
      "logps/rejected": -507.9822998046875,
      "loss": 0.488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32360023260116577,
      "rewards/margins": 0.6134942770004272,
      "rewards/rejected": -0.9370945692062378,
      "step": 3442
    },
    {
      "epoch": 0.4505438783021183,
      "grad_norm": 9.764594078063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025432050228118896,
      "logits/rejected": -0.06747383624315262,
      "logps/chosen": -407.27099609375,
      "logps/rejected": -419.17474365234375,
      "loss": 0.7472,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1577049195766449,
      "rewards/margins": 0.20711174607276917,
      "rewards/rejected": -0.36481669545173645,
      "step": 3443
    },
    {
      "epoch": 0.45067473623947,
      "grad_norm": 6.176384925842285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13312488794326782,
      "logits/rejected": -0.07850493490695953,
      "logps/chosen": -523.1296997070312,
      "logps/rejected": -474.55975341796875,
      "loss": 0.3299,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5328630805015564,
      "rewards/margins": 1.581963062286377,
      "rewards/rejected": -1.0490999221801758,
      "step": 3444
    },
    {
      "epoch": 0.4508055941768218,
      "grad_norm": 6.6965203285217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0063463374972343445,
      "logits/rejected": -0.16714254021644592,
      "logps/chosen": -611.47119140625,
      "logps/rejected": -541.5615844726562,
      "loss": 0.5118,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.241276353597641,
      "rewards/margins": 1.0658529996871948,
      "rewards/rejected": -1.3071293830871582,
      "step": 3445
    },
    {
      "epoch": 0.45093645211417355,
      "grad_norm": 8.454811096191406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13692784309387207,
      "logits/rejected": 0.1519811451435089,
      "logps/chosen": -347.6637878417969,
      "logps/rejected": -358.15838623046875,
      "loss": 0.6637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2677951753139496,
      "rewards/margins": 0.196061909198761,
      "rewards/rejected": -0.46385711431503296,
      "step": 3446
    },
    {
      "epoch": 0.45106731005152534,
      "grad_norm": 10.427197456359863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4447324275970459,
      "logits/rejected": 0.36765754222869873,
      "logps/chosen": -523.5756225585938,
      "logps/rejected": -538.7761840820312,
      "loss": 0.7649,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2454025745391846,
      "rewards/margins": 0.03238102048635483,
      "rewards/rejected": -1.277783751487732,
      "step": 3447
    },
    {
      "epoch": 0.4511981679888771,
      "grad_norm": 5.921469688415527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15716692805290222,
      "logits/rejected": -0.004281339235603809,
      "logps/chosen": -347.1413269042969,
      "logps/rejected": -514.9610595703125,
      "loss": 0.461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2125270813703537,
      "rewards/margins": 1.3861905336380005,
      "rewards/rejected": -1.1736633777618408,
      "step": 3448
    },
    {
      "epoch": 0.4513290259262288,
      "grad_norm": 6.550538539886475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007278501987457275,
      "logits/rejected": 0.1256510317325592,
      "logps/chosen": -692.19580078125,
      "logps/rejected": -579.9126586914062,
      "loss": 0.4038,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09611111879348755,
      "rewards/margins": 1.1254175901412964,
      "rewards/rejected": -1.2215287685394287,
      "step": 3449
    },
    {
      "epoch": 0.4514598838635806,
      "grad_norm": 3.575288772583008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06111983209848404,
      "logits/rejected": 0.0219663605093956,
      "logps/chosen": -440.59564208984375,
      "logps/rejected": -484.8648376464844,
      "loss": 0.1809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5078538656234741,
      "rewards/margins": 2.001235008239746,
      "rewards/rejected": -1.4933809041976929,
      "step": 3450
    },
    {
      "epoch": 0.45159074180093234,
      "grad_norm": 6.896528720855713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12394773215055466,
      "logits/rejected": -0.16199445724487305,
      "logps/chosen": -582.3361206054688,
      "logps/rejected": -429.6203918457031,
      "loss": 0.5235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5876915454864502,
      "rewards/margins": 0.7525200843811035,
      "rewards/rejected": -0.16482846438884735,
      "step": 3451
    },
    {
      "epoch": 0.45172159973828413,
      "grad_norm": 7.200247764587402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21396060287952423,
      "logits/rejected": -0.16129249334335327,
      "logps/chosen": -560.650634765625,
      "logps/rejected": -488.6046142578125,
      "loss": 0.4492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3388042449951172,
      "rewards/margins": 1.371996283531189,
      "rewards/rejected": -1.0331920385360718,
      "step": 3452
    },
    {
      "epoch": 0.45185245767563587,
      "grad_norm": 5.062640190124512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20637790858745575,
      "logits/rejected": 0.04227394610643387,
      "logps/chosen": -493.4884033203125,
      "logps/rejected": -400.938720703125,
      "loss": 0.3408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1765950918197632,
      "rewards/margins": 1.551866888999939,
      "rewards/rejected": -0.37527182698249817,
      "step": 3453
    },
    {
      "epoch": 0.45198331561298766,
      "grad_norm": 5.753418922424316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010694805532693863,
      "logits/rejected": -0.16579361259937286,
      "logps/chosen": -408.0881042480469,
      "logps/rejected": -448.6030578613281,
      "loss": 0.3373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18379761278629303,
      "rewards/margins": 1.7994513511657715,
      "rewards/rejected": -1.9832489490509033,
      "step": 3454
    },
    {
      "epoch": 0.4521141735503394,
      "grad_norm": 5.0551676750183105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17867113649845123,
      "logits/rejected": 0.2818601131439209,
      "logps/chosen": -589.1646118164062,
      "logps/rejected": -590.501220703125,
      "loss": 0.2908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8377302885055542,
      "rewards/margins": 1.605827808380127,
      "rewards/rejected": -0.7680975198745728,
      "step": 3455
    },
    {
      "epoch": 0.4522450314876912,
      "grad_norm": 8.25748062133789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06152176484465599,
      "logits/rejected": -0.2797542214393616,
      "logps/chosen": -526.5633544921875,
      "logps/rejected": -520.9021606445312,
      "loss": 0.533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08139437437057495,
      "rewards/margins": 0.8239015340805054,
      "rewards/rejected": -0.7425071597099304,
      "step": 3456
    },
    {
      "epoch": 0.4523758894250429,
      "grad_norm": 4.024947643280029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42589813470840454,
      "logits/rejected": -0.02679591439664364,
      "logps/chosen": -595.3914184570312,
      "logps/rejected": -443.36895751953125,
      "loss": 0.2502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24116991460323334,
      "rewards/margins": 2.2414417266845703,
      "rewards/rejected": -2.000272035598755,
      "step": 3457
    },
    {
      "epoch": 0.4525067473623947,
      "grad_norm": 10.074856758117676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17792312800884247,
      "logits/rejected": 0.12555283308029175,
      "logps/chosen": -370.8470764160156,
      "logps/rejected": -460.6775207519531,
      "loss": 0.7856,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3961412906646729,
      "rewards/margins": 0.09478207677602768,
      "rewards/rejected": -1.490923285484314,
      "step": 3458
    },
    {
      "epoch": 0.45263760529974645,
      "grad_norm": 3.947643756866455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0021075978875160217,
      "logits/rejected": 0.005555197596549988,
      "logps/chosen": -623.5762329101562,
      "logps/rejected": -673.4622802734375,
      "loss": 0.3485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3010280132293701,
      "rewards/margins": 1.2672699689865112,
      "rewards/rejected": 0.03375817462801933,
      "step": 3459
    },
    {
      "epoch": 0.45276846323709824,
      "grad_norm": 13.395421981811523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011939868330955505,
      "logits/rejected": -0.240159809589386,
      "logps/chosen": -491.01800537109375,
      "logps/rejected": -422.7444763183594,
      "loss": 0.9984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1641172468662262,
      "rewards/margins": -0.08770237863063812,
      "rewards/rejected": 0.2518196105957031,
      "step": 3460
    },
    {
      "epoch": 0.45289932117445,
      "grad_norm": 6.95190954208374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0705164447426796,
      "logits/rejected": 0.1728539615869522,
      "logps/chosen": -476.5758361816406,
      "logps/rejected": -415.0198974609375,
      "loss": 0.5342,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.464251309633255,
      "rewards/margins": 0.889107882976532,
      "rewards/rejected": -1.3533592224121094,
      "step": 3461
    },
    {
      "epoch": 0.45303017911180177,
      "grad_norm": 4.561653137207031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13134753704071045,
      "logits/rejected": 0.17249780893325806,
      "logps/chosen": -644.84716796875,
      "logps/rejected": -475.87384033203125,
      "loss": 0.2409,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42391854524612427,
      "rewards/margins": 1.96097993850708,
      "rewards/rejected": -1.5370615720748901,
      "step": 3462
    },
    {
      "epoch": 0.4531610370491535,
      "grad_norm": 2.846465826034546,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06714732199907303,
      "logits/rejected": -0.02921571582555771,
      "logps/chosen": -544.4241333007812,
      "logps/rejected": -435.58514404296875,
      "loss": 0.1715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35135555267333984,
      "rewards/margins": 1.8698911666870117,
      "rewards/rejected": -1.5185354948043823,
      "step": 3463
    },
    {
      "epoch": 0.4532918949865053,
      "grad_norm": 8.315167427062988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12315624952316284,
      "logits/rejected": -0.015226747840642929,
      "logps/chosen": -302.4952392578125,
      "logps/rejected": -342.971923828125,
      "loss": 0.6653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.032295405864715576,
      "rewards/margins": 0.1015741378068924,
      "rewards/rejected": -0.06927872449159622,
      "step": 3464
    },
    {
      "epoch": 0.45342275292385703,
      "grad_norm": 9.351774215698242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0992971807718277,
      "logits/rejected": -0.07296881079673767,
      "logps/chosen": -669.9815063476562,
      "logps/rejected": -511.4188537597656,
      "loss": 0.4863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7683460712432861,
      "rewards/margins": 1.0086121559143066,
      "rewards/rejected": -0.24026603996753693,
      "step": 3465
    },
    {
      "epoch": 0.4535536108612088,
      "grad_norm": 6.260478973388672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030061054974794388,
      "logits/rejected": -0.5547492504119873,
      "logps/chosen": -243.42095947265625,
      "logps/rejected": -371.8289489746094,
      "loss": 0.5632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22340160608291626,
      "rewards/margins": 0.7067875862121582,
      "rewards/rejected": -0.930189311504364,
      "step": 3466
    },
    {
      "epoch": 0.45368446879856056,
      "grad_norm": 6.089876174926758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07566733658313751,
      "logits/rejected": -0.20775994658470154,
      "logps/chosen": -399.793701171875,
      "logps/rejected": -350.89501953125,
      "loss": 0.3661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9605637192726135,
      "rewards/margins": 1.0973503589630127,
      "rewards/rejected": -2.0579140186309814,
      "step": 3467
    },
    {
      "epoch": 0.45381532673591235,
      "grad_norm": 5.662001132965088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07969451695680618,
      "logits/rejected": 0.0353403240442276,
      "logps/chosen": -463.05755615234375,
      "logps/rejected": -454.4510498046875,
      "loss": 0.3692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4235174059867859,
      "rewards/margins": 1.577531099319458,
      "rewards/rejected": -1.1540136337280273,
      "step": 3468
    },
    {
      "epoch": 0.4539461846732641,
      "grad_norm": 10.499349594116211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3009587526321411,
      "logits/rejected": -0.30286043882369995,
      "logps/chosen": -676.7801513671875,
      "logps/rejected": -428.7946472167969,
      "loss": 0.6234,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7854543328285217,
      "rewards/margins": 0.5142315626144409,
      "rewards/rejected": -1.2996858358383179,
      "step": 3469
    },
    {
      "epoch": 0.4540770426106159,
      "grad_norm": 8.081916809082031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.001271551474928856,
      "logits/rejected": 0.0016613304615020752,
      "logps/chosen": -369.9013366699219,
      "logps/rejected": -371.76031494140625,
      "loss": 0.7147,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5432438254356384,
      "rewards/margins": 0.04152965545654297,
      "rewards/rejected": -0.5847734808921814,
      "step": 3470
    },
    {
      "epoch": 0.4542079005479676,
      "grad_norm": 8.29721450805664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1492941528558731,
      "logits/rejected": -0.17951937019824982,
      "logps/chosen": -367.990234375,
      "logps/rejected": -424.27880859375,
      "loss": 0.5476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7161315679550171,
      "rewards/margins": 0.6064815521240234,
      "rewards/rejected": -1.322613000869751,
      "step": 3471
    },
    {
      "epoch": 0.45433875848531935,
      "grad_norm": 5.792232990264893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06859257817268372,
      "logits/rejected": -0.019757844507694244,
      "logps/chosen": -414.2165832519531,
      "logps/rejected": -412.8314208984375,
      "loss": 0.4004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5914060473442078,
      "rewards/margins": 0.8922210335731506,
      "rewards/rejected": -1.483627200126648,
      "step": 3472
    },
    {
      "epoch": 0.45446961642267114,
      "grad_norm": 6.401968479156494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15960454940795898,
      "logits/rejected": -0.06124277785420418,
      "logps/chosen": -373.3864440917969,
      "logps/rejected": -418.11474609375,
      "loss": 0.4541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2205567359924316,
      "rewards/margins": 1.2248963117599487,
      "rewards/rejected": -2.445452928543091,
      "step": 3473
    },
    {
      "epoch": 0.4546004743600229,
      "grad_norm": 9.349246978759766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18324948847293854,
      "logits/rejected": -0.0021783187985420227,
      "logps/chosen": -445.5583190917969,
      "logps/rejected": -428.2680969238281,
      "loss": 0.6993,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6125698089599609,
      "rewards/margins": 0.574967622756958,
      "rewards/rejected": -1.187537431716919,
      "step": 3474
    },
    {
      "epoch": 0.45473133229737467,
      "grad_norm": 8.056456565856934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028215643018484116,
      "logits/rejected": -0.029611658304929733,
      "logps/chosen": -722.6234741210938,
      "logps/rejected": -666.4873046875,
      "loss": 0.3844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6064245104789734,
      "rewards/margins": 1.6619915962219238,
      "rewards/rejected": -2.268416166305542,
      "step": 3475
    },
    {
      "epoch": 0.4548621902347264,
      "grad_norm": 7.954318046569824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11318740248680115,
      "logits/rejected": -0.05456981435418129,
      "logps/chosen": -327.7847900390625,
      "logps/rejected": -353.85748291015625,
      "loss": 0.5662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40567725896835327,
      "rewards/margins": 0.5326073169708252,
      "rewards/rejected": -0.9382845163345337,
      "step": 3476
    },
    {
      "epoch": 0.4549930481720782,
      "grad_norm": 8.40339183807373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.069192074239254,
      "logits/rejected": 0.00599149614572525,
      "logps/chosen": -471.8634338378906,
      "logps/rejected": -557.43359375,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33799418807029724,
      "rewards/margins": 1.0726624727249146,
      "rewards/rejected": -0.7346683740615845,
      "step": 3477
    },
    {
      "epoch": 0.45512390610942993,
      "grad_norm": 12.06357192993164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0694788247346878,
      "logits/rejected": -0.07749168574810028,
      "logps/chosen": -472.8257141113281,
      "logps/rejected": -520.7158203125,
      "loss": 0.5021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27574387192726135,
      "rewards/margins": 1.6531739234924316,
      "rewards/rejected": -1.9289177656173706,
      "step": 3478
    },
    {
      "epoch": 0.4552547640467817,
      "grad_norm": 7.553166389465332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025972701609134674,
      "logits/rejected": 0.03708311915397644,
      "logps/chosen": -533.497802734375,
      "logps/rejected": -457.70458984375,
      "loss": 0.706,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3170774579048157,
      "rewards/margins": 0.4789770245552063,
      "rewards/rejected": -0.16189956665039062,
      "step": 3479
    },
    {
      "epoch": 0.45538562198413346,
      "grad_norm": 9.661460876464844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17406074702739716,
      "logits/rejected": -0.1454126089811325,
      "logps/chosen": -515.5051879882812,
      "logps/rejected": -591.4227905273438,
      "loss": 0.7547,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5178781151771545,
      "rewards/margins": 0.2201162576675415,
      "rewards/rejected": -0.7379943132400513,
      "step": 3480
    },
    {
      "epoch": 0.45551647992148525,
      "grad_norm": 8.376688003540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11175455152988434,
      "logits/rejected": 0.05569657310843468,
      "logps/chosen": -355.01763916015625,
      "logps/rejected": -348.9660339355469,
      "loss": 0.6001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03332136571407318,
      "rewards/margins": 0.3758394122123718,
      "rewards/rejected": -0.34251800179481506,
      "step": 3481
    },
    {
      "epoch": 0.455647337858837,
      "grad_norm": 10.048530578613281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06496021896600723,
      "logits/rejected": 0.0102933868765831,
      "logps/chosen": -634.13134765625,
      "logps/rejected": -602.6732788085938,
      "loss": 0.6581,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.7980057001113892,
      "rewards/margins": 0.8038867712020874,
      "rewards/rejected": -0.0058811130002141,
      "step": 3482
    },
    {
      "epoch": 0.4557781957961888,
      "grad_norm": 12.533651351928711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08497260510921478,
      "logits/rejected": -0.15510965883731842,
      "logps/chosen": -376.0986328125,
      "logps/rejected": -390.7695617675781,
      "loss": 1.2905,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.1049832105636597,
      "rewards/margins": -0.5328382253646851,
      "rewards/rejected": -0.5721448659896851,
      "step": 3483
    },
    {
      "epoch": 0.4559090537335405,
      "grad_norm": 6.814491271972656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09226682782173157,
      "logits/rejected": 0.005840301513671875,
      "logps/chosen": -335.99627685546875,
      "logps/rejected": -454.5951232910156,
      "loss": 0.633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.38403093814849854,
      "rewards/margins": 1.1235145330429077,
      "rewards/rejected": -0.7394835948944092,
      "step": 3484
    },
    {
      "epoch": 0.4560399116708923,
      "grad_norm": 9.661042213439941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07105225324630737,
      "logits/rejected": -0.0304747112095356,
      "logps/chosen": -418.095703125,
      "logps/rejected": -339.51580810546875,
      "loss": 0.5771,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6681779623031616,
      "rewards/margins": 0.6719321012496948,
      "rewards/rejected": -1.3401100635528564,
      "step": 3485
    },
    {
      "epoch": 0.45617076960824404,
      "grad_norm": 7.915233135223389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10052064061164856,
      "logits/rejected": -0.08549186587333679,
      "logps/chosen": -503.83892822265625,
      "logps/rejected": -431.8395690917969,
      "loss": 0.6235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7283806204795837,
      "rewards/margins": 0.28465670347213745,
      "rewards/rejected": -1.0130372047424316,
      "step": 3486
    },
    {
      "epoch": 0.45630162754559583,
      "grad_norm": 4.0862226486206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07753640413284302,
      "logits/rejected": 0.10679779201745987,
      "logps/chosen": -502.368896484375,
      "logps/rejected": -449.76165771484375,
      "loss": 0.2078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016547977924346924,
      "rewards/margins": 2.099093198776245,
      "rewards/rejected": -2.1156413555145264,
      "step": 3487
    },
    {
      "epoch": 0.45643248548294757,
      "grad_norm": 4.554566383361816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04272467643022537,
      "logits/rejected": -0.12340621650218964,
      "logps/chosen": -541.801513671875,
      "logps/rejected": -351.09906005859375,
      "loss": 0.2211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22905787825584412,
      "rewards/margins": 1.997053861618042,
      "rewards/rejected": -1.767995834350586,
      "step": 3488
    },
    {
      "epoch": 0.45656334342029936,
      "grad_norm": 13.368918418884277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2093167006969452,
      "logits/rejected": 0.0588388554751873,
      "logps/chosen": -391.14349365234375,
      "logps/rejected": -318.23944091796875,
      "loss": 0.887,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.5984673500061035,
      "rewards/margins": -0.2388838827610016,
      "rewards/rejected": -1.3595833778381348,
      "step": 3489
    },
    {
      "epoch": 0.4566942013576511,
      "grad_norm": 7.857448101043701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02194782719016075,
      "logits/rejected": 0.09350066632032394,
      "logps/chosen": -493.46392822265625,
      "logps/rejected": -417.2108459472656,
      "loss": 0.5989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22760827839374542,
      "rewards/margins": 0.8379291296005249,
      "rewards/rejected": -0.6103209257125854,
      "step": 3490
    },
    {
      "epoch": 0.4568250592950029,
      "grad_norm": 6.420183181762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32834893465042114,
      "logits/rejected": 0.05715341120958328,
      "logps/chosen": -526.3270874023438,
      "logps/rejected": -389.68963623046875,
      "loss": 0.3105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.652879536151886,
      "rewards/margins": 1.708878517150879,
      "rewards/rejected": -1.0559990406036377,
      "step": 3491
    },
    {
      "epoch": 0.4569559172323546,
      "grad_norm": 16.57972526550293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21390292048454285,
      "logits/rejected": -0.09913919866085052,
      "logps/chosen": -586.9439697265625,
      "logps/rejected": -416.81549072265625,
      "loss": 1.1287,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5250740051269531,
      "rewards/margins": -0.4113815426826477,
      "rewards/rejected": -1.1136925220489502,
      "step": 3492
    },
    {
      "epoch": 0.45708677516970636,
      "grad_norm": 8.81193733215332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2549595832824707,
      "logits/rejected": 0.12856917083263397,
      "logps/chosen": -564.6105346679688,
      "logps/rejected": -420.7860412597656,
      "loss": 0.6677,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28624555468559265,
      "rewards/margins": 0.6113712191581726,
      "rewards/rejected": -0.8976168632507324,
      "step": 3493
    },
    {
      "epoch": 0.45721763310705815,
      "grad_norm": 4.6966233253479,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32295629382133484,
      "logits/rejected": 0.20659266412258148,
      "logps/chosen": -498.00457763671875,
      "logps/rejected": -474.5225830078125,
      "loss": 0.244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37104731798171997,
      "rewards/margins": 2.1640613079071045,
      "rewards/rejected": -1.7930138111114502,
      "step": 3494
    },
    {
      "epoch": 0.4573484910444099,
      "grad_norm": 13.741161346435547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08429544419050217,
      "logits/rejected": 0.16052605211734772,
      "logps/chosen": -392.80255126953125,
      "logps/rejected": -402.836181640625,
      "loss": 1.0643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08430461585521698,
      "rewards/margins": 0.4887201189994812,
      "rewards/rejected": -0.404415488243103,
      "step": 3495
    },
    {
      "epoch": 0.4574793489817617,
      "grad_norm": 7.361876010894775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2038663625717163,
      "logits/rejected": -0.21196743845939636,
      "logps/chosen": -343.19879150390625,
      "logps/rejected": -567.5277099609375,
      "loss": 0.6117,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.38545453548431396,
      "rewards/margins": 1.6257606744766235,
      "rewards/rejected": -2.0112154483795166,
      "step": 3496
    },
    {
      "epoch": 0.4576102069191134,
      "grad_norm": 5.517976760864258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19423538446426392,
      "logits/rejected": 0.18302135169506073,
      "logps/chosen": -559.5390625,
      "logps/rejected": -573.2698364257812,
      "loss": 0.3284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8009911179542542,
      "rewards/margins": 1.3847026824951172,
      "rewards/rejected": -0.5837116241455078,
      "step": 3497
    },
    {
      "epoch": 0.4577410648564652,
      "grad_norm": 4.231954097747803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013442538678646088,
      "logits/rejected": 0.09591670334339142,
      "logps/chosen": -292.5762634277344,
      "logps/rejected": -415.1051025390625,
      "loss": 0.2434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4338703155517578,
      "rewards/margins": 2.106363296508789,
      "rewards/rejected": -2.540233612060547,
      "step": 3498
    },
    {
      "epoch": 0.45787192279381694,
      "grad_norm": 8.556964874267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1256888508796692,
      "logits/rejected": 0.09901342540979385,
      "logps/chosen": -443.34381103515625,
      "logps/rejected": -411.23486328125,
      "loss": 0.6354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22627314925193787,
      "rewards/margins": 0.4861465096473694,
      "rewards/rejected": -0.2598734200000763,
      "step": 3499
    },
    {
      "epoch": 0.45800278073116873,
      "grad_norm": 5.359426975250244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2589736878871918,
      "logits/rejected": 0.28430384397506714,
      "logps/chosen": -424.8326416015625,
      "logps/rejected": -463.38848876953125,
      "loss": 0.3752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11849171668291092,
      "rewards/margins": 1.2540465593338013,
      "rewards/rejected": -1.1355549097061157,
      "step": 3500
    },
    {
      "epoch": 0.45813363866852047,
      "grad_norm": 6.223231315612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21813398599624634,
      "logits/rejected": 0.12246572971343994,
      "logps/chosen": -355.1102600097656,
      "logps/rejected": -340.857421875,
      "loss": 0.4614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4521423578262329,
      "rewards/margins": 1.2021665573120117,
      "rewards/rejected": -0.7500242590904236,
      "step": 3501
    },
    {
      "epoch": 0.45826449660587226,
      "grad_norm": 11.88017749786377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17906861007213593,
      "logits/rejected": 0.12787693738937378,
      "logps/chosen": -514.1674194335938,
      "logps/rejected": -544.9347534179688,
      "loss": 0.9836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9766607284545898,
      "rewards/margins": 0.1105530858039856,
      "rewards/rejected": -1.0872137546539307,
      "step": 3502
    },
    {
      "epoch": 0.458395354543224,
      "grad_norm": 4.193367004394531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06873884052038193,
      "logits/rejected": 0.09575599431991577,
      "logps/chosen": -571.6851806640625,
      "logps/rejected": -286.6654968261719,
      "loss": 0.2668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.43154338002204895,
      "rewards/margins": 1.7380226850509644,
      "rewards/rejected": -1.3064793348312378,
      "step": 3503
    },
    {
      "epoch": 0.4585262124805758,
      "grad_norm": 6.31959342956543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3531450033187866,
      "logits/rejected": 0.062470048666000366,
      "logps/chosen": -556.762451171875,
      "logps/rejected": -439.91802978515625,
      "loss": 0.4297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1401950865983963,
      "rewards/margins": 0.8165594339370728,
      "rewards/rejected": -0.6763643622398376,
      "step": 3504
    },
    {
      "epoch": 0.4586570704179275,
      "grad_norm": 5.0622334480285645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06445479393005371,
      "logits/rejected": -0.029595427215099335,
      "logps/chosen": -443.3472900390625,
      "logps/rejected": -512.5686645507812,
      "loss": 0.4302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5801788568496704,
      "rewards/margins": 1.2166757583618164,
      "rewards/rejected": -0.6364969611167908,
      "step": 3505
    },
    {
      "epoch": 0.4587879283552793,
      "grad_norm": 4.378854274749756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11731863021850586,
      "logits/rejected": -0.13501864671707153,
      "logps/chosen": -536.0760498046875,
      "logps/rejected": -491.064208984375,
      "loss": 0.3032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7485485076904297,
      "rewards/margins": 1.3132243156433105,
      "rewards/rejected": -0.5646756887435913,
      "step": 3506
    },
    {
      "epoch": 0.45891878629263105,
      "grad_norm": 6.956819534301758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1346173733472824,
      "logits/rejected": 0.1289934515953064,
      "logps/chosen": -410.7126159667969,
      "logps/rejected": -340.2817687988281,
      "loss": 0.5545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07247181236743927,
      "rewards/margins": 0.35598334670066833,
      "rewards/rejected": -0.4284551441669464,
      "step": 3507
    },
    {
      "epoch": 0.45904964422998285,
      "grad_norm": 6.093733310699463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0022885650396347046,
      "logits/rejected": 0.12384791672229767,
      "logps/chosen": -318.480712890625,
      "logps/rejected": -400.9657897949219,
      "loss": 0.4138,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4354511499404907,
      "rewards/margins": 1.0130137205123901,
      "rewards/rejected": -0.5775627493858337,
      "step": 3508
    },
    {
      "epoch": 0.4591805021673346,
      "grad_norm": 3.0470540523529053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09110987186431885,
      "logits/rejected": -0.10063913464546204,
      "logps/chosen": -551.7924194335938,
      "logps/rejected": -398.0760192871094,
      "loss": 0.2287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3852050304412842,
      "rewards/margins": 2.08022141456604,
      "rewards/rejected": -0.6950162649154663,
      "step": 3509
    },
    {
      "epoch": 0.4593113601046864,
      "grad_norm": 7.732592582702637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06307634711265564,
      "logits/rejected": -0.34042027592658997,
      "logps/chosen": -416.36688232421875,
      "logps/rejected": -583.2188110351562,
      "loss": 0.4625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9488722085952759,
      "rewards/margins": 1.084097146987915,
      "rewards/rejected": -2.0329692363739014,
      "step": 3510
    },
    {
      "epoch": 0.4594422180420381,
      "grad_norm": 10.730425834655762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19452744722366333,
      "logits/rejected": 0.21347688138484955,
      "logps/chosen": -547.3584594726562,
      "logps/rejected": -458.6397705078125,
      "loss": 0.8426,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.17660143971443176,
      "rewards/margins": 0.6581761240959167,
      "rewards/rejected": -0.4815747141838074,
      "step": 3511
    },
    {
      "epoch": 0.4595730759793899,
      "grad_norm": 8.679327011108398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18428681790828705,
      "logits/rejected": -0.07275073230266571,
      "logps/chosen": -359.203125,
      "logps/rejected": -363.48809814453125,
      "loss": 0.4815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2712087631225586,
      "rewards/margins": 0.7872544527053833,
      "rewards/rejected": -1.0584629774093628,
      "step": 3512
    },
    {
      "epoch": 0.45970393391674164,
      "grad_norm": 5.622132778167725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06679876148700714,
      "logits/rejected": -0.32666829228401184,
      "logps/chosen": -452.7521667480469,
      "logps/rejected": -612.0596313476562,
      "loss": 0.3183,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5165970325469971,
      "rewards/margins": 2.0965259075164795,
      "rewards/rejected": -1.579928994178772,
      "step": 3513
    },
    {
      "epoch": 0.4598347918540934,
      "grad_norm": 5.057809829711914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33496642112731934,
      "logits/rejected": 0.25658056139945984,
      "logps/chosen": -382.86773681640625,
      "logps/rejected": -370.7169189453125,
      "loss": 0.3553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3601972758769989,
      "rewards/margins": 1.3934814929962158,
      "rewards/rejected": -1.033284068107605,
      "step": 3514
    },
    {
      "epoch": 0.45996564979144516,
      "grad_norm": 6.050417900085449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02158753201365471,
      "logits/rejected": -0.3399568200111389,
      "logps/chosen": -582.7544555664062,
      "logps/rejected": -558.780517578125,
      "loss": 0.4135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2169804573059082,
      "rewards/margins": 1.569863200187683,
      "rewards/rejected": -0.3528827130794525,
      "step": 3515
    },
    {
      "epoch": 0.4600965077287969,
      "grad_norm": 9.916651725769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14900711178779602,
      "logits/rejected": 0.022622331976890564,
      "logps/chosen": -494.3752746582031,
      "logps/rejected": -712.31005859375,
      "loss": 0.7894,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.1214861124753952,
      "rewards/margins": 0.4267690181732178,
      "rewards/rejected": -0.3052830100059509,
      "step": 3516
    },
    {
      "epoch": 0.4602273656661487,
      "grad_norm": 9.681943893432617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10299750417470932,
      "logits/rejected": -0.0849577784538269,
      "logps/chosen": -508.060791015625,
      "logps/rejected": -505.1078796386719,
      "loss": 0.5233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1983131468296051,
      "rewards/margins": 0.7468235492706299,
      "rewards/rejected": -0.5485104322433472,
      "step": 3517
    },
    {
      "epoch": 0.4603582236035004,
      "grad_norm": 8.294474601745605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30769672989845276,
      "logits/rejected": 0.3654291331768036,
      "logps/chosen": -457.3524169921875,
      "logps/rejected": -583.0382080078125,
      "loss": 0.5709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8234337568283081,
      "rewards/margins": 0.6593188643455505,
      "rewards/rejected": -1.4827525615692139,
      "step": 3518
    },
    {
      "epoch": 0.4604890815408522,
      "grad_norm": 6.0599870681762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11708392202854156,
      "logits/rejected": 0.05585833266377449,
      "logps/chosen": -518.4766845703125,
      "logps/rejected": -415.2198486328125,
      "loss": 0.3458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.089658498764038,
      "rewards/margins": 1.4867072105407715,
      "rewards/rejected": -0.39704880118370056,
      "step": 3519
    },
    {
      "epoch": 0.46061993947820395,
      "grad_norm": 8.339529991149902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15838094055652618,
      "logits/rejected": 0.12484043836593628,
      "logps/chosen": -420.29071044921875,
      "logps/rejected": -404.9547119140625,
      "loss": 0.6052,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.296151727437973,
      "rewards/margins": 0.7102943062782288,
      "rewards/rejected": -1.0064460039138794,
      "step": 3520
    },
    {
      "epoch": 0.46075079741555575,
      "grad_norm": 6.421997547149658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24226850271224976,
      "logits/rejected": 0.1718473881483078,
      "logps/chosen": -494.6470947265625,
      "logps/rejected": -453.6416931152344,
      "loss": 0.5243,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6794876456260681,
      "rewards/margins": 0.6415504813194275,
      "rewards/rejected": 0.03793717175722122,
      "step": 3521
    },
    {
      "epoch": 0.4608816553529075,
      "grad_norm": 7.470181465148926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02081471122801304,
      "logits/rejected": -0.06824708729982376,
      "logps/chosen": -338.9323425292969,
      "logps/rejected": -345.462646484375,
      "loss": 0.6071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6525672674179077,
      "rewards/margins": 0.5407039523124695,
      "rewards/rejected": -1.193271279335022,
      "step": 3522
    },
    {
      "epoch": 0.4610125132902593,
      "grad_norm": 8.426816940307617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07598713040351868,
      "logits/rejected": 0.07880041003227234,
      "logps/chosen": -534.990478515625,
      "logps/rejected": -514.2531127929688,
      "loss": 0.4791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2952350676059723,
      "rewards/margins": 0.9719737768173218,
      "rewards/rejected": -0.6767386794090271,
      "step": 3523
    },
    {
      "epoch": 0.461143371227611,
      "grad_norm": 5.330247402191162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1957453489303589,
      "logits/rejected": 0.10459424555301666,
      "logps/chosen": -430.0884094238281,
      "logps/rejected": -420.3137512207031,
      "loss": 0.3868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1905948668718338,
      "rewards/margins": 1.2438968420028687,
      "rewards/rejected": -1.434491753578186,
      "step": 3524
    },
    {
      "epoch": 0.4612742291649628,
      "grad_norm": 5.438567161560059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10325822979211807,
      "logits/rejected": 0.04528268426656723,
      "logps/chosen": -413.4975280761719,
      "logps/rejected": -532.33447265625,
      "loss": 0.3986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23467442393302917,
      "rewards/margins": 1.0357365608215332,
      "rewards/rejected": -1.2704110145568848,
      "step": 3525
    },
    {
      "epoch": 0.46140508710231454,
      "grad_norm": 4.942768573760986,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5043520331382751,
      "logits/rejected": 0.3280787765979767,
      "logps/chosen": -466.5911865234375,
      "logps/rejected": -380.47857666015625,
      "loss": 0.2894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5164585709571838,
      "rewards/margins": 1.8219892978668213,
      "rewards/rejected": -1.3055307865142822,
      "step": 3526
    },
    {
      "epoch": 0.46153594503966633,
      "grad_norm": 10.340792655944824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1397992968559265,
      "logits/rejected": 0.006240196526050568,
      "logps/chosen": -418.5384521484375,
      "logps/rejected": -461.4913330078125,
      "loss": 0.7015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11896112561225891,
      "rewards/margins": 0.27195990085601807,
      "rewards/rejected": -0.390921026468277,
      "step": 3527
    },
    {
      "epoch": 0.46166680297701806,
      "grad_norm": 5.3342671394348145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.045846305787563324,
      "logits/rejected": 0.26630833745002747,
      "logps/chosen": -535.8670043945312,
      "logps/rejected": -389.76116943359375,
      "loss": 0.3759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7415491342544556,
      "rewards/margins": 1.0981203317642212,
      "rewards/rejected": -1.8396695852279663,
      "step": 3528
    },
    {
      "epoch": 0.46179766091436986,
      "grad_norm": 8.506574630737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11082442104816437,
      "logits/rejected": 0.4709798991680145,
      "logps/chosen": -412.183349609375,
      "logps/rejected": -445.9498596191406,
      "loss": 0.5616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.022107914090156555,
      "rewards/margins": 0.7432283759117126,
      "rewards/rejected": -0.7653362154960632,
      "step": 3529
    },
    {
      "epoch": 0.4619285188517216,
      "grad_norm": 16.687103271484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22103890776634216,
      "logits/rejected": 0.03672582656145096,
      "logps/chosen": -651.079345703125,
      "logps/rejected": -489.5631408691406,
      "loss": 1.0301,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1898374557495117,
      "rewards/margins": -0.14508682489395142,
      "rewards/rejected": -1.044750690460205,
      "step": 3530
    },
    {
      "epoch": 0.4620593767890734,
      "grad_norm": 6.629051685333252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007598813623189926,
      "logits/rejected": 0.2172616571187973,
      "logps/chosen": -631.3815307617188,
      "logps/rejected": -502.2220458984375,
      "loss": 0.4493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24672605097293854,
      "rewards/margins": 0.975069522857666,
      "rewards/rejected": -0.7283434271812439,
      "step": 3531
    },
    {
      "epoch": 0.4621902347264251,
      "grad_norm": 8.775014877319336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22066691517829895,
      "logits/rejected": -0.22622165083885193,
      "logps/chosen": -697.1906127929688,
      "logps/rejected": -488.0786437988281,
      "loss": 0.5605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2299904227256775,
      "rewards/margins": 1.0256463289260864,
      "rewards/rejected": -0.7956558465957642,
      "step": 3532
    },
    {
      "epoch": 0.4623210926637769,
      "grad_norm": 6.0370941162109375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0776066854596138,
      "logits/rejected": -0.2283850759267807,
      "logps/chosen": -437.2000427246094,
      "logps/rejected": -366.89300537109375,
      "loss": 0.5089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10563451051712036,
      "rewards/margins": 1.0636606216430664,
      "rewards/rejected": -1.169295072555542,
      "step": 3533
    },
    {
      "epoch": 0.46245195060112865,
      "grad_norm": 8.866504669189453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05920308083295822,
      "logits/rejected": 0.06130077689886093,
      "logps/chosen": -375.51544189453125,
      "logps/rejected": -423.84759521484375,
      "loss": 0.5757,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29199695587158203,
      "rewards/margins": 0.513952910900116,
      "rewards/rejected": -0.8059497475624084,
      "step": 3534
    },
    {
      "epoch": 0.46258280853848044,
      "grad_norm": 9.581130981445312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12100724130868912,
      "logits/rejected": 0.017915528267621994,
      "logps/chosen": -510.7027587890625,
      "logps/rejected": -498.69598388671875,
      "loss": 0.5975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.25036948919296265,
      "rewards/margins": 0.9477701783180237,
      "rewards/rejected": -0.697400689125061,
      "step": 3535
    },
    {
      "epoch": 0.4627136664758322,
      "grad_norm": 12.11132526397705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0024306103587150574,
      "logits/rejected": -0.17958660423755646,
      "logps/chosen": -500.934326171875,
      "logps/rejected": -420.14105224609375,
      "loss": 0.9731,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0925113707780838,
      "rewards/margins": 0.2480016052722931,
      "rewards/rejected": -0.3405129611492157,
      "step": 3536
    },
    {
      "epoch": 0.4628445244131839,
      "grad_norm": 6.888150691986084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13860425353050232,
      "logits/rejected": 0.02499866485595703,
      "logps/chosen": -460.0608825683594,
      "logps/rejected": -549.1712036132812,
      "loss": 0.4305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.406127393245697,
      "rewards/margins": 1.0974665880203247,
      "rewards/rejected": -0.6913393139839172,
      "step": 3537
    },
    {
      "epoch": 0.4629753823505357,
      "grad_norm": 6.465810298919678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03443508595228195,
      "logits/rejected": -0.1998823583126068,
      "logps/chosen": -557.4307250976562,
      "logps/rejected": -598.32666015625,
      "loss": 0.4353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.1029682159423828,
      "rewards/margins": 1.2700514793395996,
      "rewards/rejected": -0.16708336770534515,
      "step": 3538
    },
    {
      "epoch": 0.46310624028788744,
      "grad_norm": 4.28283166885376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1059255376458168,
      "logits/rejected": -0.18048818409442902,
      "logps/chosen": -431.066162109375,
      "logps/rejected": -457.240234375,
      "loss": 0.2521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17960873246192932,
      "rewards/margins": 1.731874942779541,
      "rewards/rejected": -1.911483645439148,
      "step": 3539
    },
    {
      "epoch": 0.46323709822523923,
      "grad_norm": 6.530880928039551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0076414793729782104,
      "logits/rejected": -0.21370701491832733,
      "logps/chosen": -483.31353759765625,
      "logps/rejected": -451.0527648925781,
      "loss": 0.3876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8787242770195007,
      "rewards/margins": 1.0489517450332642,
      "rewards/rejected": -0.17022745311260223,
      "step": 3540
    },
    {
      "epoch": 0.46336795616259097,
      "grad_norm": 5.265172958374023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17022578418254852,
      "logits/rejected": 0.016862429678440094,
      "logps/chosen": -404.2854309082031,
      "logps/rejected": -361.7017822265625,
      "loss": 0.3848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3327232599258423,
      "rewards/margins": 1.4789859056472778,
      "rewards/rejected": -1.146262526512146,
      "step": 3541
    },
    {
      "epoch": 0.46349881409994276,
      "grad_norm": 7.8166728019714355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12333884835243225,
      "logits/rejected": -0.07999206334352493,
      "logps/chosen": -431.7169189453125,
      "logps/rejected": -392.8329162597656,
      "loss": 0.6203,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.7153494358062744,
      "rewards/margins": 0.901684045791626,
      "rewards/rejected": -0.18633463978767395,
      "step": 3542
    },
    {
      "epoch": 0.4636296720372945,
      "grad_norm": 7.453285217285156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3124596178531647,
      "logits/rejected": 0.08829513192176819,
      "logps/chosen": -336.9908447265625,
      "logps/rejected": -425.8435363769531,
      "loss": 0.6074,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12858439981937408,
      "rewards/margins": 0.8626611828804016,
      "rewards/rejected": -0.9912455677986145,
      "step": 3543
    },
    {
      "epoch": 0.4637605299746463,
      "grad_norm": 8.523991584777832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09551737457513809,
      "logits/rejected": -0.15888643264770508,
      "logps/chosen": -549.2300415039062,
      "logps/rejected": -470.1215515136719,
      "loss": 0.6811,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.032438650727272034,
      "rewards/margins": 0.17178495228290558,
      "rewards/rejected": -0.13934633135795593,
      "step": 3544
    },
    {
      "epoch": 0.463891387911998,
      "grad_norm": 9.390231132507324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032566189765930176,
      "logits/rejected": -0.16582585871219635,
      "logps/chosen": -467.52288818359375,
      "logps/rejected": -423.3392028808594,
      "loss": 0.5748,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6664246916770935,
      "rewards/margins": 0.7993481159210205,
      "rewards/rejected": -1.4657728672027588,
      "step": 3545
    },
    {
      "epoch": 0.4640222458493498,
      "grad_norm": 8.132396697998047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01348043978214264,
      "logits/rejected": -0.3884144425392151,
      "logps/chosen": -473.2762145996094,
      "logps/rejected": -489.1793518066406,
      "loss": 0.7865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9664976596832275,
      "rewards/margins": 0.7869064211845398,
      "rewards/rejected": -1.753404140472412,
      "step": 3546
    },
    {
      "epoch": 0.46415310378670155,
      "grad_norm": 6.442074298858643,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2267480194568634,
      "logits/rejected": 0.08773207664489746,
      "logps/chosen": -609.0206909179688,
      "logps/rejected": -571.158935546875,
      "loss": 0.3992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0300309658050537,
      "rewards/margins": 1.0500497817993164,
      "rewards/rejected": -0.02001875638961792,
      "step": 3547
    },
    {
      "epoch": 0.46428396172405334,
      "grad_norm": 5.675081729888916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2587047219276428,
      "logits/rejected": 0.1941443681716919,
      "logps/chosen": -694.1009521484375,
      "logps/rejected": -728.4691162109375,
      "loss": 0.3944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6102989912033081,
      "rewards/margins": 1.0404727458953857,
      "rewards/rejected": -0.4301738739013672,
      "step": 3548
    },
    {
      "epoch": 0.4644148196614051,
      "grad_norm": 10.271838188171387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3478296101093292,
      "logits/rejected": 0.3697548508644104,
      "logps/chosen": -613.919921875,
      "logps/rejected": -683.8330688476562,
      "loss": 0.7011,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18770691752433777,
      "rewards/margins": 0.41618597507476807,
      "rewards/rejected": -0.6038928627967834,
      "step": 3549
    },
    {
      "epoch": 0.46454567759875687,
      "grad_norm": 5.031225681304932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35447007417678833,
      "logits/rejected": -0.6243689656257629,
      "logps/chosen": -476.72265625,
      "logps/rejected": -533.8112182617188,
      "loss": 0.3257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8404919505119324,
      "rewards/margins": 1.3485718965530396,
      "rewards/rejected": -2.189063549041748,
      "step": 3550
    },
    {
      "epoch": 0.4646765355361086,
      "grad_norm": 11.800487518310547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19719243049621582,
      "logits/rejected": 0.2744862735271454,
      "logps/chosen": -468.7436828613281,
      "logps/rejected": -339.2386779785156,
      "loss": 1.1001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6354387402534485,
      "rewards/margins": -0.19020453095436096,
      "rewards/rejected": -0.445234090089798,
      "step": 3551
    },
    {
      "epoch": 0.4648073934734604,
      "grad_norm": 9.43814754486084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08460577577352524,
      "logits/rejected": 0.03980567306280136,
      "logps/chosen": -602.686279296875,
      "logps/rejected": -366.5172119140625,
      "loss": 0.6599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05018483102321625,
      "rewards/margins": 1.0223376750946045,
      "rewards/rejected": -0.9721527099609375,
      "step": 3552
    },
    {
      "epoch": 0.46493825141081213,
      "grad_norm": 6.5854949951171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3376672863960266,
      "logits/rejected": 0.24619922041893005,
      "logps/chosen": -346.89764404296875,
      "logps/rejected": -338.7156066894531,
      "loss": 0.5394,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5677253603935242,
      "rewards/margins": 1.0825586318969727,
      "rewards/rejected": -0.5148332715034485,
      "step": 3553
    },
    {
      "epoch": 0.4650691093481639,
      "grad_norm": 6.4614434242248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11445293575525284,
      "logits/rejected": 0.06565338373184204,
      "logps/chosen": -596.576904296875,
      "logps/rejected": -573.8895263671875,
      "loss": 0.4487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2613517642021179,
      "rewards/margins": 1.3351880311965942,
      "rewards/rejected": -1.073836088180542,
      "step": 3554
    },
    {
      "epoch": 0.46519996728551566,
      "grad_norm": 11.543802261352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03944535553455353,
      "logits/rejected": -0.02391234040260315,
      "logps/chosen": -482.7882995605469,
      "logps/rejected": -390.529541015625,
      "loss": 0.7566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3003854751586914,
      "rewards/margins": 0.7296205163002014,
      "rewards/rejected": -0.4292350709438324,
      "step": 3555
    },
    {
      "epoch": 0.46533082522286745,
      "grad_norm": 7.48030948638916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14986032247543335,
      "logits/rejected": -0.11073483526706696,
      "logps/chosen": -259.1199951171875,
      "logps/rejected": -439.55853271484375,
      "loss": 0.4518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09881137311458588,
      "rewards/margins": 1.0754058361053467,
      "rewards/rejected": -1.1742173433303833,
      "step": 3556
    },
    {
      "epoch": 0.4654616831602192,
      "grad_norm": 7.56343936920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24158430099487305,
      "logits/rejected": 0.3347948491573334,
      "logps/chosen": -522.2269897460938,
      "logps/rejected": -515.20361328125,
      "loss": 0.5762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.037190020084381104,
      "rewards/margins": 0.746551513671875,
      "rewards/rejected": -0.7093614935874939,
      "step": 3557
    },
    {
      "epoch": 0.4655925410975709,
      "grad_norm": 7.141625881195068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03683025389909744,
      "logits/rejected": 0.1211596205830574,
      "logps/chosen": -384.659912109375,
      "logps/rejected": -477.8367919921875,
      "loss": 0.5827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07589283585548401,
      "rewards/margins": 0.5459697842597961,
      "rewards/rejected": -0.4700769782066345,
      "step": 3558
    },
    {
      "epoch": 0.4657233990349227,
      "grad_norm": 7.918883800506592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01324041560292244,
      "logits/rejected": -0.11670821905136108,
      "logps/chosen": -609.4691772460938,
      "logps/rejected": -435.48638916015625,
      "loss": 0.4831,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2813055217266083,
      "rewards/margins": 0.9646488428115845,
      "rewards/rejected": -0.6833432912826538,
      "step": 3559
    },
    {
      "epoch": 0.46585425697227445,
      "grad_norm": 10.213383674621582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1415131837129593,
      "logits/rejected": -0.14391210675239563,
      "logps/chosen": -515.681884765625,
      "logps/rejected": -322.5479431152344,
      "loss": 0.8101,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.637769341468811,
      "rewards/margins": 0.10778789222240448,
      "rewards/rejected": -0.7455571889877319,
      "step": 3560
    },
    {
      "epoch": 0.46598511490962624,
      "grad_norm": 4.86708927154541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.222635418176651,
      "logits/rejected": 0.04569244757294655,
      "logps/chosen": -660.2825317382812,
      "logps/rejected": -685.2103271484375,
      "loss": 0.2553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8379068374633789,
      "rewards/margins": 2.1239559650421143,
      "rewards/rejected": -1.2860491275787354,
      "step": 3561
    },
    {
      "epoch": 0.466115972846978,
      "grad_norm": 6.250145435333252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17842766642570496,
      "logits/rejected": 0.19331896305084229,
      "logps/chosen": -420.0487365722656,
      "logps/rejected": -499.8519287109375,
      "loss": 0.4948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4602065980434418,
      "rewards/margins": 0.8723833560943604,
      "rewards/rejected": -1.3325899839401245,
      "step": 3562
    },
    {
      "epoch": 0.46624683078432977,
      "grad_norm": 8.382014274597168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02799009159207344,
      "logits/rejected": -0.013592023402452469,
      "logps/chosen": -510.5680847167969,
      "logps/rejected": -356.4534606933594,
      "loss": 0.7282,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1250872611999512,
      "rewards/margins": 0.14945292472839355,
      "rewards/rejected": -1.2745401859283447,
      "step": 3563
    },
    {
      "epoch": 0.4663776887216815,
      "grad_norm": 6.341909885406494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2876564562320709,
      "logits/rejected": 0.06383159756660461,
      "logps/chosen": -531.3837890625,
      "logps/rejected": -454.5694885253906,
      "loss": 0.5322,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05573253333568573,
      "rewards/margins": 0.7347703576087952,
      "rewards/rejected": -0.679037868976593,
      "step": 3564
    },
    {
      "epoch": 0.4665085466590333,
      "grad_norm": 8.302992820739746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19796746969223022,
      "logits/rejected": -0.046818267554044724,
      "logps/chosen": -384.792236328125,
      "logps/rejected": -363.3984680175781,
      "loss": 0.4481,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4019005298614502,
      "rewards/margins": 0.7454191446304321,
      "rewards/rejected": -1.1473196744918823,
      "step": 3565
    },
    {
      "epoch": 0.46663940459638503,
      "grad_norm": 8.158838272094727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33059099316596985,
      "logits/rejected": -0.14507488906383514,
      "logps/chosen": -552.9649047851562,
      "logps/rejected": -336.4580383300781,
      "loss": 0.5732,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8805341720581055,
      "rewards/margins": 1.0501127243041992,
      "rewards/rejected": -0.1695784479379654,
      "step": 3566
    },
    {
      "epoch": 0.4667702625337368,
      "grad_norm": 5.942169666290283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41964036226272583,
      "logits/rejected": 0.21055898070335388,
      "logps/chosen": -305.45556640625,
      "logps/rejected": -321.6920166015625,
      "loss": 0.423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7093920111656189,
      "rewards/margins": 1.0357204675674438,
      "rewards/rejected": -0.32632848620414734,
      "step": 3567
    },
    {
      "epoch": 0.46690112047108856,
      "grad_norm": 7.196891784667969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3727620542049408,
      "logits/rejected": 0.30161651968955994,
      "logps/chosen": -584.3135986328125,
      "logps/rejected": -444.5697021484375,
      "loss": 0.6263,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8720486164093018,
      "rewards/margins": 0.9006264209747314,
      "rewards/rejected": -0.02857779711484909,
      "step": 3568
    },
    {
      "epoch": 0.46703197840844035,
      "grad_norm": 7.687424182891846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19394852221012115,
      "logits/rejected": 0.27691516280174255,
      "logps/chosen": -467.6777038574219,
      "logps/rejected": -424.3411865234375,
      "loss": 0.4944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2681768536567688,
      "rewards/margins": 1.327759861946106,
      "rewards/rejected": -1.0595828294754028,
      "step": 3569
    },
    {
      "epoch": 0.4671628363457921,
      "grad_norm": 5.722975254058838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11249658465385437,
      "logits/rejected": -0.11721882224082947,
      "logps/chosen": -363.0000915527344,
      "logps/rejected": -297.42572021484375,
      "loss": 0.4365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12253974378108978,
      "rewards/margins": 1.2280768156051636,
      "rewards/rejected": -1.1055370569229126,
      "step": 3570
    },
    {
      "epoch": 0.4672936942831439,
      "grad_norm": 7.801365375518799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05483541637659073,
      "logits/rejected": 0.10086078196763992,
      "logps/chosen": -343.02435302734375,
      "logps/rejected": -351.5627746582031,
      "loss": 0.6813,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.33545875549316406,
      "rewards/margins": 0.37099477648735046,
      "rewards/rejected": -0.7064535021781921,
      "step": 3571
    },
    {
      "epoch": 0.4674245522204956,
      "grad_norm": 9.036931991577148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01088634505867958,
      "logits/rejected": 0.021808337420225143,
      "logps/chosen": -348.1654357910156,
      "logps/rejected": -696.290771484375,
      "loss": 0.42,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8784092664718628,
      "rewards/margins": 1.5586662292480469,
      "rewards/rejected": -2.437075614929199,
      "step": 3572
    },
    {
      "epoch": 0.4675554101578474,
      "grad_norm": 8.14755630493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09508956223726273,
      "logits/rejected": 0.16812869906425476,
      "logps/chosen": -602.116455078125,
      "logps/rejected": -461.625732421875,
      "loss": 0.5961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8002426624298096,
      "rewards/margins": 0.8466376066207886,
      "rewards/rejected": -0.046394914388656616,
      "step": 3573
    },
    {
      "epoch": 0.46768626809519914,
      "grad_norm": 6.083872318267822,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2261909693479538,
      "logits/rejected": -0.09022987633943558,
      "logps/chosen": -348.6795654296875,
      "logps/rejected": -515.4996337890625,
      "loss": 0.5358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4392918050289154,
      "rewards/margins": 1.805816888809204,
      "rewards/rejected": -1.3665251731872559,
      "step": 3574
    },
    {
      "epoch": 0.46781712603255093,
      "grad_norm": 7.691524028778076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021441563963890076,
      "logits/rejected": 0.07433812320232391,
      "logps/chosen": -366.9240417480469,
      "logps/rejected": -313.603271484375,
      "loss": 0.5092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4967426061630249,
      "rewards/margins": 0.7654259204864502,
      "rewards/rejected": -1.262168526649475,
      "step": 3575
    },
    {
      "epoch": 0.46794798396990267,
      "grad_norm": 7.847244739532471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2595326900482178,
      "logits/rejected": -0.10934479534626007,
      "logps/chosen": -390.5086669921875,
      "logps/rejected": -319.0693054199219,
      "loss": 0.6406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5538901090621948,
      "rewards/margins": 0.428005188703537,
      "rewards/rejected": -0.9818952679634094,
      "step": 3576
    },
    {
      "epoch": 0.46807884190725446,
      "grad_norm": 4.799591541290283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17541038990020752,
      "logits/rejected": 0.20557039976119995,
      "logps/chosen": -248.06077575683594,
      "logps/rejected": -262.392822265625,
      "loss": 0.4435,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5132913589477539,
      "rewards/margins": 1.2409294843673706,
      "rewards/rejected": -1.754220724105835,
      "step": 3577
    },
    {
      "epoch": 0.4682096998446062,
      "grad_norm": 4.782070159912109,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025478525087237358,
      "logits/rejected": 0.09361124038696289,
      "logps/chosen": -251.71493530273438,
      "logps/rejected": -369.5382995605469,
      "loss": 0.4095,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45261555910110474,
      "rewards/margins": 1.0673233270645142,
      "rewards/rejected": -0.6147077679634094,
      "step": 3578
    },
    {
      "epoch": 0.468340557781958,
      "grad_norm": 8.29041862487793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07103454321622849,
      "logits/rejected": 0.010666940361261368,
      "logps/chosen": -455.6175842285156,
      "logps/rejected": -399.32427978515625,
      "loss": 0.5552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.039699554443359375,
      "rewards/margins": 0.6475460529327393,
      "rewards/rejected": -0.6078464984893799,
      "step": 3579
    },
    {
      "epoch": 0.4684714157193097,
      "grad_norm": 6.402584075927734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0673481673002243,
      "logits/rejected": 0.10809897631406784,
      "logps/chosen": -457.55084228515625,
      "logps/rejected": -494.3796691894531,
      "loss": 0.4434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17853854596614838,
      "rewards/margins": 1.0043866634368896,
      "rewards/rejected": -1.1829251050949097,
      "step": 3580
    },
    {
      "epoch": 0.46860227365666146,
      "grad_norm": 8.834833145141602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2719303071498871,
      "logits/rejected": 0.17742450535297394,
      "logps/chosen": -527.678466796875,
      "logps/rejected": -414.6973876953125,
      "loss": 0.4728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4534740447998047,
      "rewards/margins": 0.8332135677337646,
      "rewards/rejected": -1.2866876125335693,
      "step": 3581
    },
    {
      "epoch": 0.46873313159401325,
      "grad_norm": 7.955835342407227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04941955953836441,
      "logits/rejected": -0.030212298035621643,
      "logps/chosen": -294.03924560546875,
      "logps/rejected": -393.22589111328125,
      "loss": 0.5364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7678089737892151,
      "rewards/margins": 0.5552724599838257,
      "rewards/rejected": -1.323081374168396,
      "step": 3582
    },
    {
      "epoch": 0.468863989531365,
      "grad_norm": 11.287586212158203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0006085559725761414,
      "logits/rejected": -0.2540651857852936,
      "logps/chosen": -363.23553466796875,
      "logps/rejected": -315.1015319824219,
      "loss": 0.8062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7721649408340454,
      "rewards/margins": 0.996532678604126,
      "rewards/rejected": -1.768697738647461,
      "step": 3583
    },
    {
      "epoch": 0.4689948474687168,
      "grad_norm": 9.304848670959473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24544502794742584,
      "logits/rejected": 0.24583135545253754,
      "logps/chosen": -471.6351013183594,
      "logps/rejected": -372.8012390136719,
      "loss": 0.8194,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5634815692901611,
      "rewards/margins": -0.008201047778129578,
      "rewards/rejected": -0.5552804470062256,
      "step": 3584
    },
    {
      "epoch": 0.4691257054060685,
      "grad_norm": 7.256466388702393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12245622277259827,
      "logits/rejected": 0.12129447609186172,
      "logps/chosen": -445.15020751953125,
      "logps/rejected": -389.155029296875,
      "loss": 0.6236,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.21578174829483032,
      "rewards/margins": 0.5428826808929443,
      "rewards/rejected": -0.7586645483970642,
      "step": 3585
    },
    {
      "epoch": 0.4692565633434203,
      "grad_norm": 9.203924179077148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11145228147506714,
      "logits/rejected": 0.20074914395809174,
      "logps/chosen": -384.513671875,
      "logps/rejected": -311.7180480957031,
      "loss": 0.8508,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3351164162158966,
      "rewards/margins": 0.1533825397491455,
      "rewards/rejected": -0.48849889636039734,
      "step": 3586
    },
    {
      "epoch": 0.46938742128077204,
      "grad_norm": 3.9933667182922363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05040930211544037,
      "logits/rejected": -0.0814586654305458,
      "logps/chosen": -577.5459594726562,
      "logps/rejected": -454.0351867675781,
      "loss": 0.2718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15227067470550537,
      "rewards/margins": 1.6357682943344116,
      "rewards/rejected": -1.4834975004196167,
      "step": 3587
    },
    {
      "epoch": 0.46951827921812384,
      "grad_norm": 6.964518070220947,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06262435019016266,
      "logits/rejected": 0.11065606772899628,
      "logps/chosen": -426.70458984375,
      "logps/rejected": -511.5848083496094,
      "loss": 0.537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2515750527381897,
      "rewards/margins": 1.4911510944366455,
      "rewards/rejected": -1.74272620677948,
      "step": 3588
    },
    {
      "epoch": 0.46964913715547557,
      "grad_norm": 8.486978530883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020324520766735077,
      "logits/rejected": 0.02016940712928772,
      "logps/chosen": -335.2522888183594,
      "logps/rejected": -358.6644287109375,
      "loss": 0.6897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12601491808891296,
      "rewards/margins": 0.36759740114212036,
      "rewards/rejected": -0.49361228942871094,
      "step": 3589
    },
    {
      "epoch": 0.46977999509282736,
      "grad_norm": 5.614021301269531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2639274597167969,
      "logits/rejected": 0.3334425687789917,
      "logps/chosen": -553.4042358398438,
      "logps/rejected": -576.4050903320312,
      "loss": 0.3413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42236387729644775,
      "rewards/margins": 1.3477518558502197,
      "rewards/rejected": -0.925387978553772,
      "step": 3590
    },
    {
      "epoch": 0.4699108530301791,
      "grad_norm": 6.937095642089844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1475556641817093,
      "logits/rejected": 0.2240353673696518,
      "logps/chosen": -544.1984252929688,
      "logps/rejected": -423.3905944824219,
      "loss": 0.5984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7364408373832703,
      "rewards/margins": 0.8733115196228027,
      "rewards/rejected": -0.1368705779314041,
      "step": 3591
    },
    {
      "epoch": 0.4700417109675309,
      "grad_norm": 4.300450325012207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1327672004699707,
      "logits/rejected": 0.3799150288105011,
      "logps/chosen": -315.06781005859375,
      "logps/rejected": -328.3611145019531,
      "loss": 0.3056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.474721759557724,
      "rewards/margins": 1.5702738761901855,
      "rewards/rejected": -1.0955522060394287,
      "step": 3592
    },
    {
      "epoch": 0.4701725689048826,
      "grad_norm": 5.299264907836914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16532237827777863,
      "logits/rejected": 0.33009907603263855,
      "logps/chosen": -496.09564208984375,
      "logps/rejected": -526.6739501953125,
      "loss": 0.3892,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.516579806804657,
      "rewards/margins": 1.1419751644134521,
      "rewards/rejected": -0.6253954172134399,
      "step": 3593
    },
    {
      "epoch": 0.4703034268422344,
      "grad_norm": 5.4809465408325195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38508591055870056,
      "logits/rejected": 0.16943328082561493,
      "logps/chosen": -535.54248046875,
      "logps/rejected": -402.25677490234375,
      "loss": 0.454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.024327874183654785,
      "rewards/margins": 1.2762484550476074,
      "rewards/rejected": -1.251920461654663,
      "step": 3594
    },
    {
      "epoch": 0.47043428477958615,
      "grad_norm": 6.173375129699707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2944161295890808,
      "logits/rejected": -0.38015881180763245,
      "logps/chosen": -456.9803161621094,
      "logps/rejected": -499.1968688964844,
      "loss": 0.4068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45004549622535706,
      "rewards/margins": 1.205436110496521,
      "rewards/rejected": -1.6554818153381348,
      "step": 3595
    },
    {
      "epoch": 0.47056514271693795,
      "grad_norm": 5.924594879150391,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2010333240032196,
      "logits/rejected": -0.0016037728637456894,
      "logps/chosen": -581.7063598632812,
      "logps/rejected": -538.0923461914062,
      "loss": 0.4319,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8554306626319885,
      "rewards/margins": 1.1005958318710327,
      "rewards/rejected": -0.24516525864601135,
      "step": 3596
    },
    {
      "epoch": 0.4706960006542897,
      "grad_norm": 8.022154808044434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0395038016140461,
      "logits/rejected": 0.2068057507276535,
      "logps/chosen": -383.1446533203125,
      "logps/rejected": -359.3936462402344,
      "loss": 0.5375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08955364674329758,
      "rewards/margins": 0.8680152893066406,
      "rewards/rejected": -0.9575690031051636,
      "step": 3597
    },
    {
      "epoch": 0.4708268585916415,
      "grad_norm": 8.552159309387207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23455239832401276,
      "logits/rejected": 0.24505355954170227,
      "logps/chosen": -556.8101196289062,
      "logps/rejected": -598.9592895507812,
      "loss": 0.5501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8539581298828125,
      "rewards/margins": 0.9312335252761841,
      "rewards/rejected": -0.07727544009685516,
      "step": 3598
    },
    {
      "epoch": 0.4709577165289932,
      "grad_norm": 9.146129608154297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14283229410648346,
      "logits/rejected": 0.047701481729745865,
      "logps/chosen": -442.62335205078125,
      "logps/rejected": -427.0809020996094,
      "loss": 0.5985,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17664757370948792,
      "rewards/margins": 0.7584038972854614,
      "rewards/rejected": -0.9350515604019165,
      "step": 3599
    },
    {
      "epoch": 0.471088574466345,
      "grad_norm": 6.339561462402344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03177905082702637,
      "logits/rejected": -0.0649973675608635,
      "logps/chosen": -677.696533203125,
      "logps/rejected": -613.6767578125,
      "loss": 0.5287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6504648327827454,
      "rewards/margins": 1.024470329284668,
      "rewards/rejected": -0.3740054965019226,
      "step": 3600
    },
    {
      "epoch": 0.47121943240369674,
      "grad_norm": 9.616098403930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.032680004835128784,
      "logits/rejected": -0.1855841428041458,
      "logps/chosen": -361.3761291503906,
      "logps/rejected": -558.5991821289062,
      "loss": 0.7469,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.32003557682037354,
      "rewards/margins": 0.864348292350769,
      "rewards/rejected": -1.184383749961853,
      "step": 3601
    },
    {
      "epoch": 0.4713502903410485,
      "grad_norm": 7.700119495391846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38120532035827637,
      "logits/rejected": 0.23323847353458405,
      "logps/chosen": -405.8630065917969,
      "logps/rejected": -427.9100341796875,
      "loss": 0.5301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5578750967979431,
      "rewards/margins": 0.7289161682128906,
      "rewards/rejected": -1.286791205406189,
      "step": 3602
    },
    {
      "epoch": 0.47148114827840026,
      "grad_norm": 3.1347851753234863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16902205348014832,
      "logits/rejected": -0.13988669216632843,
      "logps/chosen": -516.8047485351562,
      "logps/rejected": -355.699462890625,
      "loss": 0.2355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2767440676689148,
      "rewards/margins": 1.6343390941619873,
      "rewards/rejected": -1.3575949668884277,
      "step": 3603
    },
    {
      "epoch": 0.471612006215752,
      "grad_norm": 7.2206501960754395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15036100149154663,
      "logits/rejected": 0.21861481666564941,
      "logps/chosen": -566.8026123046875,
      "logps/rejected": -549.1964721679688,
      "loss": 0.5682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2974728047847748,
      "rewards/margins": 1.6461548805236816,
      "rewards/rejected": -1.348681926727295,
      "step": 3604
    },
    {
      "epoch": 0.4717428641531038,
      "grad_norm": 4.862760543823242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17122727632522583,
      "logits/rejected": -0.06673794239759445,
      "logps/chosen": -441.455322265625,
      "logps/rejected": -339.36920166015625,
      "loss": 0.3389,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09348469972610474,
      "rewards/margins": 1.796407699584961,
      "rewards/rejected": -1.702923059463501,
      "step": 3605
    },
    {
      "epoch": 0.4718737220904555,
      "grad_norm": 9.037214279174805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16033132374286652,
      "logits/rejected": -0.14717009663581848,
      "logps/chosen": -531.1392822265625,
      "logps/rejected": -294.7256164550781,
      "loss": 0.7777,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7541661858558655,
      "rewards/margins": 0.6116911172866821,
      "rewards/rejected": -1.3658573627471924,
      "step": 3606
    },
    {
      "epoch": 0.4720045800278073,
      "grad_norm": 4.341184139251709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07089196145534515,
      "logits/rejected": -0.0334477424621582,
      "logps/chosen": -337.166015625,
      "logps/rejected": -496.99560546875,
      "loss": 0.3352,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.043406106531620026,
      "rewards/margins": 2.1076769828796387,
      "rewards/rejected": -2.0642709732055664,
      "step": 3607
    },
    {
      "epoch": 0.47213543796515905,
      "grad_norm": 5.272592067718506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14548718929290771,
      "logits/rejected": -0.03128064423799515,
      "logps/chosen": -301.1748962402344,
      "logps/rejected": -248.726318359375,
      "loss": 0.6569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3344350755214691,
      "rewards/margins": 0.4121536612510681,
      "rewards/rejected": -0.7465887665748596,
      "step": 3608
    },
    {
      "epoch": 0.47226629590251085,
      "grad_norm": 7.391636848449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05460657924413681,
      "logits/rejected": -0.046425748616456985,
      "logps/chosen": -533.1436157226562,
      "logps/rejected": -598.7824096679688,
      "loss": 0.3714,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1830902248620987,
      "rewards/margins": 1.1996651887893677,
      "rewards/rejected": -1.3827553987503052,
      "step": 3609
    },
    {
      "epoch": 0.4723971538398626,
      "grad_norm": 7.951437950134277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19204363226890564,
      "logits/rejected": -0.25078076124191284,
      "logps/chosen": -311.03167724609375,
      "logps/rejected": -320.7986755371094,
      "loss": 0.648,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11768627166748047,
      "rewards/margins": 0.5813325643539429,
      "rewards/rejected": -0.6990188956260681,
      "step": 3610
    },
    {
      "epoch": 0.4725280117772144,
      "grad_norm": 8.682351112365723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2980944514274597,
      "logits/rejected": 0.3781895041465759,
      "logps/chosen": -429.0497131347656,
      "logps/rejected": -467.9609069824219,
      "loss": 0.6249,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.4911715090274811,
      "rewards/margins": 0.34214574098587036,
      "rewards/rejected": 0.14902573823928833,
      "step": 3611
    },
    {
      "epoch": 0.4726588697145661,
      "grad_norm": 9.56739330291748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0014779288321733475,
      "logits/rejected": 0.20612439513206482,
      "logps/chosen": -536.47412109375,
      "logps/rejected": -458.3394775390625,
      "loss": 0.6668,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4676470160484314,
      "rewards/margins": 0.4176191985607147,
      "rewards/rejected": -0.885266125202179,
      "step": 3612
    },
    {
      "epoch": 0.4727897276519179,
      "grad_norm": 8.165114402770996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16729803383350372,
      "logits/rejected": 0.12005545198917389,
      "logps/chosen": -639.0311279296875,
      "logps/rejected": -654.3228149414062,
      "loss": 0.5443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5967038869857788,
      "rewards/margins": 1.014206051826477,
      "rewards/rejected": -0.41750219464302063,
      "step": 3613
    },
    {
      "epoch": 0.47292058558926964,
      "grad_norm": 5.964420318603516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43490469455718994,
      "logits/rejected": 0.23251554369926453,
      "logps/chosen": -504.882080078125,
      "logps/rejected": -499.9354553222656,
      "loss": 0.417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7506264448165894,
      "rewards/margins": 0.848074734210968,
      "rewards/rejected": -0.09744834154844284,
      "step": 3614
    },
    {
      "epoch": 0.47305144352662143,
      "grad_norm": 7.134206771850586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13113030791282654,
      "logits/rejected": -0.08509823679924011,
      "logps/chosen": -645.5717163085938,
      "logps/rejected": -647.6441650390625,
      "loss": 0.4899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.054815687239170074,
      "rewards/margins": 2.8701725006103516,
      "rewards/rejected": -2.815356969833374,
      "step": 3615
    },
    {
      "epoch": 0.47318230146397317,
      "grad_norm": 10.329704284667969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2987394332885742,
      "logits/rejected": 0.0615578256547451,
      "logps/chosen": -370.1143493652344,
      "logps/rejected": -286.5317077636719,
      "loss": 1.0051,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.20358049869537354,
      "rewards/margins": -0.3080179691314697,
      "rewards/rejected": 0.1044374629855156,
      "step": 3616
    },
    {
      "epoch": 0.47331315940132496,
      "grad_norm": 4.523740768432617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2130248248577118,
      "logits/rejected": 0.05642083287239075,
      "logps/chosen": -506.8818359375,
      "logps/rejected": -338.9543151855469,
      "loss": 0.3711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6200830936431885,
      "rewards/margins": 1.0460950136184692,
      "rewards/rejected": -0.4260120391845703,
      "step": 3617
    },
    {
      "epoch": 0.4734440173386767,
      "grad_norm": 4.914541244506836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15810240805149078,
      "logits/rejected": -0.22240988910198212,
      "logps/chosen": -496.2754821777344,
      "logps/rejected": -362.0787658691406,
      "loss": 0.4583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3965654671192169,
      "rewards/margins": 0.696625292301178,
      "rewards/rejected": -0.30005988478660583,
      "step": 3618
    },
    {
      "epoch": 0.4735748752760285,
      "grad_norm": 5.330208778381348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28290611505508423,
      "logits/rejected": 0.12051193416118622,
      "logps/chosen": -541.1282958984375,
      "logps/rejected": -437.0138854980469,
      "loss": 0.3596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8808607459068298,
      "rewards/margins": 1.173827886581421,
      "rewards/rejected": -0.2929672300815582,
      "step": 3619
    },
    {
      "epoch": 0.4737057332133802,
      "grad_norm": 8.624987602233887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42273131012916565,
      "logits/rejected": 0.3502381443977356,
      "logps/chosen": -548.3135375976562,
      "logps/rejected": -530.3372802734375,
      "loss": 0.7432,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11632749438285828,
      "rewards/margins": 0.26654568314552307,
      "rewards/rejected": -0.1502181738615036,
      "step": 3620
    },
    {
      "epoch": 0.473836591150732,
      "grad_norm": 4.917083263397217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17061586678028107,
      "logits/rejected": 0.023124465718865395,
      "logps/chosen": -426.5458984375,
      "logps/rejected": -448.309814453125,
      "loss": 0.4549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08784288167953491,
      "rewards/margins": 0.9778136610984802,
      "rewards/rejected": -0.8899708390235901,
      "step": 3621
    },
    {
      "epoch": 0.47396744908808375,
      "grad_norm": 11.128300666809082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11310591548681259,
      "logits/rejected": -0.08730240166187286,
      "logps/chosen": -626.2732543945312,
      "logps/rejected": -369.322509765625,
      "loss": 0.716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7008682489395142,
      "rewards/margins": 0.5339162349700928,
      "rewards/rejected": -1.234784483909607,
      "step": 3622
    },
    {
      "epoch": 0.47409830702543554,
      "grad_norm": 5.9361395835876465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15904727578163147,
      "logits/rejected": 0.19315211474895477,
      "logps/chosen": -346.03363037109375,
      "logps/rejected": -375.88885498046875,
      "loss": 0.5594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16093750298023224,
      "rewards/margins": 0.4958398640155792,
      "rewards/rejected": -0.3349023759365082,
      "step": 3623
    },
    {
      "epoch": 0.4742291649627873,
      "grad_norm": 4.785800457000732,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4718402028083801,
      "logits/rejected": 0.3917487859725952,
      "logps/chosen": -538.3148193359375,
      "logps/rejected": -433.8194580078125,
      "loss": 0.3661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1074333190917969,
      "rewards/margins": 0.9973659515380859,
      "rewards/rejected": 0.11006738245487213,
      "step": 3624
    },
    {
      "epoch": 0.474360022900139,
      "grad_norm": 6.292489051818848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22507646679878235,
      "logits/rejected": 0.2920648157596588,
      "logps/chosen": -388.7741394042969,
      "logps/rejected": -545.6498413085938,
      "loss": 0.4947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2924175262451172,
      "rewards/margins": 0.6156883239746094,
      "rewards/rejected": -0.3232707977294922,
      "step": 3625
    },
    {
      "epoch": 0.4744908808374908,
      "grad_norm": 6.074997901916504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.266971617937088,
      "logits/rejected": 0.47968757152557373,
      "logps/chosen": -394.9344177246094,
      "logps/rejected": -446.3341369628906,
      "loss": 0.3914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15021111071109772,
      "rewards/margins": 0.9025983810424805,
      "rewards/rejected": -0.7523872256278992,
      "step": 3626
    },
    {
      "epoch": 0.47462173877484254,
      "grad_norm": 6.516530513763428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39911913871765137,
      "logits/rejected": 0.36031049489974976,
      "logps/chosen": -360.75982666015625,
      "logps/rejected": -378.68951416015625,
      "loss": 0.5637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6376127004623413,
      "rewards/margins": 0.7937446236610413,
      "rewards/rejected": -0.15613192319869995,
      "step": 3627
    },
    {
      "epoch": 0.47475259671219433,
      "grad_norm": 7.710360527038574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1591842770576477,
      "logits/rejected": 0.1646154522895813,
      "logps/chosen": -596.946044921875,
      "logps/rejected": -602.4222412109375,
      "loss": 0.5297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5777113437652588,
      "rewards/margins": 0.5117629766464233,
      "rewards/rejected": -1.0894742012023926,
      "step": 3628
    },
    {
      "epoch": 0.47488345464954607,
      "grad_norm": 8.636003494262695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08837562799453735,
      "logits/rejected": 0.014290697872638702,
      "logps/chosen": -479.1954345703125,
      "logps/rejected": -530.474365234375,
      "loss": 0.8054,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3485906422138214,
      "rewards/margins": 0.6775484085083008,
      "rewards/rejected": -0.32895776629447937,
      "step": 3629
    },
    {
      "epoch": 0.47501431258689786,
      "grad_norm": 4.705561637878418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3159545660018921,
      "logits/rejected": 0.29921698570251465,
      "logps/chosen": -526.3527221679688,
      "logps/rejected": -338.8634033203125,
      "loss": 0.3263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3495016396045685,
      "rewards/margins": 1.1677263975143433,
      "rewards/rejected": -0.8182247877120972,
      "step": 3630
    },
    {
      "epoch": 0.4751451705242496,
      "grad_norm": 5.460030555725098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2089189887046814,
      "logits/rejected": 0.2608778178691864,
      "logps/chosen": -385.6378479003906,
      "logps/rejected": -288.6380615234375,
      "loss": 0.4431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2641979455947876,
      "rewards/margins": 0.7728716135025024,
      "rewards/rejected": -0.5086736679077148,
      "step": 3631
    },
    {
      "epoch": 0.4752760284616014,
      "grad_norm": 6.574944019317627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11268407106399536,
      "logits/rejected": -0.10192639380693436,
      "logps/chosen": -474.0170593261719,
      "logps/rejected": -442.7115783691406,
      "loss": 0.5967,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.41128623485565186,
      "rewards/margins": 0.5203254818916321,
      "rewards/rejected": -0.9316115975379944,
      "step": 3632
    },
    {
      "epoch": 0.4754068863989531,
      "grad_norm": 5.189497947692871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21844029426574707,
      "logits/rejected": 0.004116659983992577,
      "logps/chosen": -647.523193359375,
      "logps/rejected": -379.92523193359375,
      "loss": 0.3228,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18554210662841797,
      "rewards/margins": 1.4135771989822388,
      "rewards/rejected": -1.2280350923538208,
      "step": 3633
    },
    {
      "epoch": 0.4755377443363049,
      "grad_norm": 6.080350875854492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2155483365058899,
      "logits/rejected": 0.04836389794945717,
      "logps/chosen": -467.7251281738281,
      "logps/rejected": -503.791748046875,
      "loss": 0.5233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3905012309551239,
      "rewards/margins": 0.847231924533844,
      "rewards/rejected": -0.45673060417175293,
      "step": 3634
    },
    {
      "epoch": 0.47566860227365665,
      "grad_norm": 7.147098064422607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07257848978042603,
      "logits/rejected": -0.10417451709508896,
      "logps/chosen": -422.779052734375,
      "logps/rejected": -380.6009826660156,
      "loss": 0.5854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.374158650636673,
      "rewards/margins": 0.41931039094924927,
      "rewards/rejected": -0.7934690117835999,
      "step": 3635
    },
    {
      "epoch": 0.47579946021100844,
      "grad_norm": 7.83415412902832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19688525795936584,
      "logits/rejected": -0.025086164474487305,
      "logps/chosen": -604.7119140625,
      "logps/rejected": -498.841064453125,
      "loss": 0.3991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7310850024223328,
      "rewards/margins": 1.580975890159607,
      "rewards/rejected": -0.849890947341919,
      "step": 3636
    },
    {
      "epoch": 0.4759303181483602,
      "grad_norm": 8.133283615112305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12449739873409271,
      "logits/rejected": -0.2440289556980133,
      "logps/chosen": -478.8050231933594,
      "logps/rejected": -556.7525634765625,
      "loss": 0.5521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5620986819267273,
      "rewards/margins": 1.2743175029754639,
      "rewards/rejected": -0.7122189402580261,
      "step": 3637
    },
    {
      "epoch": 0.47606117608571197,
      "grad_norm": 4.950321674346924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14173121750354767,
      "logits/rejected": 0.28095802664756775,
      "logps/chosen": -344.8243103027344,
      "logps/rejected": -306.1565246582031,
      "loss": 0.4544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00646746251732111,
      "rewards/margins": 0.8120195865631104,
      "rewards/rejected": -0.8055521249771118,
      "step": 3638
    },
    {
      "epoch": 0.4761920340230637,
      "grad_norm": 7.0517168045043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07436589151620865,
      "logits/rejected": -0.16178354620933533,
      "logps/chosen": -357.4211730957031,
      "logps/rejected": -470.57464599609375,
      "loss": 0.5143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12779901921749115,
      "rewards/margins": 1.2415881156921387,
      "rewards/rejected": -1.1137890815734863,
      "step": 3639
    },
    {
      "epoch": 0.4763228919604155,
      "grad_norm": 11.307954788208008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01245082076638937,
      "logits/rejected": 0.09139203280210495,
      "logps/chosen": -373.61810302734375,
      "logps/rejected": -358.810302734375,
      "loss": 0.7264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8646402359008789,
      "rewards/margins": 0.3695202171802521,
      "rewards/rejected": -1.2341605424880981,
      "step": 3640
    },
    {
      "epoch": 0.47645374989776723,
      "grad_norm": 7.781092643737793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.042282454669475555,
      "logits/rejected": 0.17006957530975342,
      "logps/chosen": -526.854736328125,
      "logps/rejected": -437.7469177246094,
      "loss": 0.5817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15508724749088287,
      "rewards/margins": 0.7474257946014404,
      "rewards/rejected": -0.5923386216163635,
      "step": 3641
    },
    {
      "epoch": 0.476584607835119,
      "grad_norm": 6.168133735656738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2845091223716736,
      "logits/rejected": 0.2821476459503174,
      "logps/chosen": -332.0418395996094,
      "logps/rejected": -319.5281982421875,
      "loss": 0.4549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.543728768825531,
      "rewards/margins": 0.687923789024353,
      "rewards/rejected": -0.14419500529766083,
      "step": 3642
    },
    {
      "epoch": 0.47671546577247076,
      "grad_norm": 11.90790843963623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06266807019710541,
      "logits/rejected": -0.23408418893814087,
      "logps/chosen": -454.8277587890625,
      "logps/rejected": -512.86279296875,
      "loss": 0.9683,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18080481886863708,
      "rewards/margins": 0.5125038623809814,
      "rewards/rejected": -0.331699013710022,
      "step": 3643
    },
    {
      "epoch": 0.47684632370982255,
      "grad_norm": 5.55804443359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10093305259943008,
      "logits/rejected": -0.0025902464985847473,
      "logps/chosen": -545.4754028320312,
      "logps/rejected": -507.31182861328125,
      "loss": 0.5008,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0697207897901535,
      "rewards/margins": 0.9776960611343384,
      "rewards/rejected": -1.0474169254302979,
      "step": 3644
    },
    {
      "epoch": 0.4769771816471743,
      "grad_norm": 6.95733118057251,
      "learning_rate": 5e-06,
      "logits/chosen": -0.039608776569366455,
      "logits/rejected": -0.2799304723739624,
      "logps/chosen": -370.7861328125,
      "logps/rejected": -544.0518798828125,
      "loss": 0.6452,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6704567074775696,
      "rewards/margins": 1.174523949623108,
      "rewards/rejected": -1.8449805974960327,
      "step": 3645
    },
    {
      "epoch": 0.477108039584526,
      "grad_norm": 5.646666049957275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041341349482536316,
      "logits/rejected": -0.004457294940948486,
      "logps/chosen": -369.7147521972656,
      "logps/rejected": -453.0908508300781,
      "loss": 0.4422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5306484699249268,
      "rewards/margins": 0.8862140774726868,
      "rewards/rejected": -0.3555656671524048,
      "step": 3646
    },
    {
      "epoch": 0.4772388975218778,
      "grad_norm": 7.748988628387451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014982974156737328,
      "logits/rejected": 0.02613823488354683,
      "logps/chosen": -396.4660339355469,
      "logps/rejected": -445.5457763671875,
      "loss": 0.8687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.35005608201026917,
      "rewards/margins": -0.0805889368057251,
      "rewards/rejected": 0.4306449890136719,
      "step": 3647
    },
    {
      "epoch": 0.47736975545922955,
      "grad_norm": 8.73572826385498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18969093263149261,
      "logits/rejected": 0.21995463967323303,
      "logps/chosen": -480.34552001953125,
      "logps/rejected": -451.43402099609375,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21327973902225494,
      "rewards/margins": 1.2450257539749146,
      "rewards/rejected": -1.0317459106445312,
      "step": 3648
    },
    {
      "epoch": 0.47750061339658134,
      "grad_norm": 5.192647933959961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35567325353622437,
      "logits/rejected": 0.3293687403202057,
      "logps/chosen": -559.338623046875,
      "logps/rejected": -500.7237548828125,
      "loss": 0.3375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9867526888847351,
      "rewards/margins": 1.4216625690460205,
      "rewards/rejected": -0.4349098205566406,
      "step": 3649
    },
    {
      "epoch": 0.4776314713339331,
      "grad_norm": 6.08458948135376,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2749955356121063,
      "logits/rejected": -0.14200091361999512,
      "logps/chosen": -543.2252197265625,
      "logps/rejected": -739.19580078125,
      "loss": 0.3958,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.480119526386261,
      "rewards/margins": 1.8189560174942017,
      "rewards/rejected": -1.3388367891311646,
      "step": 3650
    },
    {
      "epoch": 0.47776232927128487,
      "grad_norm": 9.025084495544434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23391546308994293,
      "logits/rejected": 0.2377595752477646,
      "logps/chosen": -407.90960693359375,
      "logps/rejected": -320.3172607421875,
      "loss": 0.6568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.017944902181625366,
      "rewards/margins": 0.363345742225647,
      "rewards/rejected": -0.38129064440727234,
      "step": 3651
    },
    {
      "epoch": 0.4778931872086366,
      "grad_norm": 5.894155502319336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1945439875125885,
      "logits/rejected": 0.17538835108280182,
      "logps/chosen": -585.4415283203125,
      "logps/rejected": -377.5279846191406,
      "loss": 0.4189,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22277864813804626,
      "rewards/margins": 1.3224396705627441,
      "rewards/rejected": -1.0996609926223755,
      "step": 3652
    },
    {
      "epoch": 0.4780240451459884,
      "grad_norm": 7.337662696838379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3050892949104309,
      "logits/rejected": 0.22821155190467834,
      "logps/chosen": -480.4227294921875,
      "logps/rejected": -415.807861328125,
      "loss": 0.4953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21485941112041473,
      "rewards/margins": 1.047331690788269,
      "rewards/rejected": -0.8324722051620483,
      "step": 3653
    },
    {
      "epoch": 0.47815490308334013,
      "grad_norm": 6.852060317993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1266079694032669,
      "logits/rejected": -0.12867021560668945,
      "logps/chosen": -461.6947937011719,
      "logps/rejected": -454.1525573730469,
      "loss": 0.5388,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.6239122748374939,
      "rewards/margins": 0.7950090765953064,
      "rewards/rejected": -0.1710968315601349,
      "step": 3654
    },
    {
      "epoch": 0.4782857610206919,
      "grad_norm": 6.243147373199463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12940570712089539,
      "logits/rejected": 0.2504112720489502,
      "logps/chosen": -440.55377197265625,
      "logps/rejected": -435.883056640625,
      "loss": 0.584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.719352126121521,
      "rewards/margins": 1.0691673755645752,
      "rewards/rejected": -0.3498152494430542,
      "step": 3655
    },
    {
      "epoch": 0.47841661895804366,
      "grad_norm": 9.565680503845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14127488434314728,
      "logits/rejected": -0.006361909210681915,
      "logps/chosen": -362.1464538574219,
      "logps/rejected": -361.4790344238281,
      "loss": 0.6381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2174755036830902,
      "rewards/margins": 0.27069395780563354,
      "rewards/rejected": -0.48816949129104614,
      "step": 3656
    },
    {
      "epoch": 0.47854747689539545,
      "grad_norm": 8.409964561462402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.281885027885437,
      "logits/rejected": -0.04259190708398819,
      "logps/chosen": -484.3807373046875,
      "logps/rejected": -410.04205322265625,
      "loss": 0.8637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27815571427345276,
      "rewards/margins": 0.16717222332954407,
      "rewards/rejected": -0.4453279376029968,
      "step": 3657
    },
    {
      "epoch": 0.4786783348327472,
      "grad_norm": 11.03475570678711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20841532945632935,
      "logits/rejected": 0.057015515863895416,
      "logps/chosen": -393.65985107421875,
      "logps/rejected": -601.3531494140625,
      "loss": 1.107,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11951391398906708,
      "rewards/margins": 0.08462408185005188,
      "rewards/rejected": 0.03488977253437042,
      "step": 3658
    },
    {
      "epoch": 0.478809192770099,
      "grad_norm": 8.106983184814453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11466740071773529,
      "logits/rejected": 0.09797152131795883,
      "logps/chosen": -552.342041015625,
      "logps/rejected": -568.2870483398438,
      "loss": 0.6502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3637051582336426,
      "rewards/margins": 0.19990691542625427,
      "rewards/rejected": -1.5636119842529297,
      "step": 3659
    },
    {
      "epoch": 0.4789400507074507,
      "grad_norm": 9.597594261169434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07650875300168991,
      "logits/rejected": -0.052754126489162445,
      "logps/chosen": -543.1807861328125,
      "logps/rejected": -527.8060302734375,
      "loss": 1.0096,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6048554182052612,
      "rewards/margins": -0.15274447202682495,
      "rewards/rejected": -0.4521108865737915,
      "step": 3660
    },
    {
      "epoch": 0.4790709086448025,
      "grad_norm": 3.7625579833984375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13990408182144165,
      "logits/rejected": -0.056000180542469025,
      "logps/chosen": -498.04913330078125,
      "logps/rejected": -511.49639892578125,
      "loss": 0.2866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8474199175834656,
      "rewards/margins": 1.6485202312469482,
      "rewards/rejected": -0.8011003732681274,
      "step": 3661
    },
    {
      "epoch": 0.47920176658215424,
      "grad_norm": 6.283071517944336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013439375907182693,
      "logits/rejected": -0.06581638753414154,
      "logps/chosen": -727.4756469726562,
      "logps/rejected": -513.6513061523438,
      "loss": 0.3682,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3751499652862549,
      "rewards/margins": 1.9925785064697266,
      "rewards/rejected": -0.6174286603927612,
      "step": 3662
    },
    {
      "epoch": 0.47933262451950603,
      "grad_norm": 5.478930473327637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09351691603660583,
      "logits/rejected": 0.02139807865023613,
      "logps/chosen": -432.21783447265625,
      "logps/rejected": -283.3392639160156,
      "loss": 0.4321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3253067135810852,
      "rewards/margins": 0.7928662300109863,
      "rewards/rejected": -0.46755945682525635,
      "step": 3663
    },
    {
      "epoch": 0.47946348245685777,
      "grad_norm": 8.245041847229004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5424625277519226,
      "logits/rejected": 0.44588759541511536,
      "logps/chosen": -473.0478210449219,
      "logps/rejected": -508.112060546875,
      "loss": 0.5741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34350165724754333,
      "rewards/margins": 0.7312332391738892,
      "rewards/rejected": -1.0747349262237549,
      "step": 3664
    },
    {
      "epoch": 0.47959434039420956,
      "grad_norm": 5.458933353424072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3129251301288605,
      "logits/rejected": 0.2823687791824341,
      "logps/chosen": -450.0643310546875,
      "logps/rejected": -491.9331970214844,
      "loss": 0.3495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9944576025009155,
      "rewards/margins": 1.0838147401809692,
      "rewards/rejected": -0.08935704082250595,
      "step": 3665
    },
    {
      "epoch": 0.4797251983315613,
      "grad_norm": 7.454843044281006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3965912163257599,
      "logits/rejected": 0.5026612281799316,
      "logps/chosen": -390.9463195800781,
      "logps/rejected": -376.7037353515625,
      "loss": 0.6274,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08333168178796768,
      "rewards/margins": 0.2923808991909027,
      "rewards/rejected": -0.3757126033306122,
      "step": 3666
    },
    {
      "epoch": 0.47985605626891303,
      "grad_norm": 8.829578399658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4824094772338867,
      "logits/rejected": 0.4158855080604553,
      "logps/chosen": -496.396240234375,
      "logps/rejected": -445.31365966796875,
      "loss": 0.7728,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1943565309047699,
      "rewards/margins": 0.3228588104248047,
      "rewards/rejected": -0.517215371131897,
      "step": 3667
    },
    {
      "epoch": 0.4799869142062648,
      "grad_norm": 3.536400556564331,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04481007531285286,
      "logits/rejected": 0.010886133648455143,
      "logps/chosen": -328.66241455078125,
      "logps/rejected": -547.9779663085938,
      "loss": 0.1814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2858121991157532,
      "rewards/margins": 2.3523292541503906,
      "rewards/rejected": -2.0665171146392822,
      "step": 3668
    },
    {
      "epoch": 0.48011777214361656,
      "grad_norm": 7.0275349617004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08705516904592514,
      "logits/rejected": 0.22279933094978333,
      "logps/chosen": -335.67218017578125,
      "logps/rejected": -289.25665283203125,
      "loss": 0.6562,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1684524416923523,
      "rewards/margins": 0.31614363193511963,
      "rewards/rejected": -0.4845961034297943,
      "step": 3669
    },
    {
      "epoch": 0.48024863008096835,
      "grad_norm": 7.3448967933654785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30917981266975403,
      "logits/rejected": 0.2965405583381653,
      "logps/chosen": -484.2349853515625,
      "logps/rejected": -597.3878784179688,
      "loss": 0.4084,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5135976672172546,
      "rewards/margins": 1.562969446182251,
      "rewards/rejected": -1.0493717193603516,
      "step": 3670
    },
    {
      "epoch": 0.4803794880183201,
      "grad_norm": 9.73646354675293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1601412296295166,
      "logits/rejected": -0.2206183671951294,
      "logps/chosen": -679.5675048828125,
      "logps/rejected": -386.2097473144531,
      "loss": 0.5471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49793344736099243,
      "rewards/margins": 0.9121570587158203,
      "rewards/rejected": -1.410090446472168,
      "step": 3671
    },
    {
      "epoch": 0.4805103459556719,
      "grad_norm": 2.870126962661743,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016262680292129517,
      "logits/rejected": 0.09648118168115616,
      "logps/chosen": -476.4217834472656,
      "logps/rejected": -380.06964111328125,
      "loss": 0.1958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7503699660301208,
      "rewards/margins": 1.9552839994430542,
      "rewards/rejected": -1.2049139738082886,
      "step": 3672
    },
    {
      "epoch": 0.4806412038930236,
      "grad_norm": 9.879096984863281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12220276892185211,
      "logits/rejected": 0.08370564132928848,
      "logps/chosen": -472.9385070800781,
      "logps/rejected": -450.7756652832031,
      "loss": 0.8319,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7150335311889648,
      "rewards/margins": 0.475907564163208,
      "rewards/rejected": -1.1909410953521729,
      "step": 3673
    },
    {
      "epoch": 0.4807720618303754,
      "grad_norm": 7.745406150817871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26412448287010193,
      "logits/rejected": -0.1428966373205185,
      "logps/chosen": -591.4613037109375,
      "logps/rejected": -409.29833984375,
      "loss": 0.5229,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2068614810705185,
      "rewards/margins": 0.7990284562110901,
      "rewards/rejected": -1.005889892578125,
      "step": 3674
    },
    {
      "epoch": 0.48090291976772714,
      "grad_norm": 5.826622009277344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2663232386112213,
      "logits/rejected": 0.2408571094274521,
      "logps/chosen": -418.4573669433594,
      "logps/rejected": -404.9129333496094,
      "loss": 0.3957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.769640326499939,
      "rewards/margins": 1.1279034614562988,
      "rewards/rejected": -1.8975436687469482,
      "step": 3675
    },
    {
      "epoch": 0.48103377770507894,
      "grad_norm": 4.239663124084473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20337830483913422,
      "logits/rejected": 0.0454377606511116,
      "logps/chosen": -451.11541748046875,
      "logps/rejected": -414.0987243652344,
      "loss": 0.3854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4478123188018799,
      "rewards/margins": 1.1378200054168701,
      "rewards/rejected": -0.6900076270103455,
      "step": 3676
    },
    {
      "epoch": 0.48116463564243067,
      "grad_norm": 6.379122734069824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10199504345655441,
      "logits/rejected": 0.20619367063045502,
      "logps/chosen": -451.6539306640625,
      "logps/rejected": -436.8226318359375,
      "loss": 0.4248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7829598188400269,
      "rewards/margins": 0.7592905163764954,
      "rewards/rejected": 0.023669227957725525,
      "step": 3677
    },
    {
      "epoch": 0.48129549357978246,
      "grad_norm": 7.420688629150391,
      "learning_rate": 5e-06,
      "logits/chosen": 0.001003466546535492,
      "logits/rejected": 0.14414706826210022,
      "logps/chosen": -448.98370361328125,
      "logps/rejected": -375.7616882324219,
      "loss": 0.9127,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04361321032047272,
      "rewards/margins": 0.5600214004516602,
      "rewards/rejected": -0.6036347150802612,
      "step": 3678
    },
    {
      "epoch": 0.4814263515171342,
      "grad_norm": 6.630903244018555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02447318285703659,
      "logits/rejected": -0.007598251104354858,
      "logps/chosen": -508.83856201171875,
      "logps/rejected": -572.2742919921875,
      "loss": 0.4102,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2148987054824829,
      "rewards/margins": 1.2427031993865967,
      "rewards/rejected": -1.4576019048690796,
      "step": 3679
    },
    {
      "epoch": 0.481557209454486,
      "grad_norm": 8.834757804870605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3967558443546295,
      "logits/rejected": 0.24067719280719757,
      "logps/chosen": -407.6842041015625,
      "logps/rejected": -520.0601806640625,
      "loss": 0.6807,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2572464346885681,
      "rewards/margins": 0.4822757840156555,
      "rewards/rejected": -0.22502939403057098,
      "step": 3680
    },
    {
      "epoch": 0.4816880673918377,
      "grad_norm": 7.305866241455078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4020737409591675,
      "logits/rejected": 0.28665363788604736,
      "logps/chosen": -513.8870239257812,
      "logps/rejected": -548.0377807617188,
      "loss": 0.5226,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07106898725032806,
      "rewards/margins": 0.7625967264175415,
      "rewards/rejected": -0.6915277242660522,
      "step": 3681
    },
    {
      "epoch": 0.4818189253291895,
      "grad_norm": 6.177817344665527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023981764912605286,
      "logits/rejected": 0.17030441761016846,
      "logps/chosen": -416.3388977050781,
      "logps/rejected": -568.2047119140625,
      "loss": 0.3063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08976822346448898,
      "rewards/margins": 1.1955451965332031,
      "rewards/rejected": -1.2853132486343384,
      "step": 3682
    },
    {
      "epoch": 0.48194978326654125,
      "grad_norm": 8.195831298828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3306576609611511,
      "logits/rejected": 0.41568443179130554,
      "logps/chosen": -623.6984252929688,
      "logps/rejected": -629.3451538085938,
      "loss": 0.6313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16694697737693787,
      "rewards/margins": 0.5105730295181274,
      "rewards/rejected": -0.6775199770927429,
      "step": 3683
    },
    {
      "epoch": 0.48208064120389305,
      "grad_norm": 7.640514850616455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3184644281864166,
      "logits/rejected": 8.832290768623352e-05,
      "logps/chosen": -530.5977172851562,
      "logps/rejected": -325.400146484375,
      "loss": 0.5846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18560472130775452,
      "rewards/margins": 0.4167093336582184,
      "rewards/rejected": -0.23110467195510864,
      "step": 3684
    },
    {
      "epoch": 0.4822114991412448,
      "grad_norm": 5.262437343597412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06681960076093674,
      "logits/rejected": -0.012855269014835358,
      "logps/chosen": -412.8843994140625,
      "logps/rejected": -424.736083984375,
      "loss": 0.3593,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4086591601371765,
      "rewards/margins": 1.3751943111419678,
      "rewards/rejected": -1.783853530883789,
      "step": 3685
    },
    {
      "epoch": 0.4823423570785966,
      "grad_norm": 7.439699649810791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006472880952060223,
      "logits/rejected": 0.168158620595932,
      "logps/chosen": -376.4413146972656,
      "logps/rejected": -468.7820129394531,
      "loss": 0.5546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1359223872423172,
      "rewards/margins": 0.7069556713104248,
      "rewards/rejected": -0.571033239364624,
      "step": 3686
    },
    {
      "epoch": 0.4824732150159483,
      "grad_norm": 6.978878974914551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17857776582241058,
      "logits/rejected": -0.028146633878350258,
      "logps/chosen": -465.588134765625,
      "logps/rejected": -469.33477783203125,
      "loss": 0.3859,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27598971128463745,
      "rewards/margins": 1.6027202606201172,
      "rewards/rejected": -1.3267308473587036,
      "step": 3687
    },
    {
      "epoch": 0.4826040729533001,
      "grad_norm": 4.756854057312012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05311072617769241,
      "logits/rejected": -0.08304218202829361,
      "logps/chosen": -479.3401184082031,
      "logps/rejected": -366.54632568359375,
      "loss": 0.3366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42422354221343994,
      "rewards/margins": 1.2657784223556519,
      "rewards/rejected": -0.8415549397468567,
      "step": 3688
    },
    {
      "epoch": 0.48273493089065184,
      "grad_norm": 8.816282272338867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.237403005361557,
      "logits/rejected": 0.2869234085083008,
      "logps/chosen": -411.2705078125,
      "logps/rejected": -358.6640930175781,
      "loss": 0.6496,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08982830494642258,
      "rewards/margins": 0.522480845451355,
      "rewards/rejected": -0.6123090386390686,
      "step": 3689
    },
    {
      "epoch": 0.4828657888280036,
      "grad_norm": 4.02522087097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060604508966207504,
      "logits/rejected": -0.016921840608119965,
      "logps/chosen": -358.5236511230469,
      "logps/rejected": -336.78265380859375,
      "loss": 0.3225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7138356566429138,
      "rewards/margins": 1.3471190929412842,
      "rewards/rejected": -0.6332835555076599,
      "step": 3690
    },
    {
      "epoch": 0.48299664676535536,
      "grad_norm": 5.782899856567383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02698417752981186,
      "logits/rejected": -0.04033084213733673,
      "logps/chosen": -464.8489685058594,
      "logps/rejected": -416.2020568847656,
      "loss": 0.4809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3107752203941345,
      "rewards/margins": 1.0775619745254517,
      "rewards/rejected": -1.3883371353149414,
      "step": 3691
    },
    {
      "epoch": 0.4831275047027071,
      "grad_norm": 9.065817832946777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33941739797592163,
      "logits/rejected": 0.1876479983329773,
      "logps/chosen": -583.0435791015625,
      "logps/rejected": -682.5438842773438,
      "loss": 0.5333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5401504635810852,
      "rewards/margins": 1.16100013256073,
      "rewards/rejected": -0.6208497285842896,
      "step": 3692
    },
    {
      "epoch": 0.4832583626400589,
      "grad_norm": 11.048068046569824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2406228482723236,
      "logits/rejected": 0.1958930492401123,
      "logps/chosen": -589.9888916015625,
      "logps/rejected": -637.8128051757812,
      "loss": 0.6869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9152764081954956,
      "rewards/margins": 0.3828887939453125,
      "rewards/rejected": -1.2981650829315186,
      "step": 3693
    },
    {
      "epoch": 0.4833892205774106,
      "grad_norm": 7.3204755783081055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29865241050720215,
      "logits/rejected": 0.147918701171875,
      "logps/chosen": -463.9039306640625,
      "logps/rejected": -570.094970703125,
      "loss": 0.4063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09020320326089859,
      "rewards/margins": 0.9736768007278442,
      "rewards/rejected": -0.8834736347198486,
      "step": 3694
    },
    {
      "epoch": 0.4835200785147624,
      "grad_norm": 8.038102149963379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07926741242408752,
      "logits/rejected": 0.062097854912281036,
      "logps/chosen": -552.658935546875,
      "logps/rejected": -565.474365234375,
      "loss": 0.5412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4112081825733185,
      "rewards/margins": 0.7814207077026367,
      "rewards/rejected": -1.1926288604736328,
      "step": 3695
    },
    {
      "epoch": 0.48365093645211416,
      "grad_norm": 9.162827491760254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27028095722198486,
      "logits/rejected": 0.043484386056661606,
      "logps/chosen": -450.9194030761719,
      "logps/rejected": -383.264404296875,
      "loss": 0.9062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7097545862197876,
      "rewards/margins": 0.01705951988697052,
      "rewards/rejected": -0.7268141508102417,
      "step": 3696
    },
    {
      "epoch": 0.48378179438946595,
      "grad_norm": 7.563576698303223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2704101800918579,
      "logits/rejected": 0.11407262831926346,
      "logps/chosen": -558.2158203125,
      "logps/rejected": -385.5180969238281,
      "loss": 0.7071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.348224401473999,
      "rewards/margins": 0.5974923968315125,
      "rewards/rejected": -0.24926796555519104,
      "step": 3697
    },
    {
      "epoch": 0.4839126523268177,
      "grad_norm": 6.316544055938721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45922306180000305,
      "logits/rejected": 0.1717531532049179,
      "logps/chosen": -662.525146484375,
      "logps/rejected": -694.6748046875,
      "loss": 0.3825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4016481339931488,
      "rewards/margins": 1.705998420715332,
      "rewards/rejected": -1.3043502569198608,
      "step": 3698
    },
    {
      "epoch": 0.4840435102641695,
      "grad_norm": 6.306131362915039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0771760568022728,
      "logits/rejected": -0.11868956685066223,
      "logps/chosen": -502.9490966796875,
      "logps/rejected": -631.1689453125,
      "loss": 0.3695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16016024351119995,
      "rewards/margins": 1.3018569946289062,
      "rewards/rejected": -1.141696810722351,
      "step": 3699
    },
    {
      "epoch": 0.4841743682015212,
      "grad_norm": 7.064947605133057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08965140581130981,
      "logits/rejected": 0.03198094666004181,
      "logps/chosen": -324.6480407714844,
      "logps/rejected": -402.4660339355469,
      "loss": 0.5607,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7407928109169006,
      "rewards/margins": 0.6887660622596741,
      "rewards/rejected": -1.4295591115951538,
      "step": 3700
    },
    {
      "epoch": 0.484305226138873,
      "grad_norm": 7.741778373718262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2665303945541382,
      "logits/rejected": -0.049714237451553345,
      "logps/chosen": -392.86138916015625,
      "logps/rejected": -373.57513427734375,
      "loss": 0.6071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6118128299713135,
      "rewards/margins": 0.5246403813362122,
      "rewards/rejected": -1.1364532709121704,
      "step": 3701
    },
    {
      "epoch": 0.48443608407622474,
      "grad_norm": 7.92264461517334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11836694180965424,
      "logits/rejected": -0.03705529123544693,
      "logps/chosen": -587.2830810546875,
      "logps/rejected": -489.0880432128906,
      "loss": 0.5359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.059353306889534,
      "rewards/margins": 1.139062523841858,
      "rewards/rejected": -1.198415756225586,
      "step": 3702
    },
    {
      "epoch": 0.48456694201357653,
      "grad_norm": 5.813878536224365,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19662488996982574,
      "logits/rejected": 0.008568480610847473,
      "logps/chosen": -440.6883850097656,
      "logps/rejected": -476.3421325683594,
      "loss": 0.3576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.001987650990486145,
      "rewards/margins": 1.2938944101333618,
      "rewards/rejected": -1.291906714439392,
      "step": 3703
    },
    {
      "epoch": 0.48469779995092827,
      "grad_norm": 6.571241855621338,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1039678305387497,
      "logits/rejected": -0.09222062677145004,
      "logps/chosen": -376.2899475097656,
      "logps/rejected": -523.0498046875,
      "loss": 0.4558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24267534911632538,
      "rewards/margins": 0.993858277797699,
      "rewards/rejected": -1.2365336418151855,
      "step": 3704
    },
    {
      "epoch": 0.48482865788828006,
      "grad_norm": 6.397936820983887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26839014887809753,
      "logits/rejected": 0.10562343150377274,
      "logps/chosen": -362.98114013671875,
      "logps/rejected": -361.3396301269531,
      "loss": 0.3539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22793254256248474,
      "rewards/margins": 1.475948452949524,
      "rewards/rejected": -1.703881025314331,
      "step": 3705
    },
    {
      "epoch": 0.4849595158256318,
      "grad_norm": 10.296110153198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06735643744468689,
      "logits/rejected": -0.003974884748458862,
      "logps/chosen": -254.32835388183594,
      "logps/rejected": -325.84197998046875,
      "loss": 0.8634,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.060490369796753,
      "rewards/margins": 0.0690392255783081,
      "rewards/rejected": -1.129529595375061,
      "step": 3706
    },
    {
      "epoch": 0.4850903737629836,
      "grad_norm": 10.418560981750488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09571987390518188,
      "logits/rejected": 0.04926794022321701,
      "logps/chosen": -658.2051391601562,
      "logps/rejected": -314.3948974609375,
      "loss": 0.5908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14867231249809265,
      "rewards/margins": 0.9805942177772522,
      "rewards/rejected": -1.129266619682312,
      "step": 3707
    },
    {
      "epoch": 0.4852212317003353,
      "grad_norm": 7.605356216430664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12732268869876862,
      "logits/rejected": 0.08748652786016464,
      "logps/chosen": -481.3026428222656,
      "logps/rejected": -477.77044677734375,
      "loss": 0.6015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0946466326713562,
      "rewards/margins": 0.7024204730987549,
      "rewards/rejected": -0.6077738404273987,
      "step": 3708
    },
    {
      "epoch": 0.4853520896376871,
      "grad_norm": 5.639760971069336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10125014930963516,
      "logits/rejected": 0.08611342310905457,
      "logps/chosen": -254.39901733398438,
      "logps/rejected": -292.4628601074219,
      "loss": 0.5166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.50811368227005,
      "rewards/margins": 0.7085078954696655,
      "rewards/rejected": -0.20039424300193787,
      "step": 3709
    },
    {
      "epoch": 0.48548294757503885,
      "grad_norm": 4.314603805541992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15848396718502045,
      "logits/rejected": -0.11977848410606384,
      "logps/chosen": -376.6365051269531,
      "logps/rejected": -424.9565124511719,
      "loss": 0.2344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28286245465278625,
      "rewards/margins": 1.6188459396362305,
      "rewards/rejected": -1.9017083644866943,
      "step": 3710
    },
    {
      "epoch": 0.4856138055123906,
      "grad_norm": 5.254614353179932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1278303563594818,
      "logits/rejected": 0.21713700890541077,
      "logps/chosen": -319.7110900878906,
      "logps/rejected": -383.5426025390625,
      "loss": 0.4,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07746542990207672,
      "rewards/margins": 0.8978800177574158,
      "rewards/rejected": -0.8204146027565002,
      "step": 3711
    },
    {
      "epoch": 0.4857446634497424,
      "grad_norm": 7.519172668457031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06417922675609589,
      "logits/rejected": -0.019720718264579773,
      "logps/chosen": -393.7620849609375,
      "logps/rejected": -380.7986755371094,
      "loss": 0.5653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3282471001148224,
      "rewards/margins": 0.7329691052436829,
      "rewards/rejected": -0.40472206473350525,
      "step": 3712
    },
    {
      "epoch": 0.4858755213870941,
      "grad_norm": 5.603275775909424,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39155444502830505,
      "logits/rejected": 0.23958531022071838,
      "logps/chosen": -479.2122497558594,
      "logps/rejected": -481.3197021484375,
      "loss": 0.2941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.245599165558815,
      "rewards/margins": 1.4438222646713257,
      "rewards/rejected": -1.1982231140136719,
      "step": 3713
    },
    {
      "epoch": 0.4860063793244459,
      "grad_norm": 8.607426643371582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1769282966852188,
      "logits/rejected": -0.06321074068546295,
      "logps/chosen": -422.40789794921875,
      "logps/rejected": -489.9897766113281,
      "loss": 0.5751,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5453767776489258,
      "rewards/margins": 0.8533058762550354,
      "rewards/rejected": -1.3986828327178955,
      "step": 3714
    },
    {
      "epoch": 0.48613723726179764,
      "grad_norm": 5.442699432373047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10082948207855225,
      "logits/rejected": -0.10084545612335205,
      "logps/chosen": -437.0863037109375,
      "logps/rejected": -500.51031494140625,
      "loss": 0.3765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2044578492641449,
      "rewards/margins": 1.2648369073867798,
      "rewards/rejected": -1.469294786453247,
      "step": 3715
    },
    {
      "epoch": 0.48626809519914943,
      "grad_norm": 6.559051036834717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33394426107406616,
      "logits/rejected": 0.4465967118740082,
      "logps/chosen": -352.24566650390625,
      "logps/rejected": -359.5738220214844,
      "loss": 0.6475,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3342086970806122,
      "rewards/margins": 0.6272420883178711,
      "rewards/rejected": -0.9614507555961609,
      "step": 3716
    },
    {
      "epoch": 0.48639895313650117,
      "grad_norm": 5.8467888832092285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004889329895377159,
      "logits/rejected": 0.02866387739777565,
      "logps/chosen": -680.6492919921875,
      "logps/rejected": -566.1930541992188,
      "loss": 0.4023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3631061911582947,
      "rewards/margins": 1.1409307718276978,
      "rewards/rejected": -0.7778245210647583,
      "step": 3717
    },
    {
      "epoch": 0.48652981107385296,
      "grad_norm": 5.861278533935547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11233119666576385,
      "logits/rejected": 0.22024916112422943,
      "logps/chosen": -424.19000244140625,
      "logps/rejected": -427.3132019042969,
      "loss": 0.4289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09056340903043747,
      "rewards/margins": 0.8941034078598022,
      "rewards/rejected": -0.803540050983429,
      "step": 3718
    },
    {
      "epoch": 0.4866606690112047,
      "grad_norm": 5.690954685211182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10381441563367844,
      "logits/rejected": -0.05479777231812477,
      "logps/chosen": -327.0326843261719,
      "logps/rejected": -330.2112731933594,
      "loss": 0.5784,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5215189456939697,
      "rewards/margins": 0.42581939697265625,
      "rewards/rejected": -0.947338342666626,
      "step": 3719
    },
    {
      "epoch": 0.4867915269485565,
      "grad_norm": 4.520910263061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09261177480220795,
      "logits/rejected": -0.008254144340753555,
      "logps/chosen": -387.3780517578125,
      "logps/rejected": -343.30645751953125,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.03469979763031,
      "rewards/margins": 1.1704775094985962,
      "rewards/rejected": -2.2051773071289062,
      "step": 3720
    },
    {
      "epoch": 0.4869223848859082,
      "grad_norm": 6.754196643829346,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1689886450767517,
      "logits/rejected": -0.08007729053497314,
      "logps/chosen": -426.72955322265625,
      "logps/rejected": -469.7421875,
      "loss": 0.3466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03480472415685654,
      "rewards/margins": 1.6107009649276733,
      "rewards/rejected": -1.6455057859420776,
      "step": 3721
    },
    {
      "epoch": 0.48705324282326,
      "grad_norm": 4.775383472442627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1345711648464203,
      "logits/rejected": -0.001673063263297081,
      "logps/chosen": -409.33245849609375,
      "logps/rejected": -481.9978942871094,
      "loss": 0.3291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30266574025154114,
      "rewards/margins": 1.2936712503433228,
      "rewards/rejected": -1.5963369607925415,
      "step": 3722
    },
    {
      "epoch": 0.48718410076061175,
      "grad_norm": 10.73987102508545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2551807463169098,
      "logits/rejected": -0.2147253006696701,
      "logps/chosen": -429.5691223144531,
      "logps/rejected": -454.7202453613281,
      "loss": 0.6347,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14894889295101166,
      "rewards/margins": 0.5936779379844666,
      "rewards/rejected": -0.7426267862319946,
      "step": 3723
    },
    {
      "epoch": 0.48731495869796354,
      "grad_norm": 8.34632396697998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.419647753238678,
      "logits/rejected": 0.3046043813228607,
      "logps/chosen": -434.29071044921875,
      "logps/rejected": -393.34759521484375,
      "loss": 0.5736,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5163589715957642,
      "rewards/margins": 0.9018680453300476,
      "rewards/rejected": -1.4182270765304565,
      "step": 3724
    },
    {
      "epoch": 0.4874458166353153,
      "grad_norm": 6.4585418701171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11393745243549347,
      "logits/rejected": -0.02087276428937912,
      "logps/chosen": -435.7237548828125,
      "logps/rejected": -513.1241455078125,
      "loss": 0.4191,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35239943861961365,
      "rewards/margins": 0.9039208292961121,
      "rewards/rejected": -1.2563203573226929,
      "step": 3725
    },
    {
      "epoch": 0.48757667457266707,
      "grad_norm": 9.879432678222656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22664138674736023,
      "logits/rejected": 0.06513766944408417,
      "logps/chosen": -557.161865234375,
      "logps/rejected": -437.4011535644531,
      "loss": 0.6716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5252906680107117,
      "rewards/margins": 0.698532223701477,
      "rewards/rejected": -1.2238229513168335,
      "step": 3726
    },
    {
      "epoch": 0.4877075325100188,
      "grad_norm": 9.769808769226074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0150369331240654,
      "logits/rejected": -0.2223816066980362,
      "logps/chosen": -358.7096862792969,
      "logps/rejected": -383.7288513183594,
      "loss": 0.6075,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29791778326034546,
      "rewards/margins": 0.9739372730255127,
      "rewards/rejected": -1.2718548774719238,
      "step": 3727
    },
    {
      "epoch": 0.4878383904473706,
      "grad_norm": 5.489646911621094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1381222903728485,
      "logits/rejected": 0.05332237109541893,
      "logps/chosen": -342.71502685546875,
      "logps/rejected": -347.9493408203125,
      "loss": 0.4442,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13444212079048157,
      "rewards/margins": 1.1732410192489624,
      "rewards/rejected": -1.0387988090515137,
      "step": 3728
    },
    {
      "epoch": 0.48796924838472233,
      "grad_norm": 8.586228370666504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1125309020280838,
      "logits/rejected": 0.12940186262130737,
      "logps/chosen": -516.9517822265625,
      "logps/rejected": -595.092041015625,
      "loss": 0.4898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5636774301528931,
      "rewards/margins": 1.1045302152633667,
      "rewards/rejected": -1.6682074069976807,
      "step": 3729
    },
    {
      "epoch": 0.4881001063220741,
      "grad_norm": 4.573897838592529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3723854422569275,
      "logits/rejected": 0.28902021050453186,
      "logps/chosen": -414.904296875,
      "logps/rejected": -453.79119873046875,
      "loss": 0.3048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4837176203727722,
      "rewards/margins": 1.6642279624938965,
      "rewards/rejected": -1.1805102825164795,
      "step": 3730
    },
    {
      "epoch": 0.48823096425942586,
      "grad_norm": 8.214184761047363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10044310986995697,
      "logits/rejected": -0.06303808093070984,
      "logps/chosen": -424.2681579589844,
      "logps/rejected": -362.7508850097656,
      "loss": 0.5504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3434923887252808,
      "rewards/margins": 0.6260374784469604,
      "rewards/rejected": -1.9695297479629517,
      "step": 3731
    },
    {
      "epoch": 0.48836182219677765,
      "grad_norm": 7.820906639099121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09618838876485825,
      "logits/rejected": 0.22460860013961792,
      "logps/chosen": -404.78814697265625,
      "logps/rejected": -401.42694091796875,
      "loss": 0.5033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2663242816925049,
      "rewards/margins": 1.0183207988739014,
      "rewards/rejected": -1.2846451997756958,
      "step": 3732
    },
    {
      "epoch": 0.4884926801341294,
      "grad_norm": 8.967310905456543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21202367544174194,
      "logits/rejected": 0.24897536635398865,
      "logps/chosen": -528.3231201171875,
      "logps/rejected": -535.1688232421875,
      "loss": 0.524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.58306884765625,
      "rewards/margins": 0.6777297854423523,
      "rewards/rejected": -1.260798692703247,
      "step": 3733
    },
    {
      "epoch": 0.4886235380714811,
      "grad_norm": 5.158801555633545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.059042155742645264,
      "logits/rejected": 0.1812453269958496,
      "logps/chosen": -450.1680603027344,
      "logps/rejected": -453.298583984375,
      "loss": 0.3191,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18220768868923187,
      "rewards/margins": 1.3053865432739258,
      "rewards/rejected": -1.4875942468643188,
      "step": 3734
    },
    {
      "epoch": 0.4887543960088329,
      "grad_norm": 8.254264831542969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04921074956655502,
      "logits/rejected": -0.12331048399209976,
      "logps/chosen": -509.2204895019531,
      "logps/rejected": -493.31219482421875,
      "loss": 0.4773,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9685098528862,
      "rewards/margins": 0.7994723916053772,
      "rewards/rejected": -1.7679823637008667,
      "step": 3735
    },
    {
      "epoch": 0.48888525394618465,
      "grad_norm": 4.407003402709961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2562342882156372,
      "logits/rejected": 0.02253159135580063,
      "logps/chosen": -327.408447265625,
      "logps/rejected": -391.49652099609375,
      "loss": 0.2703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5255624651908875,
      "rewards/margins": 1.579567313194275,
      "rewards/rejected": -2.1051297187805176,
      "step": 3736
    },
    {
      "epoch": 0.48901611188353644,
      "grad_norm": 9.439688682556152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12633061408996582,
      "logits/rejected": -0.02297636866569519,
      "logps/chosen": -524.5464477539062,
      "logps/rejected": -512.810302734375,
      "loss": 0.8784,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5144201517105103,
      "rewards/margins": 0.08481580764055252,
      "rewards/rejected": -0.5992359519004822,
      "step": 3737
    },
    {
      "epoch": 0.4891469698208882,
      "grad_norm": 7.222402572631836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1367693990468979,
      "logits/rejected": 0.18336991965770721,
      "logps/chosen": -431.6455993652344,
      "logps/rejected": -625.650146484375,
      "loss": 0.3705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15522214770317078,
      "rewards/margins": 1.2216002941131592,
      "rewards/rejected": -1.0663782358169556,
      "step": 3738
    },
    {
      "epoch": 0.48927782775823997,
      "grad_norm": 7.09654426574707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020901687443256378,
      "logits/rejected": 0.02012103796005249,
      "logps/chosen": -434.9114074707031,
      "logps/rejected": -372.3068542480469,
      "loss": 0.5831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9950183629989624,
      "rewards/margins": 0.36930179595947266,
      "rewards/rejected": -1.364320158958435,
      "step": 3739
    },
    {
      "epoch": 0.4894086856955917,
      "grad_norm": 14.185914993286133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12434650957584381,
      "logits/rejected": -0.07438139617443085,
      "logps/chosen": -480.54168701171875,
      "logps/rejected": -357.09393310546875,
      "loss": 0.909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6389349699020386,
      "rewards/margins": 0.9507031440734863,
      "rewards/rejected": -2.5896382331848145,
      "step": 3740
    },
    {
      "epoch": 0.4895395436329435,
      "grad_norm": 11.979516983032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07063654810190201,
      "logits/rejected": -0.13366371393203735,
      "logps/chosen": -477.1627502441406,
      "logps/rejected": -584.8666381835938,
      "loss": 0.8729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1432468891143799,
      "rewards/margins": -0.014312490820884705,
      "rewards/rejected": -1.128934383392334,
      "step": 3741
    },
    {
      "epoch": 0.48967040157029523,
      "grad_norm": 8.694218635559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10298538208007812,
      "logits/rejected": 0.02647238038480282,
      "logps/chosen": -403.95062255859375,
      "logps/rejected": -418.1595458984375,
      "loss": 0.5483,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.023122787475585938,
      "rewards/margins": 0.9615801572799683,
      "rewards/rejected": -0.9847028255462646,
      "step": 3742
    },
    {
      "epoch": 0.489801259507647,
      "grad_norm": 11.894254684448242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22442936897277832,
      "logits/rejected": -0.054681941866874695,
      "logps/chosen": -493.2419738769531,
      "logps/rejected": -403.3547058105469,
      "loss": 0.8272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2080234289169312,
      "rewards/margins": 0.38580575585365295,
      "rewards/rejected": -1.5938292741775513,
      "step": 3743
    },
    {
      "epoch": 0.48993211744499876,
      "grad_norm": 5.757235050201416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10231596231460571,
      "logits/rejected": 0.1358451247215271,
      "logps/chosen": -330.6535339355469,
      "logps/rejected": -333.6271667480469,
      "loss": 0.2767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07181237637996674,
      "rewards/margins": 2.0778648853302,
      "rewards/rejected": -2.149677276611328,
      "step": 3744
    },
    {
      "epoch": 0.49006297538235055,
      "grad_norm": 10.810523986816406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07459470629692078,
      "logits/rejected": -0.04116993397474289,
      "logps/chosen": -353.57427978515625,
      "logps/rejected": -245.1622772216797,
      "loss": 0.6903,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2339674234390259,
      "rewards/margins": 0.22353535890579224,
      "rewards/rejected": -1.4575027227401733,
      "step": 3745
    },
    {
      "epoch": 0.4901938333197023,
      "grad_norm": 9.373786926269531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13015179336071014,
      "logits/rejected": 0.05967938154935837,
      "logps/chosen": -650.3194580078125,
      "logps/rejected": -511.57177734375,
      "loss": 0.5142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.334916114807129,
      "rewards/margins": 1.2966493368148804,
      "rewards/rejected": -2.6315653324127197,
      "step": 3746
    },
    {
      "epoch": 0.4903246912570541,
      "grad_norm": 11.226283073425293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06628328561782837,
      "logits/rejected": -0.012752904556691647,
      "logps/chosen": -541.9315185546875,
      "logps/rejected": -471.8329772949219,
      "loss": 0.7053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4875214397907257,
      "rewards/margins": 0.8595621585845947,
      "rewards/rejected": -1.347083568572998,
      "step": 3747
    },
    {
      "epoch": 0.4904555491944058,
      "grad_norm": 9.966740608215332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016191352158784866,
      "logits/rejected": -0.1154700368642807,
      "logps/chosen": -530.36669921875,
      "logps/rejected": -553.7423706054688,
      "loss": 0.4906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1799694299697876,
      "rewards/margins": 1.8671849966049194,
      "rewards/rejected": -3.047154426574707,
      "step": 3748
    },
    {
      "epoch": 0.4905864071317576,
      "grad_norm": 7.76417875289917,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002878338098526001,
      "logits/rejected": 0.08296520262956619,
      "logps/chosen": -367.64013671875,
      "logps/rejected": -342.9127197265625,
      "loss": 0.6117,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5925121307373047,
      "rewards/margins": 0.3861830234527588,
      "rewards/rejected": -0.9786952137947083,
      "step": 3749
    },
    {
      "epoch": 0.49071726506910934,
      "grad_norm": 9.561139106750488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16326597332954407,
      "logits/rejected": 0.1581888198852539,
      "logps/chosen": -459.3495178222656,
      "logps/rejected": -457.96441650390625,
      "loss": 0.6123,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9092115163803101,
      "rewards/margins": 0.40508919954299927,
      "rewards/rejected": -1.314300775527954,
      "step": 3750
    },
    {
      "epoch": 0.49084812300646113,
      "grad_norm": 8.042718887329102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03244173899292946,
      "logits/rejected": -0.0356699675321579,
      "logps/chosen": -455.0244140625,
      "logps/rejected": -361.38812255859375,
      "loss": 0.636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42628592252731323,
      "rewards/margins": 0.5764074325561523,
      "rewards/rejected": -1.0026934146881104,
      "step": 3751
    },
    {
      "epoch": 0.49097898094381287,
      "grad_norm": 6.92217493057251,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1209343671798706,
      "logits/rejected": -0.15101371705532074,
      "logps/chosen": -487.63323974609375,
      "logps/rejected": -491.16632080078125,
      "loss": 0.4214,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18874774873256683,
      "rewards/margins": 0.8005876541137695,
      "rewards/rejected": -0.9893354773521423,
      "step": 3752
    },
    {
      "epoch": 0.49110983888116466,
      "grad_norm": 6.332249164581299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008845759555697441,
      "logits/rejected": -0.15409645438194275,
      "logps/chosen": -448.2854309082031,
      "logps/rejected": -456.21630859375,
      "loss": 0.4326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1779390275478363,
      "rewards/margins": 1.4677361249923706,
      "rewards/rejected": -1.2897969484329224,
      "step": 3753
    },
    {
      "epoch": 0.4912406968185164,
      "grad_norm": 7.661623477935791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1404450684785843,
      "logits/rejected": -0.4172985851764679,
      "logps/chosen": -587.654052734375,
      "logps/rejected": -497.88043212890625,
      "loss": 0.5183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33186012506484985,
      "rewards/margins": 0.8177976608276367,
      "rewards/rejected": -1.1496578454971313,
      "step": 3754
    },
    {
      "epoch": 0.49137155475586813,
      "grad_norm": 7.0075483322143555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05291737616062164,
      "logits/rejected": -0.03441023826599121,
      "logps/chosen": -563.0028076171875,
      "logps/rejected": -534.4817504882812,
      "loss": 0.4657,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6007326245307922,
      "rewards/margins": 1.2127635478973389,
      "rewards/rejected": -0.6120310425758362,
      "step": 3755
    },
    {
      "epoch": 0.4915024126932199,
      "grad_norm": 8.298205375671387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09339255094528198,
      "logits/rejected": -0.08928390592336655,
      "logps/chosen": -491.0881042480469,
      "logps/rejected": -546.8487548828125,
      "loss": 0.7343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5750848650932312,
      "rewards/margins": 0.8702611923217773,
      "rewards/rejected": -1.4453461170196533,
      "step": 3756
    },
    {
      "epoch": 0.49163327063057166,
      "grad_norm": 9.05533504486084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07447021454572678,
      "logits/rejected": 0.13136614859104156,
      "logps/chosen": -452.8123474121094,
      "logps/rejected": -525.46923828125,
      "loss": 0.4413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6393534541130066,
      "rewards/margins": 1.335620641708374,
      "rewards/rejected": -1.9749741554260254,
      "step": 3757
    },
    {
      "epoch": 0.49176412856792345,
      "grad_norm": 8.8538179397583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017952002584934235,
      "logits/rejected": 0.08425118029117584,
      "logps/chosen": -474.1717224121094,
      "logps/rejected": -645.0481567382812,
      "loss": 0.5854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6507755517959595,
      "rewards/margins": 0.5934613943099976,
      "rewards/rejected": -1.2442368268966675,
      "step": 3758
    },
    {
      "epoch": 0.4918949865052752,
      "grad_norm": 7.099492073059082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15137146413326263,
      "logits/rejected": 0.00028499215841293335,
      "logps/chosen": -351.5997619628906,
      "logps/rejected": -528.6021728515625,
      "loss": 0.4395,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5149383544921875,
      "rewards/margins": 1.4192670583724976,
      "rewards/rejected": -1.9342052936553955,
      "step": 3759
    },
    {
      "epoch": 0.492025844442627,
      "grad_norm": 7.35532283782959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04169600456953049,
      "logits/rejected": 0.014104887843132019,
      "logps/chosen": -519.5302734375,
      "logps/rejected": -444.5685729980469,
      "loss": 0.442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8123838901519775,
      "rewards/margins": 0.9026145935058594,
      "rewards/rejected": -1.714998722076416,
      "step": 3760
    },
    {
      "epoch": 0.4921567023799787,
      "grad_norm": 6.22053337097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11553505808115005,
      "logits/rejected": 0.12024630606174469,
      "logps/chosen": -651.8915405273438,
      "logps/rejected": -527.7127685546875,
      "loss": 0.4107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.051692962646484375,
      "rewards/margins": 0.892728865146637,
      "rewards/rejected": -0.9444217681884766,
      "step": 3761
    },
    {
      "epoch": 0.4922875603173305,
      "grad_norm": 10.112813949584961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06520725041627884,
      "logits/rejected": -0.19967246055603027,
      "logps/chosen": -370.1728820800781,
      "logps/rejected": -461.4043273925781,
      "loss": 0.7213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4050411581993103,
      "rewards/margins": 0.7632925510406494,
      "rewards/rejected": -1.1683337688446045,
      "step": 3762
    },
    {
      "epoch": 0.49241841825468224,
      "grad_norm": 8.373884201049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22808904945850372,
      "logits/rejected": -0.09681222587823868,
      "logps/chosen": -607.1810302734375,
      "logps/rejected": -433.04608154296875,
      "loss": 0.4994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5383521914482117,
      "rewards/margins": 1.1457103490829468,
      "rewards/rejected": -1.6840624809265137,
      "step": 3763
    },
    {
      "epoch": 0.49254927619203404,
      "grad_norm": 5.910016059875488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08175379037857056,
      "logits/rejected": -0.0776996836066246,
      "logps/chosen": -534.357666015625,
      "logps/rejected": -623.296142578125,
      "loss": 0.5836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.650641679763794,
      "rewards/margins": 0.7758630514144897,
      "rewards/rejected": -1.4265047311782837,
      "step": 3764
    },
    {
      "epoch": 0.49268013412938577,
      "grad_norm": 7.452741622924805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007456477265805006,
      "logits/rejected": -0.19571124017238617,
      "logps/chosen": -495.73907470703125,
      "logps/rejected": -482.0738830566406,
      "loss": 0.5561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26979124546051025,
      "rewards/margins": 0.9965166449546814,
      "rewards/rejected": -1.2663077116012573,
      "step": 3765
    },
    {
      "epoch": 0.49281099206673756,
      "grad_norm": 9.028666496276855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19928118586540222,
      "logits/rejected": -0.28837692737579346,
      "logps/chosen": -344.922119140625,
      "logps/rejected": -316.85491943359375,
      "loss": 0.7214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3533574938774109,
      "rewards/margins": 0.17198505997657776,
      "rewards/rejected": -0.525342583656311,
      "step": 3766
    },
    {
      "epoch": 0.4929418500040893,
      "grad_norm": 4.068929195404053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.033161479979753494,
      "logits/rejected": -0.056778933852910995,
      "logps/chosen": -607.27685546875,
      "logps/rejected": -506.5963134765625,
      "loss": 0.2316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36563703417778015,
      "rewards/margins": 1.8279343843460083,
      "rewards/rejected": -1.4622973203659058,
      "step": 3767
    },
    {
      "epoch": 0.4930727079414411,
      "grad_norm": 9.288934707641602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2585597336292267,
      "logits/rejected": -0.05361529439687729,
      "logps/chosen": -639.1842651367188,
      "logps/rejected": -376.9634094238281,
      "loss": 0.7173,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6435392498970032,
      "rewards/margins": 0.5288248658180237,
      "rewards/rejected": -1.1723641157150269,
      "step": 3768
    },
    {
      "epoch": 0.4932035658787928,
      "grad_norm": 9.864190101623535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08471789956092834,
      "logits/rejected": -0.13974271714687347,
      "logps/chosen": -683.9525146484375,
      "logps/rejected": -613.9024047851562,
      "loss": 0.4029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.463889479637146,
      "rewards/margins": 1.025754690170288,
      "rewards/rejected": -2.4896440505981445,
      "step": 3769
    },
    {
      "epoch": 0.4933344238161446,
      "grad_norm": 8.287707328796387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006799928843975067,
      "logits/rejected": -0.005464054644107819,
      "logps/chosen": -359.0460205078125,
      "logps/rejected": -431.2451171875,
      "loss": 0.5747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7389005422592163,
      "rewards/margins": 0.4104083776473999,
      "rewards/rejected": -1.1493090391159058,
      "step": 3770
    },
    {
      "epoch": 0.49346528175349635,
      "grad_norm": 12.162403106689453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04386886954307556,
      "logits/rejected": -0.06229235231876373,
      "logps/chosen": -517.90673828125,
      "logps/rejected": -466.964599609375,
      "loss": 1.0097,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3661049008369446,
      "rewards/margins": 0.5665472745895386,
      "rewards/rejected": -0.9326521158218384,
      "step": 3771
    },
    {
      "epoch": 0.49359613969084815,
      "grad_norm": 4.116608142852783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17961663007736206,
      "logits/rejected": 0.39739713072776794,
      "logps/chosen": -469.7337646484375,
      "logps/rejected": -558.1167602539062,
      "loss": 0.1895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.019820213317871,
      "rewards/margins": 1.9508836269378662,
      "rewards/rejected": -0.9310634732246399,
      "step": 3772
    },
    {
      "epoch": 0.4937269976281999,
      "grad_norm": 6.128071308135986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2435685694217682,
      "logits/rejected": -0.10462142527103424,
      "logps/chosen": -487.49176025390625,
      "logps/rejected": -419.1111145019531,
      "loss": 0.498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8980789184570312,
      "rewards/margins": 0.8965318202972412,
      "rewards/rejected": -1.794610619544983,
      "step": 3773
    },
    {
      "epoch": 0.4938578555655517,
      "grad_norm": 8.456637382507324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37245139479637146,
      "logits/rejected": 0.41193094849586487,
      "logps/chosen": -456.272216796875,
      "logps/rejected": -465.2835388183594,
      "loss": 0.5859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.47573357820510864,
      "rewards/margins": 1.027018427848816,
      "rewards/rejected": -0.5512847900390625,
      "step": 3774
    },
    {
      "epoch": 0.4939887135029034,
      "grad_norm": 9.406899452209473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09901048243045807,
      "logits/rejected": 0.1633893847465515,
      "logps/chosen": -423.4835205078125,
      "logps/rejected": -347.1319580078125,
      "loss": 0.8174,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.46850359439849854,
      "rewards/margins": -0.10506973415613174,
      "rewards/rejected": -0.363433837890625,
      "step": 3775
    },
    {
      "epoch": 0.49411957144025515,
      "grad_norm": 9.576769828796387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19603681564331055,
      "logits/rejected": -0.14931559562683105,
      "logps/chosen": -414.0809326171875,
      "logps/rejected": -400.9293212890625,
      "loss": 0.7458,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8273569941520691,
      "rewards/margins": 0.030912384390830994,
      "rewards/rejected": -0.8582693338394165,
      "step": 3776
    },
    {
      "epoch": 0.49425042937760694,
      "grad_norm": 8.27717399597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024072393774986267,
      "logits/rejected": 0.1016637533903122,
      "logps/chosen": -378.4439392089844,
      "logps/rejected": -423.6842041015625,
      "loss": 0.6488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6548442244529724,
      "rewards/margins": 0.30535051226615906,
      "rewards/rejected": -0.9601947665214539,
      "step": 3777
    },
    {
      "epoch": 0.4943812873149587,
      "grad_norm": 6.376625061035156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18524877727031708,
      "logits/rejected": -0.05541103333234787,
      "logps/chosen": -517.6802978515625,
      "logps/rejected": -324.1512756347656,
      "loss": 0.4514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1405343860387802,
      "rewards/margins": 1.3491917848587036,
      "rewards/rejected": -1.2086573839187622,
      "step": 3778
    },
    {
      "epoch": 0.49451214525231046,
      "grad_norm": 10.601038932800293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23657765984535217,
      "logits/rejected": 0.20057862997055054,
      "logps/chosen": -422.358154296875,
      "logps/rejected": -386.63323974609375,
      "loss": 0.9015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003322601318359375,
      "rewards/margins": 0.09890156984329224,
      "rewards/rejected": -0.09557896852493286,
      "step": 3779
    },
    {
      "epoch": 0.4946430031896622,
      "grad_norm": 6.996967792510986,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15241335332393646,
      "logits/rejected": 0.07560845464468002,
      "logps/chosen": -400.012451171875,
      "logps/rejected": -625.6388549804688,
      "loss": 0.4698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23337039351463318,
      "rewards/margins": 0.7672656774520874,
      "rewards/rejected": -1.000636100769043,
      "step": 3780
    },
    {
      "epoch": 0.494773861127014,
      "grad_norm": 6.983987331390381,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1965693235397339,
      "logits/rejected": 0.1466923952102661,
      "logps/chosen": -525.1112060546875,
      "logps/rejected": -526.7763061523438,
      "loss": 0.5474,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18852081894874573,
      "rewards/margins": 1.0368549823760986,
      "rewards/rejected": -0.8483341336250305,
      "step": 3781
    },
    {
      "epoch": 0.49490471906436573,
      "grad_norm": 6.75112247467041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007327184081077576,
      "logits/rejected": -0.048033326864242554,
      "logps/chosen": -511.1842041015625,
      "logps/rejected": -548.4198608398438,
      "loss": 0.4553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11270084232091904,
      "rewards/margins": 0.9968622326850891,
      "rewards/rejected": -0.8841613531112671,
      "step": 3782
    },
    {
      "epoch": 0.4950355770017175,
      "grad_norm": 5.865820407867432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29864078760147095,
      "logits/rejected": 0.26165342330932617,
      "logps/chosen": -453.70867919921875,
      "logps/rejected": -479.1880187988281,
      "loss": 0.3984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3037792146205902,
      "rewards/margins": 1.680540680885315,
      "rewards/rejected": -1.3767613172531128,
      "step": 3783
    },
    {
      "epoch": 0.49516643493906926,
      "grad_norm": 6.918380260467529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23680457472801208,
      "logits/rejected": 0.0413939505815506,
      "logps/chosen": -383.06494140625,
      "logps/rejected": -587.227294921875,
      "loss": 0.4181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.325016587972641,
      "rewards/margins": 1.528560757637024,
      "rewards/rejected": -1.2035441398620605,
      "step": 3784
    },
    {
      "epoch": 0.49529729287642105,
      "grad_norm": 5.293505668640137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021625947207212448,
      "logits/rejected": -0.10674732178449631,
      "logps/chosen": -398.80975341796875,
      "logps/rejected": -430.8426513671875,
      "loss": 0.3439,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.409598171710968,
      "rewards/margins": 1.2861440181732178,
      "rewards/rejected": -1.6957422494888306,
      "step": 3785
    },
    {
      "epoch": 0.4954281508137728,
      "grad_norm": 6.606747150421143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22746190428733826,
      "logits/rejected": 0.03947281092405319,
      "logps/chosen": -360.8698425292969,
      "logps/rejected": -360.0279846191406,
      "loss": 0.6015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6609197854995728,
      "rewards/margins": 0.456961989402771,
      "rewards/rejected": -1.1178817749023438,
      "step": 3786
    },
    {
      "epoch": 0.4955590087511246,
      "grad_norm": 8.340017318725586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12551149725914001,
      "logits/rejected": -0.24320007860660553,
      "logps/chosen": -424.4770202636719,
      "logps/rejected": -634.6591186523438,
      "loss": 0.559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4519760608673096,
      "rewards/margins": 0.5630437731742859,
      "rewards/rejected": -2.0150198936462402,
      "step": 3787
    },
    {
      "epoch": 0.4956898666884763,
      "grad_norm": 9.633926391601562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08965747803449631,
      "logits/rejected": 0.027169331908226013,
      "logps/chosen": -421.45648193359375,
      "logps/rejected": -313.8057556152344,
      "loss": 0.823,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.46139928698539734,
      "rewards/margins": 0.022571183741092682,
      "rewards/rejected": -0.4839704632759094,
      "step": 3788
    },
    {
      "epoch": 0.4958207246258281,
      "grad_norm": 8.597143173217773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29560235142707825,
      "logits/rejected": 0.39026254415512085,
      "logps/chosen": -451.2930603027344,
      "logps/rejected": -325.148193359375,
      "loss": 0.7945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48393553495407104,
      "rewards/margins": 0.6157242059707642,
      "rewards/rejected": -1.0996596813201904,
      "step": 3789
    },
    {
      "epoch": 0.49595158256317984,
      "grad_norm": 12.191651344299316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018189972266554832,
      "logits/rejected": -0.013138771057128906,
      "logps/chosen": -363.27825927734375,
      "logps/rejected": -340.7130126953125,
      "loss": 0.9811,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4440734386444092,
      "rewards/margins": 0.013010859489440918,
      "rewards/rejected": -1.45708429813385,
      "step": 3790
    },
    {
      "epoch": 0.49608244050053163,
      "grad_norm": 3.9737191200256348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24590373039245605,
      "logits/rejected": 0.09279654175043106,
      "logps/chosen": -346.75738525390625,
      "logps/rejected": -270.563720703125,
      "loss": 0.3252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3600713908672333,
      "rewards/margins": 1.4147508144378662,
      "rewards/rejected": -1.0546795129776,
      "step": 3791
    },
    {
      "epoch": 0.49621329843788337,
      "grad_norm": 9.053263664245605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028294576331973076,
      "logits/rejected": 0.12296546250581741,
      "logps/chosen": -498.9381103515625,
      "logps/rejected": -508.52557373046875,
      "loss": 0.5738,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6465290188789368,
      "rewards/margins": 0.6103274822235107,
      "rewards/rejected": -1.2568565607070923,
      "step": 3792
    },
    {
      "epoch": 0.49634415637523516,
      "grad_norm": 5.511861801147461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035236455500125885,
      "logits/rejected": -0.16984470188617706,
      "logps/chosen": -409.3016357421875,
      "logps/rejected": -380.57275390625,
      "loss": 0.3811,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.731269896030426,
      "rewards/margins": 1.2844104766845703,
      "rewards/rejected": -2.0156803131103516,
      "step": 3793
    },
    {
      "epoch": 0.4964750143125869,
      "grad_norm": 8.76610279083252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34055089950561523,
      "logits/rejected": 0.1347585916519165,
      "logps/chosen": -374.59051513671875,
      "logps/rejected": -440.9796142578125,
      "loss": 0.641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4575977325439453,
      "rewards/margins": 0.9742552042007446,
      "rewards/rejected": -0.5166574716567993,
      "step": 3794
    },
    {
      "epoch": 0.4966058722499387,
      "grad_norm": 5.709857940673828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08956418186426163,
      "logits/rejected": -0.06749150902032852,
      "logps/chosen": -675.3306274414062,
      "logps/rejected": -452.76666259765625,
      "loss": 0.4587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3211216330528259,
      "rewards/margins": 1.3149573802947998,
      "rewards/rejected": -0.9938357472419739,
      "step": 3795
    },
    {
      "epoch": 0.4967367301872904,
      "grad_norm": 8.27850341796875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20246589183807373,
      "logits/rejected": 0.14602845907211304,
      "logps/chosen": -404.7080078125,
      "logps/rejected": -462.7019348144531,
      "loss": 0.4066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03075195848941803,
      "rewards/margins": 1.0823583602905273,
      "rewards/rejected": -1.0516064167022705,
      "step": 3796
    },
    {
      "epoch": 0.4968675881246422,
      "grad_norm": 6.776907920837402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1668197065591812,
      "logits/rejected": 0.05054129660129547,
      "logps/chosen": -412.0784606933594,
      "logps/rejected": -408.89459228515625,
      "loss": 0.4682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2469312697649002,
      "rewards/margins": 0.9376550912857056,
      "rewards/rejected": -1.1845864057540894,
      "step": 3797
    },
    {
      "epoch": 0.49699844606199395,
      "grad_norm": 8.054301261901855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035454943776130676,
      "logits/rejected": 0.08819027245044708,
      "logps/chosen": -563.2258911132812,
      "logps/rejected": -298.5093994140625,
      "loss": 0.562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5607119798660278,
      "rewards/margins": 0.7250217199325562,
      "rewards/rejected": -1.2857338190078735,
      "step": 3798
    },
    {
      "epoch": 0.4971293039993457,
      "grad_norm": 11.222384452819824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07649591565132141,
      "logits/rejected": -0.03283926099538803,
      "logps/chosen": -384.0520324707031,
      "logps/rejected": -405.63238525390625,
      "loss": 0.8972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6888460516929626,
      "rewards/margins": -0.016704969108104706,
      "rewards/rejected": -0.6721410155296326,
      "step": 3799
    },
    {
      "epoch": 0.4972601619366975,
      "grad_norm": 7.010350227355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01790463551878929,
      "logits/rejected": -0.19686926901340485,
      "logps/chosen": -320.3078308105469,
      "logps/rejected": -344.74627685546875,
      "loss": 0.4353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4511101245880127,
      "rewards/margins": 1.0233092308044434,
      "rewards/rejected": -0.5721991062164307,
      "step": 3800
    },
    {
      "epoch": 0.4973910198740492,
      "grad_norm": 6.101264476776123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034681301563978195,
      "logits/rejected": -0.05997384339570999,
      "logps/chosen": -472.00140380859375,
      "logps/rejected": -453.1467590332031,
      "loss": 0.5739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4081409275531769,
      "rewards/margins": 0.9873586893081665,
      "rewards/rejected": -0.5792176723480225,
      "step": 3801
    },
    {
      "epoch": 0.497521877811401,
      "grad_norm": 8.863615989685059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09251140058040619,
      "logits/rejected": -0.12153896689414978,
      "logps/chosen": -420.90728759765625,
      "logps/rejected": -363.36407470703125,
      "loss": 0.6293,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21505983173847198,
      "rewards/margins": 0.5403604507446289,
      "rewards/rejected": -0.7554203867912292,
      "step": 3802
    },
    {
      "epoch": 0.49765273574875274,
      "grad_norm": 7.183601379394531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01924218237400055,
      "logits/rejected": -0.12437036633491516,
      "logps/chosen": -354.6689147949219,
      "logps/rejected": -498.6216735839844,
      "loss": 0.5351,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15728895366191864,
      "rewards/margins": 0.8618690371513367,
      "rewards/rejected": -0.7045801877975464,
      "step": 3803
    },
    {
      "epoch": 0.49778359368610453,
      "grad_norm": 10.9625883102417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3034404516220093,
      "logits/rejected": 0.3208933174610138,
      "logps/chosen": -366.5432434082031,
      "logps/rejected": -468.2369384765625,
      "loss": 0.8574,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5779777765274048,
      "rewards/margins": -0.1124824658036232,
      "rewards/rejected": -0.4654952883720398,
      "step": 3804
    },
    {
      "epoch": 0.49791445162345627,
      "grad_norm": 7.525827407836914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12225500494241714,
      "logits/rejected": -0.011193308047950268,
      "logps/chosen": -391.39813232421875,
      "logps/rejected": -332.0198059082031,
      "loss": 0.5251,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10488185286521912,
      "rewards/margins": 0.7643032073974609,
      "rewards/rejected": -0.6594213247299194,
      "step": 3805
    },
    {
      "epoch": 0.49804530956080806,
      "grad_norm": 8.853080749511719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3686617612838745,
      "logits/rejected": -0.1603853851556778,
      "logps/chosen": -505.55780029296875,
      "logps/rejected": -666.0397338867188,
      "loss": 0.6154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6125458478927612,
      "rewards/margins": 0.4790634512901306,
      "rewards/rejected": -1.0916091203689575,
      "step": 3806
    },
    {
      "epoch": 0.4981761674981598,
      "grad_norm": 10.172920227050781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007081206887960434,
      "logits/rejected": 0.025120627135038376,
      "logps/chosen": -651.317626953125,
      "logps/rejected": -509.9859313964844,
      "loss": 0.7093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5280081033706665,
      "rewards/margins": 0.4063299298286438,
      "rewards/rejected": -0.9343380331993103,
      "step": 3807
    },
    {
      "epoch": 0.4983070254355116,
      "grad_norm": 9.51124095916748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.176972895860672,
      "logits/rejected": -0.07599843293428421,
      "logps/chosen": -680.2744140625,
      "logps/rejected": -555.36083984375,
      "loss": 0.5611,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.22629736363887787,
      "rewards/margins": 0.4289722144603729,
      "rewards/rejected": -0.20267489552497864,
      "step": 3808
    },
    {
      "epoch": 0.4984378833728633,
      "grad_norm": 6.492710590362549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12520334124565125,
      "logits/rejected": -0.0306344423443079,
      "logps/chosen": -367.4190979003906,
      "logps/rejected": -335.3848876953125,
      "loss": 0.5562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.765571653842926,
      "rewards/margins": 0.46246016025543213,
      "rewards/rejected": -1.2280317544937134,
      "step": 3809
    },
    {
      "epoch": 0.4985687413102151,
      "grad_norm": 6.260344982147217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3070032298564911,
      "logits/rejected": -0.31126731634140015,
      "logps/chosen": -331.04486083984375,
      "logps/rejected": -327.2454833984375,
      "loss": 0.5189,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4550844132900238,
      "rewards/margins": 0.7583457827568054,
      "rewards/rejected": -1.2134301662445068,
      "step": 3810
    },
    {
      "epoch": 0.49869959924756685,
      "grad_norm": 5.482639312744141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0011462382972240448,
      "logits/rejected": 0.09609214216470718,
      "logps/chosen": -446.4207763671875,
      "logps/rejected": -459.1519775390625,
      "loss": 0.363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4270744323730469,
      "rewards/margins": 1.2271785736083984,
      "rewards/rejected": -1.6542530059814453,
      "step": 3811
    },
    {
      "epoch": 0.49883045718491864,
      "grad_norm": 9.162189483642578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12160596251487732,
      "logits/rejected": 0.030843742191791534,
      "logps/chosen": -398.7639465332031,
      "logps/rejected": -537.7420654296875,
      "loss": 0.6713,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.034095361828804016,
      "rewards/margins": 0.24847260117530823,
      "rewards/rejected": -0.21437722444534302,
      "step": 3812
    },
    {
      "epoch": 0.4989613151222704,
      "grad_norm": 11.725310325622559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06693214923143387,
      "logits/rejected": 0.01008594036102295,
      "logps/chosen": -571.0919799804688,
      "logps/rejected": -359.2529602050781,
      "loss": 0.7271,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5948163270950317,
      "rewards/margins": 0.2232496291399002,
      "rewards/rejected": -0.8180660009384155,
      "step": 3813
    },
    {
      "epoch": 0.49909217305962217,
      "grad_norm": 10.036324501037598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09434985369443893,
      "logits/rejected": 0.11373992264270782,
      "logps/chosen": -368.89208984375,
      "logps/rejected": -377.4963684082031,
      "loss": 0.698,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18433648347854614,
      "rewards/margins": 0.6960718631744385,
      "rewards/rejected": -0.8804084062576294,
      "step": 3814
    },
    {
      "epoch": 0.4992230309969739,
      "grad_norm": 7.120131492614746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2518259882926941,
      "logits/rejected": 0.2885381877422333,
      "logps/chosen": -389.3193054199219,
      "logps/rejected": -417.3474426269531,
      "loss": 0.6788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1605842411518097,
      "rewards/margins": 0.14706557989120483,
      "rewards/rejected": -0.30764979124069214,
      "step": 3815
    },
    {
      "epoch": 0.4993538889343257,
      "grad_norm": 9.644598960876465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09119786322116852,
      "logits/rejected": -0.22237196564674377,
      "logps/chosen": -400.22296142578125,
      "logps/rejected": -423.6072998046875,
      "loss": 0.8763,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6828949451446533,
      "rewards/margins": -0.05785579979419708,
      "rewards/rejected": -0.6250391602516174,
      "step": 3816
    },
    {
      "epoch": 0.49948474687167743,
      "grad_norm": 5.834029674530029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22542411088943481,
      "logits/rejected": 0.28791558742523193,
      "logps/chosen": -415.5003662109375,
      "logps/rejected": -396.8848876953125,
      "loss": 0.5013,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01602724753320217,
      "rewards/margins": 0.7938386797904968,
      "rewards/rejected": -0.7778114080429077,
      "step": 3817
    },
    {
      "epoch": 0.4996156048090292,
      "grad_norm": 4.076488018035889,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07518601417541504,
      "logits/rejected": -0.18562015891075134,
      "logps/chosen": -378.544677734375,
      "logps/rejected": -342.6462707519531,
      "loss": 0.2975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11798574030399323,
      "rewards/margins": 1.297785997390747,
      "rewards/rejected": -1.179800271987915,
      "step": 3818
    },
    {
      "epoch": 0.49974646274638096,
      "grad_norm": 6.786406517028809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1717006266117096,
      "logits/rejected": 0.1782538890838623,
      "logps/chosen": -461.1631164550781,
      "logps/rejected": -558.9979248046875,
      "loss": 0.3987,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07911987602710724,
      "rewards/margins": 1.004648208618164,
      "rewards/rejected": -0.925528347492218,
      "step": 3819
    },
    {
      "epoch": 0.4998773206837327,
      "grad_norm": 9.72743034362793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05115923285484314,
      "logits/rejected": 0.12867237627506256,
      "logps/chosen": -490.463134765625,
      "logps/rejected": -431.6240234375,
      "loss": 0.6561,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.36928483843803406,
      "rewards/margins": 0.6373512744903564,
      "rewards/rejected": -0.26806640625,
      "step": 3820
    },
    {
      "epoch": 0.5000081786210845,
      "grad_norm": 6.104806423187256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10659868270158768,
      "logits/rejected": 0.005042597651481628,
      "logps/chosen": -417.10943603515625,
      "logps/rejected": -489.05352783203125,
      "loss": 0.5005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14323101937770844,
      "rewards/margins": 0.7900157570838928,
      "rewards/rejected": -0.9332468509674072,
      "step": 3821
    },
    {
      "epoch": 0.5001390365584363,
      "grad_norm": 5.524298191070557,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16260330379009247,
      "logits/rejected": -0.12203608453273773,
      "logps/chosen": -410.8310852050781,
      "logps/rejected": -384.39666748046875,
      "loss": 0.4305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41151028871536255,
      "rewards/margins": 0.7667335271835327,
      "rewards/rejected": -1.17824387550354,
      "step": 3822
    },
    {
      "epoch": 0.500269894495788,
      "grad_norm": 12.289399147033691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039170071482658386,
      "logits/rejected": -0.0316612683236599,
      "logps/chosen": -399.52484130859375,
      "logps/rejected": -398.3007507324219,
      "loss": 1.1854,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0874849334359169,
      "rewards/margins": -0.5761170387268066,
      "rewards/rejected": 0.48863205313682556,
      "step": 3823
    },
    {
      "epoch": 0.5004007524331398,
      "grad_norm": 10.103363990783691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043114110827445984,
      "logits/rejected": -0.14080464839935303,
      "logps/chosen": -363.4582824707031,
      "logps/rejected": -355.84417724609375,
      "loss": 0.6448,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2235511839389801,
      "rewards/margins": 0.3177395164966583,
      "rewards/rejected": -0.5412906408309937,
      "step": 3824
    },
    {
      "epoch": 0.5005316103704915,
      "grad_norm": 4.616311073303223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0006954353302717209,
      "logits/rejected": -0.0501520112156868,
      "logps/chosen": -378.66656494140625,
      "logps/rejected": -333.5379943847656,
      "loss": 0.3721,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5237504839897156,
      "rewards/margins": 1.148794174194336,
      "rewards/rejected": -1.6725447177886963,
      "step": 3825
    },
    {
      "epoch": 0.5006624683078433,
      "grad_norm": 7.980276107788086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17820046842098236,
      "logits/rejected": 0.1078406274318695,
      "logps/chosen": -558.2643432617188,
      "logps/rejected": -392.9316711425781,
      "loss": 0.4931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4316551387310028,
      "rewards/margins": 0.8307849764823914,
      "rewards/rejected": -0.39912986755371094,
      "step": 3826
    },
    {
      "epoch": 0.500793326245195,
      "grad_norm": 7.171003818511963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0722777470946312,
      "logits/rejected": -0.015473013743758202,
      "logps/chosen": -480.8478088378906,
      "logps/rejected": -463.1062927246094,
      "loss": 0.4034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1563209742307663,
      "rewards/margins": 1.457689642906189,
      "rewards/rejected": -1.3013687133789062,
      "step": 3827
    },
    {
      "epoch": 0.5009241841825468,
      "grad_norm": 7.38768196105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16945359110832214,
      "logits/rejected": 0.07063557952642441,
      "logps/chosen": -353.62310791015625,
      "logps/rejected": -455.55322265625,
      "loss": 0.5866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6578112244606018,
      "rewards/margins": 0.5208625197410583,
      "rewards/rejected": -1.1786737442016602,
      "step": 3828
    },
    {
      "epoch": 0.5010550421198986,
      "grad_norm": 8.168139457702637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047043465077877045,
      "logits/rejected": 0.11608454585075378,
      "logps/chosen": -590.7890014648438,
      "logps/rejected": -468.43536376953125,
      "loss": 0.3792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.32436656951904297,
      "rewards/margins": 1.1593270301818848,
      "rewards/rejected": -0.8349603414535522,
      "step": 3829
    },
    {
      "epoch": 0.5011859000572504,
      "grad_norm": 4.76793098449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01618509739637375,
      "logits/rejected": 0.045038264244794846,
      "logps/chosen": -345.914306640625,
      "logps/rejected": -289.2184143066406,
      "loss": 0.363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.028362281620502472,
      "rewards/margins": 1.1815247535705566,
      "rewards/rejected": -1.1531624794006348,
      "step": 3830
    },
    {
      "epoch": 0.5013167579946021,
      "grad_norm": 4.950937271118164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16403627395629883,
      "logits/rejected": 0.14116239547729492,
      "logps/chosen": -458.6470642089844,
      "logps/rejected": -386.6412658691406,
      "loss": 0.3633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9329587817192078,
      "rewards/margins": 0.9547809958457947,
      "rewards/rejected": -0.021822161972522736,
      "step": 3831
    },
    {
      "epoch": 0.5014476159319539,
      "grad_norm": 8.961958885192871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11314107477664948,
      "logits/rejected": 0.09797792136669159,
      "logps/chosen": -421.7645263671875,
      "logps/rejected": -420.37908935546875,
      "loss": 0.538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3921062648296356,
      "rewards/margins": 1.3061603307724,
      "rewards/rejected": -0.9140539169311523,
      "step": 3832
    },
    {
      "epoch": 0.5015784738693057,
      "grad_norm": 7.949370861053467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02905009500682354,
      "logits/rejected": -0.03397936373949051,
      "logps/chosen": -396.59814453125,
      "logps/rejected": -366.5708312988281,
      "loss": 0.5887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08589668571949005,
      "rewards/margins": 0.43380433320999146,
      "rewards/rejected": -0.3479076623916626,
      "step": 3833
    },
    {
      "epoch": 0.5017093318066574,
      "grad_norm": 11.582130432128906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10532087832689285,
      "logits/rejected": -0.030276581645011902,
      "logps/chosen": -455.3076477050781,
      "logps/rejected": -362.4909973144531,
      "loss": 0.8337,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20476262271404266,
      "rewards/margins": 0.3632010817527771,
      "rewards/rejected": -0.15843847393989563,
      "step": 3834
    },
    {
      "epoch": 0.5018401897440091,
      "grad_norm": 6.3370866775512695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019534841179847717,
      "logits/rejected": -0.18006810545921326,
      "logps/chosen": -353.3288879394531,
      "logps/rejected": -515.0907592773438,
      "loss": 0.5559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5376899838447571,
      "rewards/margins": 0.8082912564277649,
      "rewards/rejected": -1.345981240272522,
      "step": 3835
    },
    {
      "epoch": 0.5019710476813609,
      "grad_norm": 14.286253929138184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039959028363227844,
      "logits/rejected": -0.09269586205482483,
      "logps/chosen": -389.0260925292969,
      "logps/rejected": -327.78045654296875,
      "loss": 0.8292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08201293647289276,
      "rewards/margins": 0.3589804768562317,
      "rewards/rejected": -0.27696743607521057,
      "step": 3836
    },
    {
      "epoch": 0.5021019056187127,
      "grad_norm": 10.101256370544434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055744707584381104,
      "logits/rejected": -0.0942801833152771,
      "logps/chosen": -461.91314697265625,
      "logps/rejected": -438.2828369140625,
      "loss": 0.5919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2888058125972748,
      "rewards/margins": 1.0760653018951416,
      "rewards/rejected": -1.3648710250854492,
      "step": 3837
    },
    {
      "epoch": 0.5022327635560645,
      "grad_norm": 6.151357650756836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20212812721729279,
      "logits/rejected": 0.1313539743423462,
      "logps/chosen": -513.6852416992188,
      "logps/rejected": -381.1593322753906,
      "loss": 0.393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1296367645263672,
      "rewards/margins": 0.9261386394500732,
      "rewards/rejected": -0.7965018153190613,
      "step": 3838
    },
    {
      "epoch": 0.5023636214934162,
      "grad_norm": 7.4624738693237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1328527331352234,
      "logits/rejected": -0.2116658091545105,
      "logps/chosen": -351.44940185546875,
      "logps/rejected": -444.5224609375,
      "loss": 0.6378,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.658685564994812,
      "rewards/margins": 0.40935784578323364,
      "rewards/rejected": -1.0680432319641113,
      "step": 3839
    },
    {
      "epoch": 0.502494479430768,
      "grad_norm": 4.915609359741211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1436467319726944,
      "logits/rejected": 0.01091795600950718,
      "logps/chosen": -314.005859375,
      "logps/rejected": -322.8902587890625,
      "loss": 0.4615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2618221044540405,
      "rewards/margins": 0.7145134210586548,
      "rewards/rejected": -0.9763355851173401,
      "step": 3840
    },
    {
      "epoch": 0.5026253373681198,
      "grad_norm": 2.061314344406128,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05675453692674637,
      "logits/rejected": -0.3172031044960022,
      "logps/chosen": -307.976806640625,
      "logps/rejected": -394.252197265625,
      "loss": 0.1924,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10581417381763458,
      "rewards/margins": 2.34199595451355,
      "rewards/rejected": -2.447809934616089,
      "step": 3841
    },
    {
      "epoch": 0.5027561953054714,
      "grad_norm": 9.589550971984863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.250119686126709,
      "logits/rejected": -0.015105783939361572,
      "logps/chosen": -474.153564453125,
      "logps/rejected": -469.4124755859375,
      "loss": 0.7339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4020373821258545,
      "rewards/margins": 0.08960096538066864,
      "rewards/rejected": 0.3124364912509918,
      "step": 3842
    },
    {
      "epoch": 0.5028870532428232,
      "grad_norm": 5.941462993621826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016292065382003784,
      "logits/rejected": -0.03610663115978241,
      "logps/chosen": -714.4405517578125,
      "logps/rejected": -654.6027221679688,
      "loss": 0.5056,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5172388553619385,
      "rewards/margins": 0.8111395239830017,
      "rewards/rejected": -0.29390066862106323,
      "step": 3843
    },
    {
      "epoch": 0.503017911180175,
      "grad_norm": 4.877110958099365,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08016017824411392,
      "logits/rejected": 0.012727637775242329,
      "logps/chosen": -504.9040222167969,
      "logps/rejected": -522.9949340820312,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.751318633556366,
      "rewards/margins": 1.2676222324371338,
      "rewards/rejected": -0.5163035988807678,
      "step": 3844
    },
    {
      "epoch": 0.5031487691175268,
      "grad_norm": 6.455397605895996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14356714487075806,
      "logits/rejected": 0.1851734220981598,
      "logps/chosen": -518.6285400390625,
      "logps/rejected": -443.45477294921875,
      "loss": 0.5404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.055162057280540466,
      "rewards/margins": 0.8768043518066406,
      "rewards/rejected": -0.9319663643836975,
      "step": 3845
    },
    {
      "epoch": 0.5032796270548785,
      "grad_norm": 5.839802265167236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12496800720691681,
      "logits/rejected": -0.05682716518640518,
      "logps/chosen": -559.5120239257812,
      "logps/rejected": -505.61505126953125,
      "loss": 0.4049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3111650347709656,
      "rewards/margins": 1.0927733182907104,
      "rewards/rejected": -0.7816083431243896,
      "step": 3846
    },
    {
      "epoch": 0.5034104849922303,
      "grad_norm": 6.109373092651367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017688743770122528,
      "logits/rejected": -0.028245508670806885,
      "logps/chosen": -480.9361877441406,
      "logps/rejected": -379.4905700683594,
      "loss": 0.5475,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2052721083164215,
      "rewards/margins": 0.605589747428894,
      "rewards/rejected": -0.40031760931015015,
      "step": 3847
    },
    {
      "epoch": 0.5035413429295821,
      "grad_norm": 9.868839263916016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.096847303211689,
      "logits/rejected": -0.2495276927947998,
      "logps/chosen": -405.4072570800781,
      "logps/rejected": -574.4013671875,
      "loss": 0.7885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18669089674949646,
      "rewards/margins": 0.344174325466156,
      "rewards/rejected": -0.5308653116226196,
      "step": 3848
    },
    {
      "epoch": 0.5036722008669339,
      "grad_norm": 5.134943962097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26459914445877075,
      "logits/rejected": 0.04023858904838562,
      "logps/chosen": -418.76739501953125,
      "logps/rejected": -381.3998107910156,
      "loss": 0.3948,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5360714197158813,
      "rewards/margins": 1.183527946472168,
      "rewards/rejected": -0.6474565267562866,
      "step": 3849
    },
    {
      "epoch": 0.5038030588042856,
      "grad_norm": 7.5717291831970215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09012921899557114,
      "logits/rejected": -0.11443313956260681,
      "logps/chosen": -528.0220947265625,
      "logps/rejected": -347.4199523925781,
      "loss": 0.6326,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14920200407505035,
      "rewards/margins": 0.31003302335739136,
      "rewards/rejected": -0.4592350125312805,
      "step": 3850
    },
    {
      "epoch": 0.5039339167416373,
      "grad_norm": 7.318285942077637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03854307904839516,
      "logits/rejected": -0.06303702294826508,
      "logps/chosen": -409.9674987792969,
      "logps/rejected": -526.2494506835938,
      "loss": 0.5315,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6693758368492126,
      "rewards/margins": 0.4847501516342163,
      "rewards/rejected": -1.1541260480880737,
      "step": 3851
    },
    {
      "epoch": 0.5040647746789891,
      "grad_norm": 7.951963901519775,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11492704600095749,
      "logits/rejected": -0.21152859926223755,
      "logps/chosen": -473.31561279296875,
      "logps/rejected": -711.0462036132812,
      "loss": 0.5545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.32123756408691406,
      "rewards/margins": 1.1353065967559814,
      "rewards/rejected": -0.8140690326690674,
      "step": 3852
    },
    {
      "epoch": 0.5041956326163409,
      "grad_norm": 7.959275722503662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1646730899810791,
      "logits/rejected": -0.24521540105342865,
      "logps/chosen": -564.801513671875,
      "logps/rejected": -522.4686279296875,
      "loss": 0.4855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33320313692092896,
      "rewards/margins": 0.7787651419639587,
      "rewards/rejected": -1.1119682788848877,
      "step": 3853
    },
    {
      "epoch": 0.5043264905536926,
      "grad_norm": 7.272356033325195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24510152637958527,
      "logits/rejected": 0.029625410214066505,
      "logps/chosen": -503.17108154296875,
      "logps/rejected": -562.4131469726562,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08255864679813385,
      "rewards/margins": 0.7682782411575317,
      "rewards/rejected": -0.850836992263794,
      "step": 3854
    },
    {
      "epoch": 0.5044573484910444,
      "grad_norm": 6.489266395568848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008471090346574783,
      "logits/rejected": -0.13705481588840485,
      "logps/chosen": -582.7418212890625,
      "logps/rejected": -547.7848510742188,
      "loss": 0.3723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6155735850334167,
      "rewards/margins": 1.258337140083313,
      "rewards/rejected": -0.6427634954452515,
      "step": 3855
    },
    {
      "epoch": 0.5045882064283962,
      "grad_norm": 10.485393524169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07736651599407196,
      "logits/rejected": -0.2158718854188919,
      "logps/chosen": -375.032958984375,
      "logps/rejected": -360.97967529296875,
      "loss": 0.6418,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2488539218902588,
      "rewards/margins": 0.3220295011997223,
      "rewards/rejected": -1.5708832740783691,
      "step": 3856
    },
    {
      "epoch": 0.504719064365748,
      "grad_norm": 4.317868709564209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12553320825099945,
      "logits/rejected": 0.046401962637901306,
      "logps/chosen": -421.0950012207031,
      "logps/rejected": -378.1311950683594,
      "loss": 0.237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09133301675319672,
      "rewards/margins": 1.568331241607666,
      "rewards/rejected": -1.4769984483718872,
      "step": 3857
    },
    {
      "epoch": 0.5048499223030997,
      "grad_norm": 11.584938049316406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23528742790222168,
      "logits/rejected": 0.14872702956199646,
      "logps/chosen": -445.59356689453125,
      "logps/rejected": -330.7492980957031,
      "loss": 0.5409,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.049828335642814636,
      "rewards/margins": 1.0069061517715454,
      "rewards/rejected": -0.9570777416229248,
      "step": 3858
    },
    {
      "epoch": 0.5049807802404515,
      "grad_norm": 9.362961769104004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4295157194137573,
      "logits/rejected": 0.11461620032787323,
      "logps/chosen": -559.9459228515625,
      "logps/rejected": -383.6526794433594,
      "loss": 0.5403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.411995530128479,
      "rewards/margins": 0.7896478176116943,
      "rewards/rejected": -0.37765228748321533,
      "step": 3859
    },
    {
      "epoch": 0.5051116381778032,
      "grad_norm": 7.374752044677734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04267150163650513,
      "logits/rejected": -0.034163590520620346,
      "logps/chosen": -441.415283203125,
      "logps/rejected": -410.17523193359375,
      "loss": 0.6351,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4651319682598114,
      "rewards/margins": 0.2836102843284607,
      "rewards/rejected": -0.7487423419952393,
      "step": 3860
    },
    {
      "epoch": 0.505242496115155,
      "grad_norm": 6.460988998413086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23628172278404236,
      "logits/rejected": -0.09858455508947372,
      "logps/chosen": -300.2196044921875,
      "logps/rejected": -366.71282958984375,
      "loss": 0.5742,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6691595315933228,
      "rewards/margins": 0.4512481689453125,
      "rewards/rejected": -1.1204077005386353,
      "step": 3861
    },
    {
      "epoch": 0.5053733540525067,
      "grad_norm": 8.76490592956543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0012295206543058157,
      "logits/rejected": 0.07993102818727493,
      "logps/chosen": -767.50244140625,
      "logps/rejected": -644.2573852539062,
      "loss": 0.6415,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.34033966064453125,
      "rewards/margins": 0.6347748637199402,
      "rewards/rejected": -0.2944351136684418,
      "step": 3862
    },
    {
      "epoch": 0.5055042119898585,
      "grad_norm": 10.113003730773926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07234267890453339,
      "logits/rejected": -0.22038337588310242,
      "logps/chosen": -360.3337097167969,
      "logps/rejected": -391.8841247558594,
      "loss": 0.4737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2143852263689041,
      "rewards/margins": 1.077656865119934,
      "rewards/rejected": -1.2920421361923218,
      "step": 3863
    },
    {
      "epoch": 0.5056350699272103,
      "grad_norm": 7.443068027496338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04673813283443451,
      "logits/rejected": 0.12291279435157776,
      "logps/chosen": -413.3179016113281,
      "logps/rejected": -396.2896728515625,
      "loss": 0.7489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6948209404945374,
      "rewards/margins": 0.12730085849761963,
      "rewards/rejected": -0.822121798992157,
      "step": 3864
    },
    {
      "epoch": 0.505765927864562,
      "grad_norm": 5.734425067901611,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2290317416191101,
      "logits/rejected": 0.18966570496559143,
      "logps/chosen": -452.39324951171875,
      "logps/rejected": -351.8084411621094,
      "loss": 0.4071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33961760997772217,
      "rewards/margins": 0.960623025894165,
      "rewards/rejected": -0.6210054755210876,
      "step": 3865
    },
    {
      "epoch": 0.5058967858019138,
      "grad_norm": 11.468381881713867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1533176302909851,
      "logits/rejected": -0.04704873636364937,
      "logps/chosen": -529.2855224609375,
      "logps/rejected": -438.8527526855469,
      "loss": 0.7819,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5896420478820801,
      "rewards/margins": 0.1978101134300232,
      "rewards/rejected": -0.7874521017074585,
      "step": 3866
    },
    {
      "epoch": 0.5060276437392656,
      "grad_norm": 6.6529130935668945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00837753526866436,
      "logits/rejected": 0.025481248274445534,
      "logps/chosen": -371.1920471191406,
      "logps/rejected": -383.08721923828125,
      "loss": 0.8259,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.390006422996521,
      "rewards/margins": -0.013164877891540527,
      "rewards/rejected": -0.3768415153026581,
      "step": 3867
    },
    {
      "epoch": 0.5061585016766174,
      "grad_norm": 9.94361400604248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10579391568899155,
      "logits/rejected": -0.10699110478162766,
      "logps/chosen": -467.3940734863281,
      "logps/rejected": -469.7790222167969,
      "loss": 0.8408,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2904230058193207,
      "rewards/margins": -0.1536068171262741,
      "rewards/rejected": -0.13681621849536896,
      "step": 3868
    },
    {
      "epoch": 0.506289359613969,
      "grad_norm": 8.961650848388672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15473487973213196,
      "logits/rejected": 0.04039037227630615,
      "logps/chosen": -338.7415771484375,
      "logps/rejected": -326.2753601074219,
      "loss": 0.6043,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2912527024745941,
      "rewards/margins": 0.6952798962593079,
      "rewards/rejected": -0.9865325689315796,
      "step": 3869
    },
    {
      "epoch": 0.5064202175513208,
      "grad_norm": 7.189211845397949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18928730487823486,
      "logits/rejected": 0.12048465758562088,
      "logps/chosen": -448.7693786621094,
      "logps/rejected": -601.287109375,
      "loss": 0.5043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18695029616355896,
      "rewards/margins": 1.2716312408447266,
      "rewards/rejected": -1.0846809148788452,
      "step": 3870
    },
    {
      "epoch": 0.5065510754886726,
      "grad_norm": 6.295964241027832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06290293484926224,
      "logits/rejected": -0.16497638821601868,
      "logps/chosen": -427.48687744140625,
      "logps/rejected": -340.93011474609375,
      "loss": 0.4885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005178265273571014,
      "rewards/margins": 1.066165566444397,
      "rewards/rejected": -1.0609873533248901,
      "step": 3871
    },
    {
      "epoch": 0.5066819334260244,
      "grad_norm": 7.185105800628662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34257587790489197,
      "logits/rejected": 0.2973964214324951,
      "logps/chosen": -463.6536865234375,
      "logps/rejected": -513.0941772460938,
      "loss": 0.4578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5708863735198975,
      "rewards/margins": 0.868343710899353,
      "rewards/rejected": -0.2974572777748108,
      "step": 3872
    },
    {
      "epoch": 0.5068127913633761,
      "grad_norm": 6.109979152679443,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005081929266452789,
      "logits/rejected": 0.12634392082691193,
      "logps/chosen": -607.9498291015625,
      "logps/rejected": -577.1510009765625,
      "loss": 0.3524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.044515617191791534,
      "rewards/margins": 1.0656979084014893,
      "rewards/rejected": -1.1102135181427002,
      "step": 3873
    },
    {
      "epoch": 0.5069436493007279,
      "grad_norm": 10.144469261169434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09878270328044891,
      "logits/rejected": -0.25987839698791504,
      "logps/chosen": -498.7445068359375,
      "logps/rejected": -388.101318359375,
      "loss": 0.9512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8085205554962158,
      "rewards/margins": -0.10597309470176697,
      "rewards/rejected": -0.7025474309921265,
      "step": 3874
    },
    {
      "epoch": 0.5070745072380797,
      "grad_norm": 9.968939781188965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31597915291786194,
      "logits/rejected": -0.20879380404949188,
      "logps/chosen": -480.073974609375,
      "logps/rejected": -404.2821960449219,
      "loss": 0.7561,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17442798614501953,
      "rewards/margins": 0.383078396320343,
      "rewards/rejected": -0.5575063824653625,
      "step": 3875
    },
    {
      "epoch": 0.5072053651754315,
      "grad_norm": 6.111313343048096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2552988529205322,
      "logits/rejected": -0.008245393633842468,
      "logps/chosen": -426.9972839355469,
      "logps/rejected": -485.4293212890625,
      "loss": 0.3477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14753475785255432,
      "rewards/margins": 1.1666007041931152,
      "rewards/rejected": -1.0190658569335938,
      "step": 3876
    },
    {
      "epoch": 0.5073362231127831,
      "grad_norm": 7.10341215133667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042618848383426666,
      "logits/rejected": -0.09482140839099884,
      "logps/chosen": -452.40704345703125,
      "logps/rejected": -444.67889404296875,
      "loss": 0.6197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.737221360206604,
      "rewards/margins": 0.2935422956943512,
      "rewards/rejected": 0.4436790943145752,
      "step": 3877
    },
    {
      "epoch": 0.5074670810501349,
      "grad_norm": 9.54907512664795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1329788863658905,
      "logits/rejected": 0.06592771410942078,
      "logps/chosen": -490.237548828125,
      "logps/rejected": -409.2825012207031,
      "loss": 0.7267,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.36306267976760864,
      "rewards/margins": 0.4376317858695984,
      "rewards/rejected": -0.07456915825605392,
      "step": 3878
    },
    {
      "epoch": 0.5075979389874867,
      "grad_norm": 5.520009517669678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0731118768453598,
      "logits/rejected": 0.14763665199279785,
      "logps/chosen": -613.1502075195312,
      "logps/rejected": -542.8916625976562,
      "loss": 0.4599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1259903609752655,
      "rewards/margins": 0.7405183911323547,
      "rewards/rejected": -0.8665087819099426,
      "step": 3879
    },
    {
      "epoch": 0.5077287969248385,
      "grad_norm": 5.854357719421387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032727353274822235,
      "logits/rejected": 0.006625339388847351,
      "logps/chosen": -381.3424987792969,
      "logps/rejected": -383.43609619140625,
      "loss": 0.4353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2535640597343445,
      "rewards/margins": 0.8253587484359741,
      "rewards/rejected": -0.5717947483062744,
      "step": 3880
    },
    {
      "epoch": 0.5078596548621902,
      "grad_norm": 6.6163010597229,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20972023904323578,
      "logits/rejected": 0.13770705461502075,
      "logps/chosen": -389.0001220703125,
      "logps/rejected": -314.5440673828125,
      "loss": 0.6747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28737008571624756,
      "rewards/margins": 0.4071587920188904,
      "rewards/rejected": -0.6945289373397827,
      "step": 3881
    },
    {
      "epoch": 0.507990512799542,
      "grad_norm": 2.9683332443237305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01676139235496521,
      "logits/rejected": -0.15490949153900146,
      "logps/chosen": -363.7423095703125,
      "logps/rejected": -425.538818359375,
      "loss": 0.2397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9191877245903015,
      "rewards/margins": 1.575649380683899,
      "rewards/rejected": -0.6564614772796631,
      "step": 3882
    },
    {
      "epoch": 0.5081213707368938,
      "grad_norm": 6.938782215118408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04347119852900505,
      "logits/rejected": 0.1278507113456726,
      "logps/chosen": -476.5567321777344,
      "logps/rejected": -396.42559814453125,
      "loss": 0.4933,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6397072076797485,
      "rewards/margins": 0.8060925006866455,
      "rewards/rejected": -0.16638527810573578,
      "step": 3883
    },
    {
      "epoch": 0.5082522286742456,
      "grad_norm": 8.600446701049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10601920634508133,
      "logits/rejected": -0.08008898049592972,
      "logps/chosen": -549.599853515625,
      "logps/rejected": -468.5374755859375,
      "loss": 0.5909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.23855799436569214,
      "rewards/margins": 0.8482144474983215,
      "rewards/rejected": -0.6096563339233398,
      "step": 3884
    },
    {
      "epoch": 0.5083830866115973,
      "grad_norm": 6.012819766998291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04830557480454445,
      "logits/rejected": 0.060519877821207047,
      "logps/chosen": -529.5308837890625,
      "logps/rejected": -635.8388061523438,
      "loss": 0.2554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8023822903633118,
      "rewards/margins": 1.5413318872451782,
      "rewards/rejected": -0.7389495372772217,
      "step": 3885
    },
    {
      "epoch": 0.508513944548949,
      "grad_norm": 6.3142876625061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26600348949432373,
      "logits/rejected": -0.1271561086177826,
      "logps/chosen": -584.3206787109375,
      "logps/rejected": -399.4749755859375,
      "loss": 0.3747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33962076902389526,
      "rewards/margins": 1.2523539066314697,
      "rewards/rejected": -0.9127331376075745,
      "step": 3886
    },
    {
      "epoch": 0.5086448024863008,
      "grad_norm": 6.378666877746582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1801699995994568,
      "logits/rejected": -0.03541130572557449,
      "logps/chosen": -357.7886657714844,
      "logps/rejected": -298.737060546875,
      "loss": 0.4849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12205374240875244,
      "rewards/margins": 1.0568933486938477,
      "rewards/rejected": -0.9348396062850952,
      "step": 3887
    },
    {
      "epoch": 0.5087756604236525,
      "grad_norm": 6.836721897125244,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1596495807170868,
      "logits/rejected": -0.10208684951066971,
      "logps/chosen": -256.8427429199219,
      "logps/rejected": -298.1047058105469,
      "loss": 0.5743,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09401074051856995,
      "rewards/margins": 0.8334348797798157,
      "rewards/rejected": -0.7394241094589233,
      "step": 3888
    },
    {
      "epoch": 0.5089065183610043,
      "grad_norm": 10.530349731445312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39890536665916443,
      "logits/rejected": 0.20960426330566406,
      "logps/chosen": -542.6188354492188,
      "logps/rejected": -471.7245178222656,
      "loss": 0.8278,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33455201983451843,
      "rewards/margins": 0.5904701352119446,
      "rewards/rejected": -0.25591814517974854,
      "step": 3889
    },
    {
      "epoch": 0.5090373762983561,
      "grad_norm": 10.694319725036621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03634493425488472,
      "logits/rejected": 0.21384301781654358,
      "logps/chosen": -284.5919494628906,
      "logps/rejected": -445.7142028808594,
      "loss": 1.0129,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.12914752960205078,
      "rewards/margins": -0.38979852199554443,
      "rewards/rejected": 0.51894611120224,
      "step": 3890
    },
    {
      "epoch": 0.5091682342357079,
      "grad_norm": 6.017115116119385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06788694858551025,
      "logits/rejected": -0.13791175186634064,
      "logps/chosen": -455.16412353515625,
      "logps/rejected": -467.439208984375,
      "loss": 0.3396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24732056260108948,
      "rewards/margins": 1.2260539531707764,
      "rewards/rejected": -0.9787333607673645,
      "step": 3891
    },
    {
      "epoch": 0.5092990921730596,
      "grad_norm": 11.00070571899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049369487911462784,
      "logits/rejected": -0.16541126370429993,
      "logps/chosen": -542.4365234375,
      "logps/rejected": -432.71160888671875,
      "loss": 0.9172,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15725421905517578,
      "rewards/margins": 0.09293881058692932,
      "rewards/rejected": 0.06431539356708527,
      "step": 3892
    },
    {
      "epoch": 0.5094299501104114,
      "grad_norm": 5.03533935546875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08181113749742508,
      "logits/rejected": 0.1487017124891281,
      "logps/chosen": -411.6049499511719,
      "logps/rejected": -421.31121826171875,
      "loss": 0.4072,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.17548808455467224,
      "rewards/margins": 1.2874327898025513,
      "rewards/rejected": -1.1119447946548462,
      "step": 3893
    },
    {
      "epoch": 0.5095608080477632,
      "grad_norm": 5.03542947769165,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0350029431283474,
      "logits/rejected": 0.1170758306980133,
      "logps/chosen": -351.3801574707031,
      "logps/rejected": -429.1540222167969,
      "loss": 0.3151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7481987476348877,
      "rewards/margins": 1.1941158771514893,
      "rewards/rejected": -0.44591715931892395,
      "step": 3894
    },
    {
      "epoch": 0.509691665985115,
      "grad_norm": 7.814517498016357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31836822628974915,
      "logits/rejected": 0.14604970812797546,
      "logps/chosen": -462.489013671875,
      "logps/rejected": -512.3633422851562,
      "loss": 0.6117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06479224562644958,
      "rewards/margins": 0.5987716913223267,
      "rewards/rejected": -0.6635639667510986,
      "step": 3895
    },
    {
      "epoch": 0.5098225239224666,
      "grad_norm": 5.247933387756348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21065695583820343,
      "logits/rejected": -0.08716575801372528,
      "logps/chosen": -363.594482421875,
      "logps/rejected": -374.6636657714844,
      "loss": 0.3861,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5037282705307007,
      "rewards/margins": 1.2697317600250244,
      "rewards/rejected": -0.7660034894943237,
      "step": 3896
    },
    {
      "epoch": 0.5099533818598184,
      "grad_norm": 11.165066719055176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03819015622138977,
      "logits/rejected": -0.00738903135061264,
      "logps/chosen": -363.5098571777344,
      "logps/rejected": -302.7313537597656,
      "loss": 0.6479,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.22379285097122192,
      "rewards/margins": 0.4961565136909485,
      "rewards/rejected": -0.7199493050575256,
      "step": 3897
    },
    {
      "epoch": 0.5100842397971702,
      "grad_norm": 4.534543037414551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016766726970672607,
      "logits/rejected": 0.1001654639840126,
      "logps/chosen": -427.9505310058594,
      "logps/rejected": -497.21795654296875,
      "loss": 0.3263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.060279637575149536,
      "rewards/margins": 1.6107237339019775,
      "rewards/rejected": -1.6710034608840942,
      "step": 3898
    },
    {
      "epoch": 0.510215097734522,
      "grad_norm": 8.511924743652344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14083331823349,
      "logits/rejected": 0.2574983835220337,
      "logps/chosen": -331.80023193359375,
      "logps/rejected": -388.5089416503906,
      "loss": 0.7051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09030991792678833,
      "rewards/margins": 0.41496849060058594,
      "rewards/rejected": -0.32465860247612,
      "step": 3899
    },
    {
      "epoch": 0.5103459556718737,
      "grad_norm": 8.047821998596191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08118607103824615,
      "logits/rejected": 0.05248478055000305,
      "logps/chosen": -343.1304626464844,
      "logps/rejected": -364.23114013671875,
      "loss": 0.7019,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12596912682056427,
      "rewards/margins": 0.17359961569309235,
      "rewards/rejected": -0.04763051122426987,
      "step": 3900
    },
    {
      "epoch": 0.5104768136092255,
      "grad_norm": 7.42818546295166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15690100193023682,
      "logits/rejected": -0.01663932576775551,
      "logps/chosen": -678.3021240234375,
      "logps/rejected": -511.3766174316406,
      "loss": 0.4814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.019694477319717407,
      "rewards/margins": 0.5912948846817017,
      "rewards/rejected": -0.6109893918037415,
      "step": 3901
    },
    {
      "epoch": 0.5106076715465773,
      "grad_norm": 8.1851167678833,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25127658247947693,
      "logits/rejected": 0.23143386840820312,
      "logps/chosen": -453.3785095214844,
      "logps/rejected": -536.0272216796875,
      "loss": 0.6119,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01092987135052681,
      "rewards/margins": 0.723251223564148,
      "rewards/rejected": -0.7123213410377502,
      "step": 3902
    },
    {
      "epoch": 0.5107385294839291,
      "grad_norm": 7.565095901489258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45367148518562317,
      "logits/rejected": -0.5650128126144409,
      "logps/chosen": -421.35235595703125,
      "logps/rejected": -375.781005859375,
      "loss": 0.5805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7345327138900757,
      "rewards/margins": 0.3740871846675873,
      "rewards/rejected": -1.1086199283599854,
      "step": 3903
    },
    {
      "epoch": 0.5108693874212807,
      "grad_norm": 15.059305191040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3228524923324585,
      "logits/rejected": -0.22896811366081238,
      "logps/chosen": -353.55499267578125,
      "logps/rejected": -319.5336608886719,
      "loss": 0.7685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13892385363578796,
      "rewards/margins": 0.5409040451049805,
      "rewards/rejected": -0.6798279285430908,
      "step": 3904
    },
    {
      "epoch": 0.5110002453586325,
      "grad_norm": 11.17052173614502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23785331845283508,
      "logits/rejected": 0.0768883004784584,
      "logps/chosen": -469.65179443359375,
      "logps/rejected": -355.73858642578125,
      "loss": 0.6134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38306599855422974,
      "rewards/margins": 0.5444311499595642,
      "rewards/rejected": -0.927497148513794,
      "step": 3905
    },
    {
      "epoch": 0.5111311032959843,
      "grad_norm": 6.584640979766846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09816038608551025,
      "logits/rejected": 0.042029887437820435,
      "logps/chosen": -372.79638671875,
      "logps/rejected": -565.7674560546875,
      "loss": 0.4277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5010913610458374,
      "rewards/margins": 1.1595426797866821,
      "rewards/rejected": -0.6584512591362,
      "step": 3906
    },
    {
      "epoch": 0.511261961233336,
      "grad_norm": 9.419108390808105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25496119260787964,
      "logits/rejected": 0.0835922583937645,
      "logps/chosen": -649.1075439453125,
      "logps/rejected": -501.76385498046875,
      "loss": 0.707,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19525527954101562,
      "rewards/margins": 0.6018309593200684,
      "rewards/rejected": -0.40657564997673035,
      "step": 3907
    },
    {
      "epoch": 0.5113928191706878,
      "grad_norm": 8.894588470458984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16472233831882477,
      "logits/rejected": 0.014212831854820251,
      "logps/chosen": -329.9513854980469,
      "logps/rejected": -354.08563232421875,
      "loss": 0.7485,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4158592224121094,
      "rewards/margins": 0.13117104768753052,
      "rewards/rejected": -0.5470302104949951,
      "step": 3908
    },
    {
      "epoch": 0.5115236771080396,
      "grad_norm": 5.741969108581543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12171468138694763,
      "logits/rejected": -0.13867339491844177,
      "logps/chosen": -403.67242431640625,
      "logps/rejected": -420.9014587402344,
      "loss": 0.3504,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44975465536117554,
      "rewards/margins": 1.379986047744751,
      "rewards/rejected": -0.9302314519882202,
      "step": 3909
    },
    {
      "epoch": 0.5116545350453914,
      "grad_norm": 4.3835930824279785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040646787732839584,
      "logits/rejected": 0.19579342007637024,
      "logps/chosen": -406.7637939453125,
      "logps/rejected": -460.79443359375,
      "loss": 0.2663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05996611714363098,
      "rewards/margins": 1.2972829341888428,
      "rewards/rejected": -1.3572490215301514,
      "step": 3910
    },
    {
      "epoch": 0.5117853929827431,
      "grad_norm": 6.55866813659668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3264234960079193,
      "logits/rejected": 0.032281383872032166,
      "logps/chosen": -354.4895935058594,
      "logps/rejected": -328.9232177734375,
      "loss": 0.4987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3730226457118988,
      "rewards/margins": 0.8626066446304321,
      "rewards/rejected": -0.48958396911621094,
      "step": 3911
    },
    {
      "epoch": 0.5119162509200949,
      "grad_norm": 6.377630710601807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12828317284584045,
      "logits/rejected": 0.07637969404459,
      "logps/chosen": -372.7618408203125,
      "logps/rejected": -448.8425598144531,
      "loss": 0.5977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04870530962944031,
      "rewards/margins": 0.6117829084396362,
      "rewards/rejected": -0.6604881882667542,
      "step": 3912
    },
    {
      "epoch": 0.5120471088574466,
      "grad_norm": 5.258462905883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018811184912919998,
      "logits/rejected": -0.015047460794448853,
      "logps/chosen": -380.18878173828125,
      "logps/rejected": -389.84100341796875,
      "loss": 0.4171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06346721202135086,
      "rewards/margins": 0.9043012261390686,
      "rewards/rejected": -0.8408340811729431,
      "step": 3913
    },
    {
      "epoch": 0.5121779667947984,
      "grad_norm": 6.573488235473633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025378182530403137,
      "logits/rejected": -0.155218705534935,
      "logps/chosen": -489.8701477050781,
      "logps/rejected": -489.4140625,
      "loss": 0.4835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2818988859653473,
      "rewards/margins": 0.9003738164901733,
      "rewards/rejected": -1.1822726726531982,
      "step": 3914
    },
    {
      "epoch": 0.5123088247321501,
      "grad_norm": 6.062608242034912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13928614556789398,
      "logits/rejected": 0.1616715043783188,
      "logps/chosen": -281.33270263671875,
      "logps/rejected": -340.5302734375,
      "loss": 0.4432,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021768562495708466,
      "rewards/margins": 1.2561299800872803,
      "rewards/rejected": -1.2343614101409912,
      "step": 3915
    },
    {
      "epoch": 0.5124396826695019,
      "grad_norm": 5.6284685134887695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018477536737918854,
      "logits/rejected": -0.0010248608887195587,
      "logps/chosen": -482.78411865234375,
      "logps/rejected": -459.86993408203125,
      "loss": 0.3763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6666765213012695,
      "rewards/margins": 1.0292580127716064,
      "rewards/rejected": -0.36258143186569214,
      "step": 3916
    },
    {
      "epoch": 0.5125705406068537,
      "grad_norm": 7.271169185638428,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13935501873493195,
      "logits/rejected": 0.03007015958428383,
      "logps/chosen": -452.7602233886719,
      "logps/rejected": -546.2323608398438,
      "loss": 0.5635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6294445991516113,
      "rewards/margins": 0.4254043698310852,
      "rewards/rejected": -1.0548489093780518,
      "step": 3917
    },
    {
      "epoch": 0.5127013985442055,
      "grad_norm": 7.519438743591309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06968503445386887,
      "logits/rejected": -0.04803881421685219,
      "logps/chosen": -337.0189208984375,
      "logps/rejected": -378.9295959472656,
      "loss": 0.5662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1763996183872223,
      "rewards/margins": 0.6485733389854431,
      "rewards/rejected": -0.47217369079589844,
      "step": 3918
    },
    {
      "epoch": 0.5128322564815572,
      "grad_norm": 7.782371520996094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30562958121299744,
      "logits/rejected": -0.39129525423049927,
      "logps/chosen": -268.60552978515625,
      "logps/rejected": -337.57501220703125,
      "loss": 0.7806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9307892918586731,
      "rewards/margins": 0.366474449634552,
      "rewards/rejected": -1.2972638607025146,
      "step": 3919
    },
    {
      "epoch": 0.512963114418909,
      "grad_norm": 9.732104301452637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1711564064025879,
      "logits/rejected": -0.21873874962329865,
      "logps/chosen": -329.6796875,
      "logps/rejected": -570.8505859375,
      "loss": 0.7784,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23273125290870667,
      "rewards/margins": 0.35170862078666687,
      "rewards/rejected": -0.5844398736953735,
      "step": 3920
    },
    {
      "epoch": 0.5130939723562608,
      "grad_norm": 4.914346218109131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3225858509540558,
      "logits/rejected": -0.4556625783443451,
      "logps/chosen": -323.9493713378906,
      "logps/rejected": -365.4621276855469,
      "loss": 0.4096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2778669595718384,
      "rewards/margins": 0.8382612466812134,
      "rewards/rejected": -1.1161280870437622,
      "step": 3921
    },
    {
      "epoch": 0.5132248302936125,
      "grad_norm": 4.625301837921143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1342260092496872,
      "logits/rejected": 0.011383526027202606,
      "logps/chosen": -460.9368896484375,
      "logps/rejected": -330.846923828125,
      "loss": 0.3815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3102409839630127,
      "rewards/margins": 1.0367385149002075,
      "rewards/rejected": -0.7264975309371948,
      "step": 3922
    },
    {
      "epoch": 0.5133556882309642,
      "grad_norm": 7.595100402832031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12879976630210876,
      "logits/rejected": 0.05107800289988518,
      "logps/chosen": -500.160400390625,
      "logps/rejected": -444.8232421875,
      "loss": 0.5077,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.289224237203598,
      "rewards/margins": 0.6170974969863892,
      "rewards/rejected": -0.9063217043876648,
      "step": 3923
    },
    {
      "epoch": 0.513486546168316,
      "grad_norm": 6.530997276306152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0795561671257019,
      "logits/rejected": 0.043689556419849396,
      "logps/chosen": -383.35589599609375,
      "logps/rejected": -428.960693359375,
      "loss": 0.4365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2136983871459961,
      "rewards/margins": 1.352770209312439,
      "rewards/rejected": -1.5664689540863037,
      "step": 3924
    },
    {
      "epoch": 0.5136174041056678,
      "grad_norm": 9.705775260925293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07339097559452057,
      "logits/rejected": -0.047875843942165375,
      "logps/chosen": -441.4250183105469,
      "logps/rejected": -374.3927307128906,
      "loss": 0.8526,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8305618166923523,
      "rewards/margins": 0.07417847216129303,
      "rewards/rejected": -0.9047403335571289,
      "step": 3925
    },
    {
      "epoch": 0.5137482620430196,
      "grad_norm": 5.817694664001465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03647126257419586,
      "logits/rejected": 0.07204780727624893,
      "logps/chosen": -442.73681640625,
      "logps/rejected": -448.7718505859375,
      "loss": 0.612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05935915559530258,
      "rewards/margins": 0.7479238510131836,
      "rewards/rejected": -0.6885647177696228,
      "step": 3926
    },
    {
      "epoch": 0.5138791199803713,
      "grad_norm": 3.9318549633026123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12937185168266296,
      "logits/rejected": -0.046035751700401306,
      "logps/chosen": -444.52496337890625,
      "logps/rejected": -344.6493225097656,
      "loss": 0.3294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25811824202537537,
      "rewards/margins": 1.1998217105865479,
      "rewards/rejected": -0.9417034387588501,
      "step": 3927
    },
    {
      "epoch": 0.5140099779177231,
      "grad_norm": 7.292409420013428,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20312263071537018,
      "logits/rejected": -0.15038681030273438,
      "logps/chosen": -380.537109375,
      "logps/rejected": -431.7279357910156,
      "loss": 0.6582,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4644257128238678,
      "rewards/margins": 0.28191259503364563,
      "rewards/rejected": -0.7463383674621582,
      "step": 3928
    },
    {
      "epoch": 0.5141408358550749,
      "grad_norm": 7.541519641876221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06822045892477036,
      "logits/rejected": 0.08266527205705643,
      "logps/chosen": -457.2991943359375,
      "logps/rejected": -504.9281005859375,
      "loss": 0.4434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5132352709770203,
      "rewards/margins": 0.9075338840484619,
      "rewards/rejected": -1.420769214630127,
      "step": 3929
    },
    {
      "epoch": 0.5142716937924265,
      "grad_norm": 6.223341464996338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23323550820350647,
      "logits/rejected": -0.010716922581195831,
      "logps/chosen": -503.69805908203125,
      "logps/rejected": -380.0538024902344,
      "loss": 0.411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35045889019966125,
      "rewards/margins": 1.0559399127960205,
      "rewards/rejected": -0.7054809331893921,
      "step": 3930
    },
    {
      "epoch": 0.5144025517297783,
      "grad_norm": 8.150788307189941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22723111510276794,
      "logits/rejected": -0.028306538239121437,
      "logps/chosen": -482.6776123046875,
      "logps/rejected": -445.7302551269531,
      "loss": 0.6361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28771841526031494,
      "rewards/margins": 0.6084861159324646,
      "rewards/rejected": -0.8962045907974243,
      "step": 3931
    },
    {
      "epoch": 0.5145334096671301,
      "grad_norm": 6.406595706939697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1974021941423416,
      "logits/rejected": 0.08871720731258392,
      "logps/chosen": -514.4134521484375,
      "logps/rejected": -429.7598571777344,
      "loss": 0.5133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07262443006038666,
      "rewards/margins": 0.5659869909286499,
      "rewards/rejected": -0.6386114954948425,
      "step": 3932
    },
    {
      "epoch": 0.5146642676044819,
      "grad_norm": 5.311567783355713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07663751393556595,
      "logits/rejected": 0.14327357709407806,
      "logps/chosen": -311.374267578125,
      "logps/rejected": -388.6029052734375,
      "loss": 0.4072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21681326627731323,
      "rewards/margins": 0.9339914321899414,
      "rewards/rejected": -1.1508046388626099,
      "step": 3933
    },
    {
      "epoch": 0.5147951255418336,
      "grad_norm": 7.003937244415283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027531612664461136,
      "logits/rejected": -0.011067166924476624,
      "logps/chosen": -452.7222900390625,
      "logps/rejected": -365.9896240234375,
      "loss": 0.5474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5996702313423157,
      "rewards/margins": 0.7610052824020386,
      "rewards/rejected": -1.36067533493042,
      "step": 3934
    },
    {
      "epoch": 0.5149259834791854,
      "grad_norm": 4.924116134643555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08425093442201614,
      "logits/rejected": -0.011052867397665977,
      "logps/chosen": -333.0093078613281,
      "logps/rejected": -365.902099609375,
      "loss": 0.4173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1949867457151413,
      "rewards/margins": 1.2593690156936646,
      "rewards/rejected": -1.0643821954727173,
      "step": 3935
    },
    {
      "epoch": 0.5150568414165372,
      "grad_norm": 12.213345527648926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10885412991046906,
      "logits/rejected": -0.08783446997404099,
      "logps/chosen": -862.8838500976562,
      "logps/rejected": -715.9842529296875,
      "loss": 0.7765,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.273877501487732,
      "rewards/margins": 0.7983079552650452,
      "rewards/rejected": -2.072185754776001,
      "step": 3936
    },
    {
      "epoch": 0.515187699353889,
      "grad_norm": 7.507009029388428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0201072096824646,
      "logits/rejected": -0.04563760757446289,
      "logps/chosen": -394.6468505859375,
      "logps/rejected": -638.818115234375,
      "loss": 0.5169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25524675846099854,
      "rewards/margins": 1.5510706901550293,
      "rewards/rejected": -1.8063173294067383,
      "step": 3937
    },
    {
      "epoch": 0.5153185572912407,
      "grad_norm": 6.91778564453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11335200071334839,
      "logits/rejected": -0.025310233235359192,
      "logps/chosen": -388.5705261230469,
      "logps/rejected": -391.8270263671875,
      "loss": 0.4335,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12768670916557312,
      "rewards/margins": 1.0896483659744263,
      "rewards/rejected": -0.9619618058204651,
      "step": 3938
    },
    {
      "epoch": 0.5154494152285924,
      "grad_norm": 6.297643184661865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23239994049072266,
      "logits/rejected": 0.13038842380046844,
      "logps/chosen": -692.5051879882812,
      "logps/rejected": -400.0379638671875,
      "loss": 0.4296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.044640734791755676,
      "rewards/margins": 0.9096121191978455,
      "rewards/rejected": -0.9542527794837952,
      "step": 3939
    },
    {
      "epoch": 0.5155802731659442,
      "grad_norm": 12.817424774169922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14904607832431793,
      "logits/rejected": -0.07178296893835068,
      "logps/chosen": -606.075927734375,
      "logps/rejected": -433.1461181640625,
      "loss": 0.9314,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4536413252353668,
      "rewards/margins": 0.030212990939617157,
      "rewards/rejected": -0.4838542938232422,
      "step": 3940
    },
    {
      "epoch": 0.515711131103296,
      "grad_norm": 7.113108158111572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2859342098236084,
      "logits/rejected": 0.06724383682012558,
      "logps/chosen": -688.8682250976562,
      "logps/rejected": -604.4442138671875,
      "loss": 0.3491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03673878312110901,
      "rewards/margins": 1.3432353734970093,
      "rewards/rejected": -1.306496262550354,
      "step": 3941
    },
    {
      "epoch": 0.5158419890406477,
      "grad_norm": 5.346119403839111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06433666497468948,
      "logits/rejected": -0.041157376021146774,
      "logps/chosen": -394.41741943359375,
      "logps/rejected": -329.29962158203125,
      "loss": 0.4417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4439085125923157,
      "rewards/margins": 0.8328540921211243,
      "rewards/rejected": -1.2767627239227295,
      "step": 3942
    },
    {
      "epoch": 0.5159728469779995,
      "grad_norm": 5.880134105682373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10716655850410461,
      "logits/rejected": -0.08523321151733398,
      "logps/chosen": -304.23980712890625,
      "logps/rejected": -282.5461730957031,
      "loss": 0.5013,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.083094596862793,
      "rewards/margins": 0.5640732049942017,
      "rewards/rejected": -1.647167682647705,
      "step": 3943
    },
    {
      "epoch": 0.5161037049153513,
      "grad_norm": 5.470525741577148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1025155633687973,
      "logits/rejected": 0.21473899483680725,
      "logps/chosen": -241.2449951171875,
      "logps/rejected": -337.2394104003906,
      "loss": 0.4249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32571086287498474,
      "rewards/margins": 0.9045820236206055,
      "rewards/rejected": -1.230292797088623,
      "step": 3944
    },
    {
      "epoch": 0.5162345628527031,
      "grad_norm": 7.204455852508545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06213618442416191,
      "logits/rejected": 0.09263408184051514,
      "logps/chosen": -345.5322265625,
      "logps/rejected": -418.613037109375,
      "loss": 0.4903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04936792701482773,
      "rewards/margins": 0.9209994077682495,
      "rewards/rejected": -0.8716314435005188,
      "step": 3945
    },
    {
      "epoch": 0.5163654207900548,
      "grad_norm": 8.220226287841797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03537147492170334,
      "logits/rejected": -0.01759537309408188,
      "logps/chosen": -414.7420654296875,
      "logps/rejected": -370.21575927734375,
      "loss": 0.7195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12986624240875244,
      "rewards/margins": 0.601386308670044,
      "rewards/rejected": -0.7312524318695068,
      "step": 3946
    },
    {
      "epoch": 0.5164962787274066,
      "grad_norm": 8.924498558044434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23030756413936615,
      "logits/rejected": 0.1381216198205948,
      "logps/chosen": -500.2237854003906,
      "logps/rejected": -466.8005065917969,
      "loss": 0.5583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16278745234012604,
      "rewards/margins": 0.934107780456543,
      "rewards/rejected": -1.0968952178955078,
      "step": 3947
    },
    {
      "epoch": 0.5166271366647583,
      "grad_norm": 8.362846374511719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17794686555862427,
      "logits/rejected": -0.034174539148807526,
      "logps/chosen": -529.3894653320312,
      "logps/rejected": -473.2281188964844,
      "loss": 0.5776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3252097964286804,
      "rewards/margins": 1.2462719678878784,
      "rewards/rejected": -1.571481704711914,
      "step": 3948
    },
    {
      "epoch": 0.5167579946021101,
      "grad_norm": 5.983473777770996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1224021166563034,
      "logits/rejected": -0.15382786095142365,
      "logps/chosen": -351.6263122558594,
      "logps/rejected": -343.3177490234375,
      "loss": 0.474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8923529982566833,
      "rewards/margins": 0.7845001220703125,
      "rewards/rejected": -1.676853060722351,
      "step": 3949
    },
    {
      "epoch": 0.5168888525394618,
      "grad_norm": 12.013175010681152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2778591513633728,
      "logits/rejected": 0.13119500875473022,
      "logps/chosen": -460.83050537109375,
      "logps/rejected": -457.638427734375,
      "loss": 0.7326,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2782226800918579,
      "rewards/margins": 0.1923067420721054,
      "rewards/rejected": -0.47052937746047974,
      "step": 3950
    },
    {
      "epoch": 0.5170197104768136,
      "grad_norm": 7.020506858825684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10686636716127396,
      "logits/rejected": -0.026297010481357574,
      "logps/chosen": -392.28973388671875,
      "logps/rejected": -342.584716796875,
      "loss": 0.5531,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6720619797706604,
      "rewards/margins": 0.46412622928619385,
      "rewards/rejected": -1.1361881494522095,
      "step": 3951
    },
    {
      "epoch": 0.5171505684141654,
      "grad_norm": 5.591020107269287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15732601284980774,
      "logits/rejected": -0.17095214128494263,
      "logps/chosen": -378.1058044433594,
      "logps/rejected": -290.63580322265625,
      "loss": 0.4039,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2459976226091385,
      "rewards/margins": 1.184801697731018,
      "rewards/rejected": -0.9388041496276855,
      "step": 3952
    },
    {
      "epoch": 0.5172814263515171,
      "grad_norm": 8.76465892791748,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07849340885877609,
      "logits/rejected": -0.05193484574556351,
      "logps/chosen": -447.45947265625,
      "logps/rejected": -438.77496337890625,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0880504846572876,
      "rewards/margins": 0.16170328855514526,
      "rewards/rejected": -0.07365284115076065,
      "step": 3953
    },
    {
      "epoch": 0.5174122842888689,
      "grad_norm": 8.375192642211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1237860918045044,
      "logits/rejected": -0.17465844750404358,
      "logps/chosen": -482.37603759765625,
      "logps/rejected": -501.0916442871094,
      "loss": 0.4702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28918570280075073,
      "rewards/margins": 1.1261701583862305,
      "rewards/rejected": -1.415355920791626,
      "step": 3954
    },
    {
      "epoch": 0.5175431422262207,
      "grad_norm": 7.536647319793701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009986383840441704,
      "logits/rejected": -0.2158658504486084,
      "logps/chosen": -773.9750366210938,
      "logps/rejected": -713.8306274414062,
      "loss": 0.3771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.249224454164505,
      "rewards/margins": 1.7211719751358032,
      "rewards/rejected": -1.9703962802886963,
      "step": 3955
    },
    {
      "epoch": 0.5176740001635725,
      "grad_norm": 10.837820053100586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05904591456055641,
      "logits/rejected": 0.01863127574324608,
      "logps/chosen": -336.16705322265625,
      "logps/rejected": -335.04559326171875,
      "loss": 1.0859,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1659024953842163,
      "rewards/margins": -0.2917167842388153,
      "rewards/rejected": -0.8741857409477234,
      "step": 3956
    },
    {
      "epoch": 0.5178048581009241,
      "grad_norm": 9.18129825592041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15653638541698456,
      "logits/rejected": -0.10153249651193619,
      "logps/chosen": -426.5252990722656,
      "logps/rejected": -331.5987548828125,
      "loss": 0.6779,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3464805483818054,
      "rewards/margins": 0.5686620473861694,
      "rewards/rejected": -0.9151425957679749,
      "step": 3957
    },
    {
      "epoch": 0.5179357160382759,
      "grad_norm": 8.651657104492188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12384749948978424,
      "logits/rejected": -0.05176449194550514,
      "logps/chosen": -498.4656677246094,
      "logps/rejected": -410.9400634765625,
      "loss": 0.7042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19146519899368286,
      "rewards/margins": 0.4032360017299652,
      "rewards/rejected": -0.5947012305259705,
      "step": 3958
    },
    {
      "epoch": 0.5180665739756277,
      "grad_norm": 9.481913566589355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.126665398478508,
      "logits/rejected": 0.034128155559301376,
      "logps/chosen": -451.4614562988281,
      "logps/rejected": -338.18157958984375,
      "loss": 0.6164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003911018371582031,
      "rewards/margins": 0.8018474578857422,
      "rewards/rejected": -0.8057584762573242,
      "step": 3959
    },
    {
      "epoch": 0.5181974319129795,
      "grad_norm": 6.77030086517334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02114778757095337,
      "logits/rejected": 0.16149771213531494,
      "logps/chosen": -416.18353271484375,
      "logps/rejected": -566.9768676757812,
      "loss": 0.4809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019991876557469368,
      "rewards/margins": 1.053567886352539,
      "rewards/rejected": -1.0335760116577148,
      "step": 3960
    },
    {
      "epoch": 0.5183282898503312,
      "grad_norm": 12.234280586242676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02500690519809723,
      "logits/rejected": 0.0739494115114212,
      "logps/chosen": -485.55120849609375,
      "logps/rejected": -524.560546875,
      "loss": 1.0628,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.547325074672699,
      "rewards/margins": -0.462299644947052,
      "rewards/rejected": -0.08502551913261414,
      "step": 3961
    },
    {
      "epoch": 0.518459147787683,
      "grad_norm": 8.617932319641113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1220329999923706,
      "logits/rejected": 0.1137569472193718,
      "logps/chosen": -505.5519714355469,
      "logps/rejected": -565.3890991210938,
      "loss": 0.7961,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.020941536873579025,
      "rewards/margins": 0.6904808282852173,
      "rewards/rejected": -0.7114223837852478,
      "step": 3962
    },
    {
      "epoch": 0.5185900057250348,
      "grad_norm": 6.812308311462402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24445851147174835,
      "logits/rejected": -0.18949922919273376,
      "logps/chosen": -373.7320251464844,
      "logps/rejected": -362.41473388671875,
      "loss": 0.441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5896397829055786,
      "rewards/margins": 0.9304256439208984,
      "rewards/rejected": -1.5200653076171875,
      "step": 3963
    },
    {
      "epoch": 0.5187208636623866,
      "grad_norm": 6.468463897705078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20868685841560364,
      "logits/rejected": -0.27513769268989563,
      "logps/chosen": -838.7112426757812,
      "logps/rejected": -633.0830688476562,
      "loss": 0.3074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2788839340209961,
      "rewards/margins": 1.4381479024887085,
      "rewards/rejected": -1.1592639684677124,
      "step": 3964
    },
    {
      "epoch": 0.5188517215997382,
      "grad_norm": 23.231998443603516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2689525783061981,
      "logits/rejected": -0.16550377011299133,
      "logps/chosen": -950.4761962890625,
      "logps/rejected": -364.7172546386719,
      "loss": 1.3962,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3523802757263184,
      "rewards/margins": -0.7047324776649475,
      "rewards/rejected": -0.6476476788520813,
      "step": 3965
    },
    {
      "epoch": 0.51898257953709,
      "grad_norm": 8.135236740112305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05420941486954689,
      "logits/rejected": -0.14360949397087097,
      "logps/chosen": -456.63299560546875,
      "logps/rejected": -530.596435546875,
      "loss": 0.4625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28514596819877625,
      "rewards/margins": 1.2745115756988525,
      "rewards/rejected": -0.9893655776977539,
      "step": 3966
    },
    {
      "epoch": 0.5191134374744418,
      "grad_norm": 7.292454719543457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07385275512933731,
      "logits/rejected": -0.007874462753534317,
      "logps/chosen": -458.61004638671875,
      "logps/rejected": -489.17578125,
      "loss": 0.5213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4163261651992798,
      "rewards/margins": 1.024233102798462,
      "rewards/rejected": -1.4405591487884521,
      "step": 3967
    },
    {
      "epoch": 0.5192442954117936,
      "grad_norm": 8.942955017089844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07653802633285522,
      "logits/rejected": -0.14440320432186127,
      "logps/chosen": -340.139404296875,
      "logps/rejected": -436.1186828613281,
      "loss": 0.7848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11438215523958206,
      "rewards/margins": 0.7280889749526978,
      "rewards/rejected": -0.8424710035324097,
      "step": 3968
    },
    {
      "epoch": 0.5193751533491453,
      "grad_norm": 7.18812894821167,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04526948183774948,
      "logits/rejected": -0.13087798655033112,
      "logps/chosen": -480.73516845703125,
      "logps/rejected": -576.0106201171875,
      "loss": 0.5881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6587282419204712,
      "rewards/margins": 0.5384702682495117,
      "rewards/rejected": -1.1971986293792725,
      "step": 3969
    },
    {
      "epoch": 0.5195060112864971,
      "grad_norm": 8.574028968811035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09907569736242294,
      "logits/rejected": 0.20168758928775787,
      "logps/chosen": -532.6265258789062,
      "logps/rejected": -566.4205932617188,
      "loss": 0.8289,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24384060502052307,
      "rewards/margins": 0.11427725851535797,
      "rewards/rejected": -0.35811784863471985,
      "step": 3970
    },
    {
      "epoch": 0.5196368692238489,
      "grad_norm": 5.062802314758301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021473169326782227,
      "logits/rejected": 0.17545169591903687,
      "logps/chosen": -347.10064697265625,
      "logps/rejected": -320.4217224121094,
      "loss": 0.537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02362290769815445,
      "rewards/margins": 0.8069429993629456,
      "rewards/rejected": -0.830565869808197,
      "step": 3971
    },
    {
      "epoch": 0.5197677271612007,
      "grad_norm": 7.60056734085083,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03150060772895813,
      "logits/rejected": -0.02605310082435608,
      "logps/chosen": -395.005615234375,
      "logps/rejected": -312.0280456542969,
      "loss": 0.6953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21196526288986206,
      "rewards/margins": 0.43391868472099304,
      "rewards/rejected": -0.6458839774131775,
      "step": 3972
    },
    {
      "epoch": 0.5198985850985524,
      "grad_norm": 7.963038921356201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.058100782334804535,
      "logits/rejected": 0.12459838390350342,
      "logps/chosen": -570.2479248046875,
      "logps/rejected": -600.9970092773438,
      "loss": 0.6951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.053529735654592514,
      "rewards/margins": 0.7926696538925171,
      "rewards/rejected": -0.739139974117279,
      "step": 3973
    },
    {
      "epoch": 0.5200294430359041,
      "grad_norm": 10.181180953979492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13881248235702515,
      "logits/rejected": -0.04788465052843094,
      "logps/chosen": -511.4057922363281,
      "logps/rejected": -374.79156494140625,
      "loss": 0.7055,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14571113884449005,
      "rewards/margins": 0.33066824078559875,
      "rewards/rejected": -0.47637939453125,
      "step": 3974
    },
    {
      "epoch": 0.5201603009732559,
      "grad_norm": 6.936094760894775,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09710390865802765,
      "logits/rejected": 0.22321951389312744,
      "logps/chosen": -511.6304931640625,
      "logps/rejected": -434.4503173828125,
      "loss": 0.5391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10553798079490662,
      "rewards/margins": 0.5738235712051392,
      "rewards/rejected": -0.6793615818023682,
      "step": 3975
    },
    {
      "epoch": 0.5202911589106076,
      "grad_norm": 5.338468551635742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0831759050488472,
      "logits/rejected": -0.12203393876552582,
      "logps/chosen": -391.8908386230469,
      "logps/rejected": -410.4106140136719,
      "loss": 0.4687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30942249298095703,
      "rewards/margins": 0.817604660987854,
      "rewards/rejected": -1.127027153968811,
      "step": 3976
    },
    {
      "epoch": 0.5204220168479594,
      "grad_norm": 8.059516906738281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15633414685726166,
      "logits/rejected": -0.04223378002643585,
      "logps/chosen": -436.7762756347656,
      "logps/rejected": -487.6896057128906,
      "loss": 0.7078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35779210925102234,
      "rewards/margins": 0.5179899334907532,
      "rewards/rejected": -0.8757820725440979,
      "step": 3977
    },
    {
      "epoch": 0.5205528747853112,
      "grad_norm": 11.116769790649414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2788146138191223,
      "logits/rejected": -0.5388547778129578,
      "logps/chosen": -414.93603515625,
      "logps/rejected": -404.95721435546875,
      "loss": 0.7577,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.998688817024231,
      "rewards/margins": 0.6523866653442383,
      "rewards/rejected": -1.6510756015777588,
      "step": 3978
    },
    {
      "epoch": 0.520683732722663,
      "grad_norm": 6.072272300720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04593118280172348,
      "logits/rejected": -0.26009976863861084,
      "logps/chosen": -511.3309020996094,
      "logps/rejected": -370.69873046875,
      "loss": 0.4831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10332602262496948,
      "rewards/margins": 0.7052175402641296,
      "rewards/rejected": -0.8085435628890991,
      "step": 3979
    },
    {
      "epoch": 0.5208145906600147,
      "grad_norm": 13.177166938781738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2875480651855469,
      "logits/rejected": 0.1314411461353302,
      "logps/chosen": -554.2197875976562,
      "logps/rejected": -481.0134582519531,
      "loss": 0.6148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1660329103469849,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -1.9980640411376953,
      "step": 3980
    },
    {
      "epoch": 0.5209454485973665,
      "grad_norm": 9.116774559020996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06779604405164719,
      "logits/rejected": 0.021998250856995583,
      "logps/chosen": -322.4963073730469,
      "logps/rejected": -580.565673828125,
      "loss": 0.6049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9884184002876282,
      "rewards/margins": 0.9102061986923218,
      "rewards/rejected": -1.8986246585845947,
      "step": 3981
    },
    {
      "epoch": 0.5210763065347183,
      "grad_norm": 8.769682884216309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3196064233779907,
      "logits/rejected": -0.06826996058225632,
      "logps/chosen": -600.1097412109375,
      "logps/rejected": -398.3440246582031,
      "loss": 0.5994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6680431365966797,
      "rewards/margins": 0.40369129180908203,
      "rewards/rejected": -1.0717344284057617,
      "step": 3982
    },
    {
      "epoch": 0.52120716447207,
      "grad_norm": 8.217375755310059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07843822985887527,
      "logits/rejected": 0.09314187616109848,
      "logps/chosen": -555.3128662109375,
      "logps/rejected": -533.6302490234375,
      "loss": 0.6336,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.36772918701171875,
      "rewards/margins": 0.3875257968902588,
      "rewards/rejected": -0.7552550435066223,
      "step": 3983
    },
    {
      "epoch": 0.5213380224094217,
      "grad_norm": 9.049598693847656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09363877773284912,
      "logits/rejected": 0.076777383685112,
      "logps/chosen": -615.0503540039062,
      "logps/rejected": -447.10162353515625,
      "loss": 0.6156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3265388309955597,
      "rewards/margins": 0.7445360422134399,
      "rewards/rejected": -1.0710748434066772,
      "step": 3984
    },
    {
      "epoch": 0.5214688803467735,
      "grad_norm": 6.9805402755737305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11015508323907852,
      "logits/rejected": -0.2184123992919922,
      "logps/chosen": -344.39105224609375,
      "logps/rejected": -387.7012939453125,
      "loss": 0.6611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15626317262649536,
      "rewards/margins": 0.7177495956420898,
      "rewards/rejected": -0.5614864230155945,
      "step": 3985
    },
    {
      "epoch": 0.5215997382841253,
      "grad_norm": 4.787172317504883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07721208035945892,
      "logits/rejected": -0.07115399837493896,
      "logps/chosen": -627.6802978515625,
      "logps/rejected": -460.4831848144531,
      "loss": 0.3775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3065914511680603,
      "rewards/margins": 1.2408318519592285,
      "rewards/rejected": -0.9342405200004578,
      "step": 3986
    },
    {
      "epoch": 0.5217305962214771,
      "grad_norm": 6.651206016540527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10629405081272125,
      "logits/rejected": 0.06857191026210785,
      "logps/chosen": -591.1427612304688,
      "logps/rejected": -550.2325439453125,
      "loss": 0.4838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0393487885594368,
      "rewards/margins": 1.1384539604187012,
      "rewards/rejected": -1.1778028011322021,
      "step": 3987
    },
    {
      "epoch": 0.5218614541588288,
      "grad_norm": 9.856256484985352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10961519181728363,
      "logits/rejected": 0.06642716377973557,
      "logps/chosen": -534.0835571289062,
      "logps/rejected": -562.2257080078125,
      "loss": 0.6972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006253063678741455,
      "rewards/margins": 0.5621436834335327,
      "rewards/rejected": -0.5683967471122742,
      "step": 3988
    },
    {
      "epoch": 0.5219923120961806,
      "grad_norm": 9.282024383544922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1302407681941986,
      "logits/rejected": 0.11831101775169373,
      "logps/chosen": -474.41339111328125,
      "logps/rejected": -467.0511169433594,
      "loss": 0.6767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35802915692329407,
      "rewards/margins": 0.35380974411964417,
      "rewards/rejected": -0.7118389010429382,
      "step": 3989
    },
    {
      "epoch": 0.5221231700335324,
      "grad_norm": 7.562792778015137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2934282720088959,
      "logits/rejected": 0.3723214268684387,
      "logps/chosen": -536.3599243164062,
      "logps/rejected": -498.4983825683594,
      "loss": 0.5123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4031868278980255,
      "rewards/margins": 1.1208077669143677,
      "rewards/rejected": -0.7176210284233093,
      "step": 3990
    },
    {
      "epoch": 0.5222540279708842,
      "grad_norm": 9.217364311218262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15985675156116486,
      "logits/rejected": 0.2469845414161682,
      "logps/chosen": -495.545166015625,
      "logps/rejected": -506.38238525390625,
      "loss": 0.7919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4682201147079468,
      "rewards/margins": 0.055709488689899445,
      "rewards/rejected": -0.5239295959472656,
      "step": 3991
    },
    {
      "epoch": 0.5223848859082358,
      "grad_norm": 6.518014430999756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25327983498573303,
      "logits/rejected": 0.028215736150741577,
      "logps/chosen": -472.8162841796875,
      "logps/rejected": -659.5718383789062,
      "loss": 0.4625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.015848936513066292,
      "rewards/margins": 1.172239065170288,
      "rewards/rejected": -1.188088059425354,
      "step": 3992
    },
    {
      "epoch": 0.5225157438455876,
      "grad_norm": 7.328805923461914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08584155142307281,
      "logits/rejected": 0.10722528398036957,
      "logps/chosen": -304.2840881347656,
      "logps/rejected": -373.68939208984375,
      "loss": 0.6148,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24021092057228088,
      "rewards/margins": 0.3777569830417633,
      "rewards/rejected": -0.13754606246948242,
      "step": 3993
    },
    {
      "epoch": 0.5226466017829394,
      "grad_norm": 4.437521457672119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1798948049545288,
      "logits/rejected": 0.3031827211380005,
      "logps/chosen": -398.0152282714844,
      "logps/rejected": -356.59747314453125,
      "loss": 0.2932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3134514093399048,
      "rewards/margins": 1.5837997198104858,
      "rewards/rejected": -1.2703484296798706,
      "step": 3994
    },
    {
      "epoch": 0.5227774597202911,
      "grad_norm": 8.24556827545166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21375629305839539,
      "logits/rejected": 0.290044367313385,
      "logps/chosen": -441.6778869628906,
      "logps/rejected": -499.6434020996094,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3673042356967926,
      "rewards/margins": 0.12666533887386322,
      "rewards/rejected": 0.24063891172409058,
      "step": 3995
    },
    {
      "epoch": 0.5229083176576429,
      "grad_norm": 4.46319580078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19042299687862396,
      "logits/rejected": -0.4537513256072998,
      "logps/chosen": -571.053466796875,
      "logps/rejected": -474.49591064453125,
      "loss": 0.3164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3628759980201721,
      "rewards/margins": 1.3089661598205566,
      "rewards/rejected": -0.9460903406143188,
      "step": 3996
    },
    {
      "epoch": 0.5230391755949947,
      "grad_norm": 11.125692367553711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07896676659584045,
      "logits/rejected": -0.14871875941753387,
      "logps/chosen": -459.8673095703125,
      "logps/rejected": -531.5961303710938,
      "loss": 0.9178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4152508080005646,
      "rewards/margins": 1.0523253679275513,
      "rewards/rejected": -1.4675761461257935,
      "step": 3997
    },
    {
      "epoch": 0.5231700335323465,
      "grad_norm": 4.889538288116455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3020506203174591,
      "logits/rejected": 0.19831188023090363,
      "logps/chosen": -460.49676513671875,
      "logps/rejected": -425.8544921875,
      "loss": 0.3259,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.060424596071243286,
      "rewards/margins": 1.3124276399612427,
      "rewards/rejected": -1.3728523254394531,
      "step": 3998
    },
    {
      "epoch": 0.5233008914696982,
      "grad_norm": 9.345829963684082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2225552797317505,
      "logits/rejected": -0.08046215772628784,
      "logps/chosen": -392.07379150390625,
      "logps/rejected": -387.4979248046875,
      "loss": 0.6663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4081173241138458,
      "rewards/margins": 0.593599259853363,
      "rewards/rejected": -1.0017166137695312,
      "step": 3999
    },
    {
      "epoch": 0.52343174940705,
      "grad_norm": 4.43065071105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037614524364471436,
      "logits/rejected": 0.09074211120605469,
      "logps/chosen": -497.0627136230469,
      "logps/rejected": -389.798095703125,
      "loss": 0.2826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12693217396736145,
      "rewards/margins": 1.3546174764633179,
      "rewards/rejected": -1.2276853322982788,
      "step": 4000
    },
    {
      "epoch": 0.5235626073444017,
      "grad_norm": 9.5242338180542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27001455426216125,
      "logits/rejected": 0.10540315508842468,
      "logps/chosen": -396.3360595703125,
      "logps/rejected": -529.3840942382812,
      "loss": 0.7898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0017849355936050415,
      "rewards/margins": 0.5422236919403076,
      "rewards/rejected": -0.5404388904571533,
      "step": 4001
    },
    {
      "epoch": 0.5236934652817535,
      "grad_norm": 5.963055610656738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0019111790461465716,
      "logits/rejected": 0.08375042676925659,
      "logps/chosen": -406.034912109375,
      "logps/rejected": -531.0979614257812,
      "loss": 0.4237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4796135127544403,
      "rewards/margins": 0.9719609022140503,
      "rewards/rejected": -1.4515743255615234,
      "step": 4002
    },
    {
      "epoch": 0.5238243232191052,
      "grad_norm": 6.867508888244629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20508509874343872,
      "logits/rejected": 0.1705968677997589,
      "logps/chosen": -460.8042297363281,
      "logps/rejected": -416.3355712890625,
      "loss": 0.4937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3748852014541626,
      "rewards/margins": 1.346164584159851,
      "rewards/rejected": -0.9712793231010437,
      "step": 4003
    },
    {
      "epoch": 0.523955181156457,
      "grad_norm": 6.918466567993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23094597458839417,
      "logits/rejected": -0.19553957879543304,
      "logps/chosen": -459.2167663574219,
      "logps/rejected": -432.2181701660156,
      "loss": 0.6121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11030884087085724,
      "rewards/margins": 0.47770026326179504,
      "rewards/rejected": -0.3673914074897766,
      "step": 4004
    },
    {
      "epoch": 0.5240860390938088,
      "grad_norm": 5.948727130889893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0958782285451889,
      "logits/rejected": -0.23645353317260742,
      "logps/chosen": -551.2451171875,
      "logps/rejected": -627.8349609375,
      "loss": 0.4713,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14404812455177307,
      "rewards/margins": 0.789134681224823,
      "rewards/rejected": -0.9331827163696289,
      "step": 4005
    },
    {
      "epoch": 0.5242168970311606,
      "grad_norm": 7.610803604125977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29504749178886414,
      "logits/rejected": -0.3295343816280365,
      "logps/chosen": -548.1421508789062,
      "logps/rejected": -413.1068115234375,
      "loss": 0.6724,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1104152649641037,
      "rewards/margins": 0.25563928484916687,
      "rewards/rejected": -0.3660545349121094,
      "step": 4006
    },
    {
      "epoch": 0.5243477549685123,
      "grad_norm": 9.768004417419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16291287541389465,
      "logits/rejected": 0.12792961299419403,
      "logps/chosen": -442.0555419921875,
      "logps/rejected": -469.764404296875,
      "loss": 0.802,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20379717648029327,
      "rewards/margins": 0.48188722133636475,
      "rewards/rejected": -0.27809008955955505,
      "step": 4007
    },
    {
      "epoch": 0.5244786129058641,
      "grad_norm": 5.973780632019043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10164067894220352,
      "logits/rejected": 0.08595087379217148,
      "logps/chosen": -390.8092041015625,
      "logps/rejected": -542.3200073242188,
      "loss": 0.4089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4264249801635742,
      "rewards/margins": 1.0191209316253662,
      "rewards/rejected": -1.44554603099823,
      "step": 4008
    },
    {
      "epoch": 0.5246094708432159,
      "grad_norm": 7.82408332824707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04445875436067581,
      "logits/rejected": -0.12098876386880875,
      "logps/chosen": -610.9336547851562,
      "logps/rejected": -556.9862060546875,
      "loss": 0.6245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7256388068199158,
      "rewards/margins": 0.9594089984893799,
      "rewards/rejected": -0.2337701916694641,
      "step": 4009
    },
    {
      "epoch": 0.5247403287805676,
      "grad_norm": 7.237621784210205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18061695992946625,
      "logits/rejected": 0.33384692668914795,
      "logps/chosen": -315.29150390625,
      "logps/rejected": -396.989013671875,
      "loss": 0.6283,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25770360231399536,
      "rewards/margins": 0.3569008409976959,
      "rewards/rejected": -0.6146044135093689,
      "step": 4010
    },
    {
      "epoch": 0.5248711867179193,
      "grad_norm": 6.566856861114502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22972311079502106,
      "logits/rejected": -0.5002700090408325,
      "logps/chosen": -519.2891845703125,
      "logps/rejected": -499.4530029296875,
      "loss": 0.5252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18532583117485046,
      "rewards/margins": 1.1998025178909302,
      "rewards/rejected": -1.0144767761230469,
      "step": 4011
    },
    {
      "epoch": 0.5250020446552711,
      "grad_norm": 8.786784172058105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14785026013851166,
      "logits/rejected": -0.3244922161102295,
      "logps/chosen": -444.33477783203125,
      "logps/rejected": -444.0309753417969,
      "loss": 0.8832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1716318130493164,
      "rewards/margins": 0.6405718326568604,
      "rewards/rejected": -0.812203586101532,
      "step": 4012
    },
    {
      "epoch": 0.5251329025926229,
      "grad_norm": 8.860549926757812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07461684197187424,
      "logits/rejected": -0.08655239641666412,
      "logps/chosen": -431.5525207519531,
      "logps/rejected": -463.4372863769531,
      "loss": 0.7433,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17071594297885895,
      "rewards/margins": 0.24141843616962433,
      "rewards/rejected": -0.4121343493461609,
      "step": 4013
    },
    {
      "epoch": 0.5252637605299747,
      "grad_norm": 5.810158729553223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2779444754123688,
      "logits/rejected": -0.4580722153186798,
      "logps/chosen": -520.057373046875,
      "logps/rejected": -450.2669372558594,
      "loss": 0.3806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4856773018836975,
      "rewards/margins": 1.1283196210861206,
      "rewards/rejected": -0.6426422595977783,
      "step": 4014
    },
    {
      "epoch": 0.5253946184673264,
      "grad_norm": 6.485047817230225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005757823586463928,
      "logits/rejected": 0.025192417204380035,
      "logps/chosen": -569.59033203125,
      "logps/rejected": -562.97802734375,
      "loss": 0.3855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5950281023979187,
      "rewards/margins": 1.0636228322982788,
      "rewards/rejected": -0.4685947597026825,
      "step": 4015
    },
    {
      "epoch": 0.5255254764046782,
      "grad_norm": 5.8429388999938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06198578327894211,
      "logits/rejected": -0.13206623494625092,
      "logps/chosen": -371.1278381347656,
      "logps/rejected": -326.6773681640625,
      "loss": 0.3956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38362637162208557,
      "rewards/margins": 0.9118768572807312,
      "rewards/rejected": -1.2955032587051392,
      "step": 4016
    },
    {
      "epoch": 0.52565633434203,
      "grad_norm": 9.720330238342285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13551545143127441,
      "logits/rejected": 0.05790059268474579,
      "logps/chosen": -502.8227233886719,
      "logps/rejected": -550.2972412109375,
      "loss": 0.6666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.051963627338409424,
      "rewards/margins": 0.45636749267578125,
      "rewards/rejected": -0.4044038951396942,
      "step": 4017
    },
    {
      "epoch": 0.5257871922793816,
      "grad_norm": 6.196717262268066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09428105503320694,
      "logits/rejected": 0.16159717738628387,
      "logps/chosen": -577.5313720703125,
      "logps/rejected": -573.2308959960938,
      "loss": 0.4129,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00046617351472377777,
      "rewards/margins": 0.9845567345619202,
      "rewards/rejected": -0.9850229024887085,
      "step": 4018
    },
    {
      "epoch": 0.5259180502167334,
      "grad_norm": 7.066431045532227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06026101112365723,
      "logits/rejected": -0.19829393923282623,
      "logps/chosen": -350.6497497558594,
      "logps/rejected": -242.8857421875,
      "loss": 0.6667,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.682378351688385,
      "rewards/margins": 0.17716675996780396,
      "rewards/rejected": -0.8595451712608337,
      "step": 4019
    },
    {
      "epoch": 0.5260489081540852,
      "grad_norm": 8.972099304199219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12087704986333847,
      "logits/rejected": -0.011086894199252129,
      "logps/chosen": -546.4940185546875,
      "logps/rejected": -616.6629028320312,
      "loss": 0.6802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18712691962718964,
      "rewards/margins": 0.4179079532623291,
      "rewards/rejected": -0.6050348877906799,
      "step": 4020
    },
    {
      "epoch": 0.526179766091437,
      "grad_norm": 3.426069498062134,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37789386510849,
      "logits/rejected": 0.08113028109073639,
      "logps/chosen": -452.7088317871094,
      "logps/rejected": -344.5194396972656,
      "loss": 0.1924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6111957430839539,
      "rewards/margins": 2.067194938659668,
      "rewards/rejected": -1.4559991359710693,
      "step": 4021
    },
    {
      "epoch": 0.5263106240287887,
      "grad_norm": 10.951324462890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23065254092216492,
      "logits/rejected": 0.17601445317268372,
      "logps/chosen": -479.00811767578125,
      "logps/rejected": -512.8197021484375,
      "loss": 1.0429,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12817327678203583,
      "rewards/margins": -0.25012245774269104,
      "rewards/rejected": 0.1219491958618164,
      "step": 4022
    },
    {
      "epoch": 0.5264414819661405,
      "grad_norm": 8.342342376708984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19025492668151855,
      "logits/rejected": 0.15785355865955353,
      "logps/chosen": -416.9457702636719,
      "logps/rejected": -463.18927001953125,
      "loss": 0.4511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3514343500137329,
      "rewards/margins": 1.041558861732483,
      "rewards/rejected": -0.69012451171875,
      "step": 4023
    },
    {
      "epoch": 0.5265723399034923,
      "grad_norm": 7.937889099121094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09101852774620056,
      "logits/rejected": 0.14161911606788635,
      "logps/chosen": -659.3258666992188,
      "logps/rejected": -728.0236206054688,
      "loss": 0.4013,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18660524487495422,
      "rewards/margins": 0.8381021022796631,
      "rewards/rejected": -1.0247074365615845,
      "step": 4024
    },
    {
      "epoch": 0.5267031978408441,
      "grad_norm": 4.374515533447266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24169829487800598,
      "logits/rejected": 0.24711841344833374,
      "logps/chosen": -544.2547607421875,
      "logps/rejected": -427.9283142089844,
      "loss": 0.2678,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3336639404296875,
      "rewards/margins": 1.5382957458496094,
      "rewards/rejected": -1.2046319246292114,
      "step": 4025
    },
    {
      "epoch": 0.5268340557781958,
      "grad_norm": 8.266079902648926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07412898540496826,
      "logits/rejected": -0.0004980135709047318,
      "logps/chosen": -531.0277099609375,
      "logps/rejected": -569.7692260742188,
      "loss": 0.7089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.053242094814777374,
      "rewards/margins": 0.640791118144989,
      "rewards/rejected": -0.5875490307807922,
      "step": 4026
    },
    {
      "epoch": 0.5269649137155475,
      "grad_norm": 5.225355625152588,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12386444211006165,
      "logits/rejected": 0.005240589380264282,
      "logps/chosen": -441.8896179199219,
      "logps/rejected": -431.5251770019531,
      "loss": 0.3863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39021608233451843,
      "rewards/margins": 1.0702295303344727,
      "rewards/rejected": -1.4604456424713135,
      "step": 4027
    },
    {
      "epoch": 0.5270957716528993,
      "grad_norm": 8.095986366271973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06860105693340302,
      "logits/rejected": 0.21066416800022125,
      "logps/chosen": -451.828125,
      "logps/rejected": -416.2135009765625,
      "loss": 0.6199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3201858699321747,
      "rewards/margins": 0.6948565244674683,
      "rewards/rejected": -0.3746705949306488,
      "step": 4028
    },
    {
      "epoch": 0.5272266295902511,
      "grad_norm": 6.485457897186279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25465890765190125,
      "logits/rejected": 0.37325355410575867,
      "logps/chosen": -635.39453125,
      "logps/rejected": -513.9095458984375,
      "loss": 0.551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12405796349048615,
      "rewards/margins": 0.9325705170631409,
      "rewards/rejected": -1.0566284656524658,
      "step": 4029
    },
    {
      "epoch": 0.5273574875276028,
      "grad_norm": 6.735620498657227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0052415356040000916,
      "logits/rejected": -0.18480730056762695,
      "logps/chosen": -266.2172546386719,
      "logps/rejected": -483.20013427734375,
      "loss": 0.575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0860191285610199,
      "rewards/margins": 0.6318320035934448,
      "rewards/rejected": -0.7178511023521423,
      "step": 4030
    },
    {
      "epoch": 0.5274883454649546,
      "grad_norm": 5.764782428741455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2763543128967285,
      "logits/rejected": 0.1665704846382141,
      "logps/chosen": -444.2283020019531,
      "logps/rejected": -340.628173828125,
      "loss": 0.3579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7204803824424744,
      "rewards/margins": 0.9315958619117737,
      "rewards/rejected": -0.21111544966697693,
      "step": 4031
    },
    {
      "epoch": 0.5276192034023064,
      "grad_norm": 9.35166072845459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15606477856636047,
      "logits/rejected": 0.29945120215415955,
      "logps/chosen": -589.2882080078125,
      "logps/rejected": -597.5565795898438,
      "loss": 0.7759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20983678102493286,
      "rewards/margins": 0.3333287835121155,
      "rewards/rejected": -0.5431655645370483,
      "step": 4032
    },
    {
      "epoch": 0.5277500613396582,
      "grad_norm": 8.666297912597656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2870897650718689,
      "logits/rejected": 0.18229365348815918,
      "logps/chosen": -437.4504699707031,
      "logps/rejected": -305.5751647949219,
      "loss": 0.7017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008968710899353027,
      "rewards/margins": 0.6762176752090454,
      "rewards/rejected": -0.6672489047050476,
      "step": 4033
    },
    {
      "epoch": 0.5278809192770099,
      "grad_norm": 10.028203964233398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3305610418319702,
      "logits/rejected": 0.18685904145240784,
      "logps/chosen": -306.93170166015625,
      "logps/rejected": -342.609375,
      "loss": 0.7609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08805007487535477,
      "rewards/margins": 0.32385486364364624,
      "rewards/rejected": -0.411904901266098,
      "step": 4034
    },
    {
      "epoch": 0.5280117772143617,
      "grad_norm": 7.130601406097412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08305875211954117,
      "logits/rejected": -0.06684268265962601,
      "logps/chosen": -441.9286804199219,
      "logps/rejected": -479.131103515625,
      "loss": 0.3964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5029655694961548,
      "rewards/margins": 0.9950157403945923,
      "rewards/rejected": -1.4979811906814575,
      "step": 4035
    },
    {
      "epoch": 0.5281426351517134,
      "grad_norm": 9.18037223815918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.158395454287529,
      "logits/rejected": 0.20480231940746307,
      "logps/chosen": -365.66912841796875,
      "logps/rejected": -462.096435546875,
      "loss": 0.7587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34970131516456604,
      "rewards/margins": 0.04522436857223511,
      "rewards/rejected": -0.39492568373680115,
      "step": 4036
    },
    {
      "epoch": 0.5282734930890652,
      "grad_norm": 4.424439907073975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15239517390727997,
      "logits/rejected": 0.18216566741466522,
      "logps/chosen": -397.61749267578125,
      "logps/rejected": -418.180419921875,
      "loss": 0.2802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2605126202106476,
      "rewards/margins": 1.69309401512146,
      "rewards/rejected": -1.4325815439224243,
      "step": 4037
    },
    {
      "epoch": 0.5284043510264169,
      "grad_norm": 8.800400733947754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23050907254219055,
      "logits/rejected": 0.17388969659805298,
      "logps/chosen": -373.0267639160156,
      "logps/rejected": -288.8460388183594,
      "loss": 0.4706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13393059372901917,
      "rewards/margins": 0.7279055714607239,
      "rewards/rejected": -0.8618361353874207,
      "step": 4038
    },
    {
      "epoch": 0.5285352089637687,
      "grad_norm": 9.633666038513184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3866034150123596,
      "logits/rejected": 0.026263916864991188,
      "logps/chosen": -466.6746520996094,
      "logps/rejected": -425.3505859375,
      "loss": 0.7534,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06401805579662323,
      "rewards/margins": 0.22654743492603302,
      "rewards/rejected": -0.29056549072265625,
      "step": 4039
    },
    {
      "epoch": 0.5286660669011205,
      "grad_norm": 8.232136726379395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0828949362039566,
      "logits/rejected": 0.22932438552379608,
      "logps/chosen": -688.9428100585938,
      "logps/rejected": -678.9909057617188,
      "loss": 0.7247,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3988708555698395,
      "rewards/margins": 0.24257126450538635,
      "rewards/rejected": 0.15629959106445312,
      "step": 4040
    },
    {
      "epoch": 0.5287969248384722,
      "grad_norm": 5.221860408782959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0417032428085804,
      "logits/rejected": 0.046870097517967224,
      "logps/chosen": -258.6596374511719,
      "logps/rejected": -273.811279296875,
      "loss": 0.5116,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02433585375547409,
      "rewards/margins": 0.8562285304069519,
      "rewards/rejected": -0.8318926095962524,
      "step": 4041
    },
    {
      "epoch": 0.528927782775824,
      "grad_norm": 8.776744842529297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2484123855829239,
      "logits/rejected": 0.06399485468864441,
      "logps/chosen": -611.6065673828125,
      "logps/rejected": -424.4477844238281,
      "loss": 0.4216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0780327096581459,
      "rewards/margins": 1.2501846551895142,
      "rewards/rejected": -1.172152042388916,
      "step": 4042
    },
    {
      "epoch": 0.5290586407131758,
      "grad_norm": 9.110895156860352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12544333934783936,
      "logits/rejected": 0.2530701458454132,
      "logps/chosen": -508.35943603515625,
      "logps/rejected": -472.3162841796875,
      "loss": 0.8502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8854997158050537,
      "rewards/margins": 0.15572413802146912,
      "rewards/rejected": -1.0412238836288452,
      "step": 4043
    },
    {
      "epoch": 0.5291894986505276,
      "grad_norm": 7.881999492645264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10090550035238266,
      "logits/rejected": -0.24550557136535645,
      "logps/chosen": -529.4449462890625,
      "logps/rejected": -563.3504638671875,
      "loss": 0.5263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29460448026657104,
      "rewards/margins": 0.5111027956008911,
      "rewards/rejected": -0.21649837493896484,
      "step": 4044
    },
    {
      "epoch": 0.5293203565878792,
      "grad_norm": 8.233038902282715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22025232017040253,
      "logits/rejected": -0.01065070927143097,
      "logps/chosen": -572.12451171875,
      "logps/rejected": -512.42822265625,
      "loss": 0.39,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3381125032901764,
      "rewards/margins": 1.1294147968292236,
      "rewards/rejected": -1.4675272703170776,
      "step": 4045
    },
    {
      "epoch": 0.529451214525231,
      "grad_norm": 4.317324161529541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2488785982131958,
      "logits/rejected": -0.066778264939785,
      "logps/chosen": -443.4012451171875,
      "logps/rejected": -597.3299560546875,
      "loss": 0.3138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14620763063430786,
      "rewards/margins": 1.2905011177062988,
      "rewards/rejected": -1.436708688735962,
      "step": 4046
    },
    {
      "epoch": 0.5295820724625828,
      "grad_norm": 8.078804016113281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08329307287931442,
      "logits/rejected": -0.17132586240768433,
      "logps/chosen": -585.2877807617188,
      "logps/rejected": -619.1696166992188,
      "loss": 0.4928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8335323929786682,
      "rewards/margins": 0.9861933588981628,
      "rewards/rejected": -1.819725751876831,
      "step": 4047
    },
    {
      "epoch": 0.5297129303999346,
      "grad_norm": 7.4670491218566895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07740156352519989,
      "logits/rejected": -0.1847505271434784,
      "logps/chosen": -689.5392456054688,
      "logps/rejected": -506.3824462890625,
      "loss": 0.4556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35385704040527344,
      "rewards/margins": 1.0942238569259644,
      "rewards/rejected": -1.4480810165405273,
      "step": 4048
    },
    {
      "epoch": 0.5298437883372863,
      "grad_norm": 5.960665702819824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06678833812475204,
      "logits/rejected": -0.026966333389282227,
      "logps/chosen": -564.7167358398438,
      "logps/rejected": -391.6473083496094,
      "loss": 0.3235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.4825347661972046,
      "rewards/margins": 1.6249858140945435,
      "rewards/rejected": -0.14245089888572693,
      "step": 4049
    },
    {
      "epoch": 0.5299746462746381,
      "grad_norm": 6.200805187225342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04869270324707031,
      "logits/rejected": 0.13260626792907715,
      "logps/chosen": -391.3543395996094,
      "logps/rejected": -361.64776611328125,
      "loss": 0.623,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3635837733745575,
      "rewards/margins": 0.41065460443496704,
      "rewards/rejected": -0.7742384076118469,
      "step": 4050
    },
    {
      "epoch": 0.5301055042119899,
      "grad_norm": 13.629607200622559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08985470235347748,
      "logits/rejected": 0.010030403733253479,
      "logps/chosen": -592.3383178710938,
      "logps/rejected": -354.7164306640625,
      "loss": 1.1136,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9988248348236084,
      "rewards/margins": -0.2129124402999878,
      "rewards/rejected": -0.7859125733375549,
      "step": 4051
    },
    {
      "epoch": 0.5302363621493417,
      "grad_norm": 8.541414260864258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009771343320608139,
      "logits/rejected": -0.10270270705223083,
      "logps/chosen": -439.132080078125,
      "logps/rejected": -450.3875427246094,
      "loss": 0.8246,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34819433093070984,
      "rewards/margins": -0.07979796081781387,
      "rewards/rejected": -0.2683963477611542,
      "step": 4052
    },
    {
      "epoch": 0.5303672200866933,
      "grad_norm": 6.6679511070251465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05187579244375229,
      "logits/rejected": -0.005401907488703728,
      "logps/chosen": -385.3368835449219,
      "logps/rejected": -326.38641357421875,
      "loss": 0.4916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18654970824718475,
      "rewards/margins": 0.9490870237350464,
      "rewards/rejected": -0.76253741979599,
      "step": 4053
    },
    {
      "epoch": 0.5304980780240451,
      "grad_norm": 13.32484245300293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020864054560661316,
      "logits/rejected": -0.1015608087182045,
      "logps/chosen": -407.2744445800781,
      "logps/rejected": -477.37371826171875,
      "loss": 0.8111,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7061381936073303,
      "rewards/margins": 0.17795085906982422,
      "rewards/rejected": -0.8840891122817993,
      "step": 4054
    },
    {
      "epoch": 0.5306289359613969,
      "grad_norm": 5.536850929260254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2748289704322815,
      "logits/rejected": 0.1729688048362732,
      "logps/chosen": -444.4342041015625,
      "logps/rejected": -454.5400390625,
      "loss": 0.2886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9745504260063171,
      "rewards/margins": 1.3900866508483887,
      "rewards/rejected": -0.41553616523742676,
      "step": 4055
    },
    {
      "epoch": 0.5307597938987487,
      "grad_norm": 10.650622367858887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10244949162006378,
      "logits/rejected": -0.1937028467655182,
      "logps/chosen": -543.6082153320312,
      "logps/rejected": -495.43499755859375,
      "loss": 0.7859,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2230137139558792,
      "rewards/margins": -0.09947795420885086,
      "rewards/rejected": -0.12353572249412537,
      "step": 4056
    },
    {
      "epoch": 0.5308906518361004,
      "grad_norm": 6.138617038726807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1936277598142624,
      "logits/rejected": -0.1668168157339096,
      "logps/chosen": -494.4559631347656,
      "logps/rejected": -432.69110107421875,
      "loss": 0.5654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2085437923669815,
      "rewards/margins": 0.7015312314033508,
      "rewards/rejected": -0.9100748896598816,
      "step": 4057
    },
    {
      "epoch": 0.5310215097734522,
      "grad_norm": 7.182518005371094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14159086346626282,
      "logits/rejected": -0.12142810970544815,
      "logps/chosen": -542.1633911132812,
      "logps/rejected": -658.0610961914062,
      "loss": 0.4394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08909054845571518,
      "rewards/margins": 1.1646037101745605,
      "rewards/rejected": -1.0755130052566528,
      "step": 4058
    },
    {
      "epoch": 0.531152367710804,
      "grad_norm": 6.622463226318359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0054678842425346375,
      "logits/rejected": 0.26804181933403015,
      "logps/chosen": -420.2951354980469,
      "logps/rejected": -492.2767333984375,
      "loss": 0.6608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.54482501745224,
      "rewards/margins": 0.5422199368476868,
      "rewards/rejected": -1.0870449542999268,
      "step": 4059
    },
    {
      "epoch": 0.5312832256481557,
      "grad_norm": 4.006893634796143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028210368007421494,
      "logits/rejected": -0.11752405762672424,
      "logps/chosen": -545.3682861328125,
      "logps/rejected": -431.2640380859375,
      "loss": 0.3345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43148139119148254,
      "rewards/margins": 1.2350518703460693,
      "rewards/rejected": -1.666533350944519,
      "step": 4060
    },
    {
      "epoch": 0.5314140835855075,
      "grad_norm": 6.365230560302734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2603575587272644,
      "logits/rejected": 0.2735167443752289,
      "logps/chosen": -429.6379089355469,
      "logps/rejected": -401.9610595703125,
      "loss": 0.4438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04307440668344498,
      "rewards/margins": 0.7486826181411743,
      "rewards/rejected": -0.7056081891059875,
      "step": 4061
    },
    {
      "epoch": 0.5315449415228592,
      "grad_norm": 4.19665002822876,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06705905497074127,
      "logits/rejected": 0.10188280045986176,
      "logps/chosen": -565.5685424804688,
      "logps/rejected": -426.72332763671875,
      "loss": 0.2426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.046959854662418365,
      "rewards/margins": 1.3854799270629883,
      "rewards/rejected": -1.4324398040771484,
      "step": 4062
    },
    {
      "epoch": 0.531675799460211,
      "grad_norm": 8.73235034942627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15569573640823364,
      "logits/rejected": 0.18706335127353668,
      "logps/chosen": -480.20611572265625,
      "logps/rejected": -469.74163818359375,
      "loss": 0.7466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5862094759941101,
      "rewards/margins": 0.3550488352775574,
      "rewards/rejected": -0.9412583112716675,
      "step": 4063
    },
    {
      "epoch": 0.5318066573975627,
      "grad_norm": 8.89962100982666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2612132728099823,
      "logits/rejected": 0.378945529460907,
      "logps/chosen": -362.24639892578125,
      "logps/rejected": -349.867919921875,
      "loss": 0.8003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6450567245483398,
      "rewards/margins": -0.026395387947559357,
      "rewards/rejected": -0.6186612844467163,
      "step": 4064
    },
    {
      "epoch": 0.5319375153349145,
      "grad_norm": 5.156415939331055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12482011318206787,
      "logits/rejected": 0.21594282984733582,
      "logps/chosen": -446.4936218261719,
      "logps/rejected": -495.31890869140625,
      "loss": 0.3555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16611041128635406,
      "rewards/margins": 1.2051602602005005,
      "rewards/rejected": -1.371270775794983,
      "step": 4065
    },
    {
      "epoch": 0.5320683732722663,
      "grad_norm": 8.753931999206543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13701830804347992,
      "logits/rejected": -0.24848994612693787,
      "logps/chosen": -489.9908142089844,
      "logps/rejected": -388.34234619140625,
      "loss": 0.5256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.609179675579071,
      "rewards/margins": 0.6272581219673157,
      "rewards/rejected": -1.2364377975463867,
      "step": 4066
    },
    {
      "epoch": 0.5321992312096181,
      "grad_norm": 9.221651077270508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0029211672954261303,
      "logits/rejected": -0.13101646304130554,
      "logps/chosen": -639.5909423828125,
      "logps/rejected": -535.4281005859375,
      "loss": 0.4765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.054423727095127106,
      "rewards/margins": 1.1528958082199097,
      "rewards/rejected": -1.2073196172714233,
      "step": 4067
    },
    {
      "epoch": 0.5323300891469698,
      "grad_norm": 3.1285245418548584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01341371238231659,
      "logits/rejected": 0.1251632273197174,
      "logps/chosen": -521.2562255859375,
      "logps/rejected": -434.1268310546875,
      "loss": 0.2491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5679121613502502,
      "rewards/margins": 1.8277746438980103,
      "rewards/rejected": -1.2598625421524048,
      "step": 4068
    },
    {
      "epoch": 0.5324609470843216,
      "grad_norm": 7.490180969238281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11511149257421494,
      "logits/rejected": 0.07220005989074707,
      "logps/chosen": -411.0904235839844,
      "logps/rejected": -434.82855224609375,
      "loss": 0.579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8243463635444641,
      "rewards/margins": 0.5466514825820923,
      "rewards/rejected": -1.3709979057312012,
      "step": 4069
    },
    {
      "epoch": 0.5325918050216734,
      "grad_norm": 7.534322738647461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26557719707489014,
      "logits/rejected": 0.10493077337741852,
      "logps/chosen": -357.91461181640625,
      "logps/rejected": -277.9149169921875,
      "loss": 0.4689,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27991431951522827,
      "rewards/margins": 0.9978125691413879,
      "rewards/rejected": -1.2777268886566162,
      "step": 4070
    },
    {
      "epoch": 0.5327226629590252,
      "grad_norm": 7.590730667114258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003279969096183777,
      "logits/rejected": 0.08278468251228333,
      "logps/chosen": -556.6553955078125,
      "logps/rejected": -624.9810791015625,
      "loss": 0.6413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.050400346517562866,
      "rewards/margins": 0.4005129337310791,
      "rewards/rejected": -0.35011255741119385,
      "step": 4071
    },
    {
      "epoch": 0.5328535208963768,
      "grad_norm": 8.442882537841797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08968054503202438,
      "logits/rejected": -0.022136636078357697,
      "logps/chosen": -648.1829223632812,
      "logps/rejected": -554.9063110351562,
      "loss": 0.6403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1552223116159439,
      "rewards/margins": 0.3927226662635803,
      "rewards/rejected": -0.2375003695487976,
      "step": 4072
    },
    {
      "epoch": 0.5329843788337286,
      "grad_norm": 6.558647155761719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2082851529121399,
      "logits/rejected": 0.2705993354320526,
      "logps/chosen": -504.36688232421875,
      "logps/rejected": -578.7752685546875,
      "loss": 0.6418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0927097350358963,
      "rewards/margins": 0.6886001825332642,
      "rewards/rejected": -0.7813099026679993,
      "step": 4073
    },
    {
      "epoch": 0.5331152367710804,
      "grad_norm": 8.62764835357666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1694951355457306,
      "logits/rejected": -0.1119893342256546,
      "logps/chosen": -357.05133056640625,
      "logps/rejected": -482.2313537597656,
      "loss": 0.8953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17742404341697693,
      "rewards/margins": -0.12172506749629974,
      "rewards/rejected": -0.055698931217193604,
      "step": 4074
    },
    {
      "epoch": 0.5332460947084322,
      "grad_norm": 9.32921028137207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060909707099199295,
      "logits/rejected": -0.08816315978765488,
      "logps/chosen": -343.88311767578125,
      "logps/rejected": -251.14068603515625,
      "loss": 0.8232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7835837602615356,
      "rewards/margins": 0.27599868178367615,
      "rewards/rejected": -1.0595823526382446,
      "step": 4075
    },
    {
      "epoch": 0.5333769526457839,
      "grad_norm": 9.20068645477295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0010514520108699799,
      "logits/rejected": 0.03197607025504112,
      "logps/chosen": -525.507568359375,
      "logps/rejected": -606.62451171875,
      "loss": 0.6124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.03985595703125,
      "rewards/margins": 0.30182725191116333,
      "rewards/rejected": -1.3416831493377686,
      "step": 4076
    },
    {
      "epoch": 0.5335078105831357,
      "grad_norm": 6.32391881942749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040562644600868225,
      "logits/rejected": -0.08048483729362488,
      "logps/chosen": -680.3616943359375,
      "logps/rejected": -392.6192932128906,
      "loss": 0.5583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5068285465240479,
      "rewards/margins": 1.1657251119613647,
      "rewards/rejected": -0.6588966846466064,
      "step": 4077
    },
    {
      "epoch": 0.5336386685204875,
      "grad_norm": 8.876477241516113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0011482089757919312,
      "logits/rejected": 0.10404382646083832,
      "logps/chosen": -420.75469970703125,
      "logps/rejected": -498.576416015625,
      "loss": 0.6529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3260273337364197,
      "rewards/margins": 0.19085730612277985,
      "rewards/rejected": -0.5168846249580383,
      "step": 4078
    },
    {
      "epoch": 0.5337695264578393,
      "grad_norm": 5.8213324546813965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10145257413387299,
      "logits/rejected": 0.038287222385406494,
      "logps/chosen": -500.0852355957031,
      "logps/rejected": -454.3224792480469,
      "loss": 0.4995,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2890968322753906,
      "rewards/margins": 0.6577152609825134,
      "rewards/rejected": -0.9468120336532593,
      "step": 4079
    },
    {
      "epoch": 0.5339003843951909,
      "grad_norm": 8.121923446655273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09060049802064896,
      "logits/rejected": 0.04846132546663284,
      "logps/chosen": -510.0050964355469,
      "logps/rejected": -661.6058959960938,
      "loss": 0.4862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.46735936403274536,
      "rewards/margins": 1.0813558101654053,
      "rewards/rejected": -0.6139965057373047,
      "step": 4080
    },
    {
      "epoch": 0.5340312423325427,
      "grad_norm": 5.393446445465088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09566745162010193,
      "logits/rejected": -0.0317872017621994,
      "logps/chosen": -288.96087646484375,
      "logps/rejected": -322.3176574707031,
      "loss": 0.4383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1674955189228058,
      "rewards/margins": 1.140830397605896,
      "rewards/rejected": -1.308326005935669,
      "step": 4081
    },
    {
      "epoch": 0.5341621002698945,
      "grad_norm": 6.862785339355469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34407082200050354,
      "logits/rejected": 0.16706246137619019,
      "logps/chosen": -454.40789794921875,
      "logps/rejected": -497.5823974609375,
      "loss": 0.4491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5664941668510437,
      "rewards/margins": 0.817319929599762,
      "rewards/rejected": -1.3838140964508057,
      "step": 4082
    },
    {
      "epoch": 0.5342929582072462,
      "grad_norm": 10.250139236450195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.041242510080337524,
      "logits/rejected": 0.026192009449005127,
      "logps/chosen": -542.97119140625,
      "logps/rejected": -472.146240234375,
      "loss": 0.566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17707519233226776,
      "rewards/margins": 0.6188013553619385,
      "rewards/rejected": -0.7958765625953674,
      "step": 4083
    },
    {
      "epoch": 0.534423816144598,
      "grad_norm": 3.7872698307037354,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16839823126792908,
      "logits/rejected": 0.09349125623703003,
      "logps/chosen": -758.8602905273438,
      "logps/rejected": -742.6021728515625,
      "loss": 0.1867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4101793169975281,
      "rewards/margins": 2.1365914344787598,
      "rewards/rejected": -1.726412296295166,
      "step": 4084
    },
    {
      "epoch": 0.5345546740819498,
      "grad_norm": 8.286765098571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13359852135181427,
      "logits/rejected": -0.3925953507423401,
      "logps/chosen": -469.5865478515625,
      "logps/rejected": -641.6478271484375,
      "loss": 0.5518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5229211449623108,
      "rewards/margins": 0.6936092376708984,
      "rewards/rejected": -1.216530442237854,
      "step": 4085
    },
    {
      "epoch": 0.5346855320193016,
      "grad_norm": 8.558491706848145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31471043825149536,
      "logits/rejected": 0.2792849838733673,
      "logps/chosen": -336.7552490234375,
      "logps/rejected": -364.22186279296875,
      "loss": 0.7738,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24906063079833984,
      "rewards/margins": 0.2903503477573395,
      "rewards/rejected": -0.04128970950841904,
      "step": 4086
    },
    {
      "epoch": 0.5348163899566533,
      "grad_norm": 6.999720573425293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5098339319229126,
      "logits/rejected": 0.5024735331535339,
      "logps/chosen": -562.756591796875,
      "logps/rejected": -441.7775573730469,
      "loss": 0.3783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24272537231445312,
      "rewards/margins": 1.4253817796707153,
      "rewards/rejected": -1.1826565265655518,
      "step": 4087
    },
    {
      "epoch": 0.534947247894005,
      "grad_norm": 5.994884490966797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2555221617221832,
      "logits/rejected": 0.2270805537700653,
      "logps/chosen": -756.017578125,
      "logps/rejected": -549.0606689453125,
      "loss": 0.4141,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.103179931640625,
      "rewards/margins": 1.1522806882858276,
      "rewards/rejected": -0.04910075664520264,
      "step": 4088
    },
    {
      "epoch": 0.5350781058313568,
      "grad_norm": 5.400518417358398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38750961422920227,
      "logits/rejected": 0.30311548709869385,
      "logps/chosen": -339.35638427734375,
      "logps/rejected": -256.121337890625,
      "loss": 0.4479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.701214611530304,
      "rewards/margins": 1.25419282913208,
      "rewards/rejected": -0.5529781579971313,
      "step": 4089
    },
    {
      "epoch": 0.5352089637687086,
      "grad_norm": 4.8143768310546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1542203575372696,
      "logits/rejected": 0.00904962420463562,
      "logps/chosen": -332.41278076171875,
      "logps/rejected": -301.5425720214844,
      "loss": 0.4782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3368730843067169,
      "rewards/margins": 0.813325822353363,
      "rewards/rejected": -1.1501989364624023,
      "step": 4090
    },
    {
      "epoch": 0.5353398217060603,
      "grad_norm": 9.03353214263916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022562874481081963,
      "logits/rejected": 0.14422497153282166,
      "logps/chosen": -497.2037048339844,
      "logps/rejected": -512.4998168945312,
      "loss": 0.5037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06784896552562714,
      "rewards/margins": 0.8594160079956055,
      "rewards/rejected": -0.7915670275688171,
      "step": 4091
    },
    {
      "epoch": 0.5354706796434121,
      "grad_norm": 7.269810676574707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18234577775001526,
      "logits/rejected": 0.0679878443479538,
      "logps/chosen": -484.46209716796875,
      "logps/rejected": -441.1544189453125,
      "loss": 0.5451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.23733465373516083,
      "rewards/margins": 0.7850514650344849,
      "rewards/rejected": -0.5477169156074524,
      "step": 4092
    },
    {
      "epoch": 0.5356015375807639,
      "grad_norm": 6.474686622619629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5151897072792053,
      "logits/rejected": 0.2199259102344513,
      "logps/chosen": -396.79583740234375,
      "logps/rejected": -376.61395263671875,
      "loss": 0.4965,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20686417818069458,
      "rewards/margins": 0.5489917993545532,
      "rewards/rejected": -0.34212762117385864,
      "step": 4093
    },
    {
      "epoch": 0.5357323955181157,
      "grad_norm": 7.243160724639893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39630788564682007,
      "logits/rejected": 0.2965598702430725,
      "logps/chosen": -382.52728271484375,
      "logps/rejected": -404.2137451171875,
      "loss": 0.6213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32103231549263,
      "rewards/margins": 0.6855378746986389,
      "rewards/rejected": -1.0065703392028809,
      "step": 4094
    },
    {
      "epoch": 0.5358632534554674,
      "grad_norm": 7.91137170791626,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19416791200637817,
      "logits/rejected": 0.14563514292240143,
      "logps/chosen": -351.3129577636719,
      "logps/rejected": -439.4901428222656,
      "loss": 0.6569,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23890173435211182,
      "rewards/margins": 0.29905569553375244,
      "rewards/rejected": -0.5379573702812195,
      "step": 4095
    },
    {
      "epoch": 0.5359941113928192,
      "grad_norm": 7.114429473876953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10400094836950302,
      "logits/rejected": 0.13132965564727783,
      "logps/chosen": -387.87835693359375,
      "logps/rejected": -612.200927734375,
      "loss": 0.4754,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4574126899242401,
      "rewards/margins": 1.1763603687286377,
      "rewards/rejected": -1.6337729692459106,
      "step": 4096
    },
    {
      "epoch": 0.536124969330171,
      "grad_norm": 3.6064844131469727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4712876081466675,
      "logits/rejected": 0.0456731840968132,
      "logps/chosen": -507.9774169921875,
      "logps/rejected": -416.37469482421875,
      "loss": 0.2322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34289589524269104,
      "rewards/margins": 1.477168321609497,
      "rewards/rejected": -1.134272575378418,
      "step": 4097
    },
    {
      "epoch": 0.5362558272675227,
      "grad_norm": 4.975959300994873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2488814890384674,
      "logits/rejected": 0.11233659088611603,
      "logps/chosen": -532.2151489257812,
      "logps/rejected": -615.428955078125,
      "loss": 0.2863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1198333501815796,
      "rewards/margins": 1.6371629238128662,
      "rewards/rejected": -0.5173294544219971,
      "step": 4098
    },
    {
      "epoch": 0.5363866852048744,
      "grad_norm": 11.706917762756348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003221653401851654,
      "logits/rejected": 0.11425245553255081,
      "logps/chosen": -455.830322265625,
      "logps/rejected": -380.92071533203125,
      "loss": 0.4895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5480269193649292,
      "rewards/margins": 1.3806111812591553,
      "rewards/rejected": -1.9286381006240845,
      "step": 4099
    },
    {
      "epoch": 0.5365175431422262,
      "grad_norm": 7.427364826202393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11765368282794952,
      "logits/rejected": 0.0728546679019928,
      "logps/chosen": -481.0947570800781,
      "logps/rejected": -537.3947143554688,
      "loss": 0.6204,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.38482344150543213,
      "rewards/margins": 0.30330145359039307,
      "rewards/rejected": 0.08152198791503906,
      "step": 4100
    },
    {
      "epoch": 0.536648401079578,
      "grad_norm": 6.257697105407715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1466667205095291,
      "logits/rejected": 0.11177211999893188,
      "logps/chosen": -598.0692749023438,
      "logps/rejected": -597.4248046875,
      "loss": 0.3859,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19510002434253693,
      "rewards/margins": 1.2985080480575562,
      "rewards/rejected": -1.1034079790115356,
      "step": 4101
    },
    {
      "epoch": 0.5367792590169298,
      "grad_norm": 9.610864639282227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05446641147136688,
      "logits/rejected": 0.16517585515975952,
      "logps/chosen": -549.0543212890625,
      "logps/rejected": -366.23089599609375,
      "loss": 0.643,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06321335583925247,
      "rewards/margins": 0.904118537902832,
      "rewards/rejected": -0.9673318862915039,
      "step": 4102
    },
    {
      "epoch": 0.5369101169542815,
      "grad_norm": 5.231910705566406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3044060170650482,
      "logits/rejected": 0.13457955420017242,
      "logps/chosen": -539.4122924804688,
      "logps/rejected": -340.5953369140625,
      "loss": 0.3367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.804934561252594,
      "rewards/margins": 2.4139480590820312,
      "rewards/rejected": -1.6090137958526611,
      "step": 4103
    },
    {
      "epoch": 0.5370409748916333,
      "grad_norm": 5.228001117706299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3752480745315552,
      "logits/rejected": 0.35929572582244873,
      "logps/chosen": -413.6296081542969,
      "logps/rejected": -452.42578125,
      "loss": 0.4543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014848709106445312,
      "rewards/margins": 0.7298287153244019,
      "rewards/rejected": -0.7149799466133118,
      "step": 4104
    },
    {
      "epoch": 0.5371718328289851,
      "grad_norm": 3.8365375995635986,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15133166313171387,
      "logits/rejected": 0.15170201659202576,
      "logps/chosen": -520.702392578125,
      "logps/rejected": -520.2836303710938,
      "loss": 0.2014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14123228192329407,
      "rewards/margins": 1.7364356517791748,
      "rewards/rejected": -1.8776679039001465,
      "step": 4105
    },
    {
      "epoch": 0.5373026907663367,
      "grad_norm": 4.087429046630859,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43846455216407776,
      "logits/rejected": 0.43345075845718384,
      "logps/chosen": -343.7646179199219,
      "logps/rejected": -353.1029052734375,
      "loss": 0.3688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17286279797554016,
      "rewards/margins": 1.0577198266983032,
      "rewards/rejected": -1.2305827140808105,
      "step": 4106
    },
    {
      "epoch": 0.5374335487036885,
      "grad_norm": 6.190799236297607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09561692178249359,
      "logits/rejected": -0.1724083423614502,
      "logps/chosen": -412.1396484375,
      "logps/rejected": -428.0650329589844,
      "loss": 0.6092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2528131604194641,
      "rewards/margins": 0.9665511250495911,
      "rewards/rejected": -0.7137379050254822,
      "step": 4107
    },
    {
      "epoch": 0.5375644066410403,
      "grad_norm": 6.698362350463867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09385426342487335,
      "logits/rejected": -0.02631456032395363,
      "logps/chosen": -452.73699951171875,
      "logps/rejected": -370.49981689453125,
      "loss": 0.581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43792426586151123,
      "rewards/margins": 0.6617664098739624,
      "rewards/rejected": -1.0996906757354736,
      "step": 4108
    },
    {
      "epoch": 0.5376952645783921,
      "grad_norm": 9.048669815063477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.6191563606262207,
      "logits/rejected": 0.3309535086154938,
      "logps/chosen": -577.1454467773438,
      "logps/rejected": -541.3922119140625,
      "loss": 0.5733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0996847078204155,
      "rewards/margins": 0.6398521661758423,
      "rewards/rejected": -0.5401674509048462,
      "step": 4109
    },
    {
      "epoch": 0.5378261225157438,
      "grad_norm": 5.314155578613281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4287305176258087,
      "logits/rejected": 0.22529985010623932,
      "logps/chosen": -405.31158447265625,
      "logps/rejected": -385.7847900390625,
      "loss": 0.3632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6045045256614685,
      "rewards/margins": 1.1638033390045166,
      "rewards/rejected": -1.7683079242706299,
      "step": 4110
    },
    {
      "epoch": 0.5379569804530956,
      "grad_norm": 5.929203510284424,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11705373227596283,
      "logits/rejected": -0.017672255635261536,
      "logps/chosen": -402.982177734375,
      "logps/rejected": -364.6373291015625,
      "loss": 0.5321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19560794532299042,
      "rewards/margins": 1.0057928562164307,
      "rewards/rejected": -1.201400637626648,
      "step": 4111
    },
    {
      "epoch": 0.5380878383904474,
      "grad_norm": 7.080294609069824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053785666823387146,
      "logits/rejected": -0.061148203909397125,
      "logps/chosen": -393.489990234375,
      "logps/rejected": -348.5603942871094,
      "loss": 0.5004,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31432241201400757,
      "rewards/margins": 0.9400955438613892,
      "rewards/rejected": -1.2544180154800415,
      "step": 4112
    },
    {
      "epoch": 0.5382186963277992,
      "grad_norm": 7.460629463195801,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10098957270383835,
      "logits/rejected": 0.07910072803497314,
      "logps/chosen": -493.7586975097656,
      "logps/rejected": -535.28271484375,
      "loss": 0.4444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23274247348308563,
      "rewards/margins": 0.962714672088623,
      "rewards/rejected": -1.195457100868225,
      "step": 4113
    },
    {
      "epoch": 0.5383495542651509,
      "grad_norm": 8.923087120056152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03545049950480461,
      "logits/rejected": -0.009547337889671326,
      "logps/chosen": -508.9688720703125,
      "logps/rejected": -402.3642883300781,
      "loss": 0.7822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9054261445999146,
      "rewards/margins": 0.19221189618110657,
      "rewards/rejected": -1.0976380109786987,
      "step": 4114
    },
    {
      "epoch": 0.5384804122025026,
      "grad_norm": 7.375777244567871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3139096200466156,
      "logits/rejected": 0.08079242706298828,
      "logps/chosen": -520.0699462890625,
      "logps/rejected": -637.0352783203125,
      "loss": 0.3873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2875358462333679,
      "rewards/margins": 1.3261690139770508,
      "rewards/rejected": -1.613704800605774,
      "step": 4115
    },
    {
      "epoch": 0.5386112701398544,
      "grad_norm": 5.946732521057129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5583796501159668,
      "logits/rejected": 0.3862883448600769,
      "logps/chosen": -352.5166320800781,
      "logps/rejected": -317.2437744140625,
      "loss": 0.3988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11082762479782104,
      "rewards/margins": 1.2533293962478638,
      "rewards/rejected": -1.1425018310546875,
      "step": 4116
    },
    {
      "epoch": 0.5387421280772062,
      "grad_norm": 10.339164733886719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2325173020362854,
      "logits/rejected": 0.19095107913017273,
      "logps/chosen": -502.832275390625,
      "logps/rejected": -551.2452392578125,
      "loss": 0.7702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35358506441116333,
      "rewards/margins": 0.4361213743686676,
      "rewards/rejected": -0.7897064089775085,
      "step": 4117
    },
    {
      "epoch": 0.5388729860145579,
      "grad_norm": 3.8560078144073486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12838736176490784,
      "logits/rejected": -0.05717812478542328,
      "logps/chosen": -585.009033203125,
      "logps/rejected": -494.0899353027344,
      "loss": 0.281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9950382113456726,
      "rewards/margins": 1.6028871536254883,
      "rewards/rejected": -0.6078487634658813,
      "step": 4118
    },
    {
      "epoch": 0.5390038439519097,
      "grad_norm": 5.680768966674805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13220366835594177,
      "logits/rejected": -0.0797906219959259,
      "logps/chosen": -419.86370849609375,
      "logps/rejected": -378.9706726074219,
      "loss": 0.5032,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13085328042507172,
      "rewards/margins": 0.7568211555480957,
      "rewards/rejected": -0.625967800617218,
      "step": 4119
    },
    {
      "epoch": 0.5391347018892615,
      "grad_norm": 5.702802658081055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2298315167427063,
      "logits/rejected": 0.11954911053180695,
      "logps/chosen": -466.3118896484375,
      "logps/rejected": -595.741943359375,
      "loss": 0.382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08624975383281708,
      "rewards/margins": 1.5243805646896362,
      "rewards/rejected": -1.4381309747695923,
      "step": 4120
    },
    {
      "epoch": 0.5392655598266133,
      "grad_norm": 7.89821195602417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08819873631000519,
      "logits/rejected": -0.13838762044906616,
      "logps/chosen": -525.7612915039062,
      "logps/rejected": -587.28271484375,
      "loss": 0.4384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11889608949422836,
      "rewards/margins": 1.1737254858016968,
      "rewards/rejected": -1.054829478263855,
      "step": 4121
    },
    {
      "epoch": 0.539396417763965,
      "grad_norm": 5.0944905281066895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1296626478433609,
      "logits/rejected": 0.12364774942398071,
      "logps/chosen": -490.28253173828125,
      "logps/rejected": -355.8598327636719,
      "loss": 0.4272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3476886749267578,
      "rewards/margins": 1.2860809564590454,
      "rewards/rejected": -1.6337697505950928,
      "step": 4122
    },
    {
      "epoch": 0.5395272757013168,
      "grad_norm": 7.60693883895874,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23498128354549408,
      "logits/rejected": -0.04104907810688019,
      "logps/chosen": -393.2745361328125,
      "logps/rejected": -491.0629577636719,
      "loss": 0.3643,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.48591652512550354,
      "rewards/margins": 1.2323254346847534,
      "rewards/rejected": -0.7464088201522827,
      "step": 4123
    },
    {
      "epoch": 0.5396581336386685,
      "grad_norm": 7.383779525756836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08379939198493958,
      "logits/rejected": -0.04517296701669693,
      "logps/chosen": -643.4212646484375,
      "logps/rejected": -431.8191223144531,
      "loss": 0.3765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42544329166412354,
      "rewards/margins": 1.1562154293060303,
      "rewards/rejected": -1.5816586017608643,
      "step": 4124
    },
    {
      "epoch": 0.5397889915760203,
      "grad_norm": 9.089682579040527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09956999123096466,
      "logits/rejected": 0.2853808104991913,
      "logps/chosen": -480.0960693359375,
      "logps/rejected": -508.7442321777344,
      "loss": 0.7207,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5715939998626709,
      "rewards/margins": 0.7352119088172913,
      "rewards/rejected": -1.306805968284607,
      "step": 4125
    },
    {
      "epoch": 0.539919849513372,
      "grad_norm": 6.959366321563721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03529594466090202,
      "logits/rejected": 0.057711996138095856,
      "logps/chosen": -462.2287292480469,
      "logps/rejected": -555.4818115234375,
      "loss": 0.4994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2802679240703583,
      "rewards/margins": 1.1571569442749023,
      "rewards/rejected": -1.4374247789382935,
      "step": 4126
    },
    {
      "epoch": 0.5400507074507238,
      "grad_norm": 11.838032722473145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04483220353722572,
      "logits/rejected": 0.021469540894031525,
      "logps/chosen": -346.02642822265625,
      "logps/rejected": -504.6833190917969,
      "loss": 1.0345,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7081313133239746,
      "rewards/margins": -0.1361064314842224,
      "rewards/rejected": -1.572024941444397,
      "step": 4127
    },
    {
      "epoch": 0.5401815653880756,
      "grad_norm": 4.4864678382873535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028986485674977303,
      "logits/rejected": 0.03278965875506401,
      "logps/chosen": -585.8131103515625,
      "logps/rejected": -458.59649658203125,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9069185256958008,
      "rewards/margins": 2.056366443634033,
      "rewards/rejected": -1.1494479179382324,
      "step": 4128
    },
    {
      "epoch": 0.5403124233254273,
      "grad_norm": 8.854829788208008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12771821022033691,
      "logits/rejected": 0.10277104377746582,
      "logps/chosen": -569.431640625,
      "logps/rejected": -689.6688842773438,
      "loss": 0.6094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20603157579898834,
      "rewards/margins": 0.9268061518669128,
      "rewards/rejected": -0.7207744121551514,
      "step": 4129
    },
    {
      "epoch": 0.5404432812627791,
      "grad_norm": 13.149803161621094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2708258032798767,
      "logits/rejected": 0.2081460952758789,
      "logps/chosen": -615.0992431640625,
      "logps/rejected": -574.474365234375,
      "loss": 0.9252,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32362693548202515,
      "rewards/margins": 0.530003011226654,
      "rewards/rejected": -0.8536298274993896,
      "step": 4130
    },
    {
      "epoch": 0.5405741392001309,
      "grad_norm": 4.449957370758057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.056631386280059814,
      "logits/rejected": 0.021062983199954033,
      "logps/chosen": -386.7738952636719,
      "logps/rejected": -393.9751892089844,
      "loss": 0.361,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06988183408975601,
      "rewards/margins": 1.2745130062103271,
      "rewards/rejected": -1.2046313285827637,
      "step": 4131
    },
    {
      "epoch": 0.5407049971374827,
      "grad_norm": 7.970259189605713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20613326132297516,
      "logits/rejected": -0.37997114658355713,
      "logps/chosen": -293.8669128417969,
      "logps/rejected": -328.83416748046875,
      "loss": 0.9155,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0980571508407593,
      "rewards/margins": -0.08448255062103271,
      "rewards/rejected": -1.0135746002197266,
      "step": 4132
    },
    {
      "epoch": 0.5408358550748343,
      "grad_norm": 8.489128112792969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30919742584228516,
      "logits/rejected": 0.2108459770679474,
      "logps/chosen": -526.7161865234375,
      "logps/rejected": -453.69268798828125,
      "loss": 0.721,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3942663073539734,
      "rewards/margins": 0.3269893527030945,
      "rewards/rejected": 0.06727693974971771,
      "step": 4133
    },
    {
      "epoch": 0.5409667130121861,
      "grad_norm": 10.808921813964844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2469455897808075,
      "logits/rejected": 0.10697485506534576,
      "logps/chosen": -689.0925903320312,
      "logps/rejected": -472.3670654296875,
      "loss": 0.6401,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0416591539978981,
      "rewards/margins": 0.4768989682197571,
      "rewards/rejected": -0.4352398216724396,
      "step": 4134
    },
    {
      "epoch": 0.5410975709495379,
      "grad_norm": 9.156250953674316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.208446204662323,
      "logits/rejected": -0.2628566324710846,
      "logps/chosen": -424.88427734375,
      "logps/rejected": -395.8179626464844,
      "loss": 0.5781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2341727912425995,
      "rewards/margins": 0.7453127503395081,
      "rewards/rejected": -0.9794855117797852,
      "step": 4135
    },
    {
      "epoch": 0.5412284288868897,
      "grad_norm": 3.4382152557373047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25613829493522644,
      "logits/rejected": 0.17516237497329712,
      "logps/chosen": -446.07196044921875,
      "logps/rejected": -305.2655944824219,
      "loss": 0.2104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19942818582057953,
      "rewards/margins": 1.639423131942749,
      "rewards/rejected": -1.4399948120117188,
      "step": 4136
    },
    {
      "epoch": 0.5413592868242414,
      "grad_norm": 4.564956188201904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002882927656173706,
      "logits/rejected": -0.18839894235134125,
      "logps/chosen": -352.2744140625,
      "logps/rejected": -310.25799560546875,
      "loss": 0.4007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5566728115081787,
      "rewards/margins": 1.033874750137329,
      "rewards/rejected": -1.5905476808547974,
      "step": 4137
    },
    {
      "epoch": 0.5414901447615932,
      "grad_norm": 5.570894718170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1609206199645996,
      "logits/rejected": -0.06919317692518234,
      "logps/chosen": -380.7422790527344,
      "logps/rejected": -356.29437255859375,
      "loss": 0.6369,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.44598323106765747,
      "rewards/margins": 0.4106482267379761,
      "rewards/rejected": 0.03533497452735901,
      "step": 4138
    },
    {
      "epoch": 0.541621002698945,
      "grad_norm": 5.119296073913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1737915426492691,
      "logits/rejected": -0.19047051668167114,
      "logps/chosen": -364.7525939941406,
      "logps/rejected": -289.02276611328125,
      "loss": 0.4283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49158501625061035,
      "rewards/margins": 0.7410781383514404,
      "rewards/rejected": -1.2326630353927612,
      "step": 4139
    },
    {
      "epoch": 0.5417518606362968,
      "grad_norm": 7.211817264556885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15011043846607208,
      "logits/rejected": 0.12033397704362869,
      "logps/chosen": -561.6773681640625,
      "logps/rejected": -459.24249267578125,
      "loss": 0.5183,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07198600471019745,
      "rewards/margins": 0.7035755515098572,
      "rewards/rejected": -0.6315895318984985,
      "step": 4140
    },
    {
      "epoch": 0.5418827185736484,
      "grad_norm": 9.744399070739746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3174765706062317,
      "logits/rejected": -0.0681782141327858,
      "logps/chosen": -494.1035461425781,
      "logps/rejected": -554.6060791015625,
      "loss": 0.7956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0532528162002563,
      "rewards/margins": 0.09203359484672546,
      "rewards/rejected": -1.1452863216400146,
      "step": 4141
    },
    {
      "epoch": 0.5420135765110002,
      "grad_norm": 2.595693588256836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051167938858270645,
      "logits/rejected": -0.301535964012146,
      "logps/chosen": -256.0499572753906,
      "logps/rejected": -492.3689270019531,
      "loss": 0.166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19504281878471375,
      "rewards/margins": 2.8724162578582764,
      "rewards/rejected": -3.0674591064453125,
      "step": 4142
    },
    {
      "epoch": 0.542144434448352,
      "grad_norm": 4.13629150390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0795089453458786,
      "logits/rejected": 0.11158978193998337,
      "logps/chosen": -625.2794189453125,
      "logps/rejected": -445.9882507324219,
      "loss": 0.3058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11199226975440979,
      "rewards/margins": 1.3227897882461548,
      "rewards/rejected": -1.4347820281982422,
      "step": 4143
    },
    {
      "epoch": 0.5422752923857038,
      "grad_norm": 7.018155574798584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16521406173706055,
      "logits/rejected": 0.11463682353496552,
      "logps/chosen": -391.4933776855469,
      "logps/rejected": -397.30853271484375,
      "loss": 0.6306,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16621381044387817,
      "rewards/margins": 0.39184799790382385,
      "rewards/rejected": -0.5580617189407349,
      "step": 4144
    },
    {
      "epoch": 0.5424061503230555,
      "grad_norm": 6.221100330352783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35137978196144104,
      "logits/rejected": 0.14733541011810303,
      "logps/chosen": -457.5617980957031,
      "logps/rejected": -327.87603759765625,
      "loss": 0.4053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0013521462678909302,
      "rewards/margins": 0.9704074859619141,
      "rewards/rejected": -0.9690554141998291,
      "step": 4145
    },
    {
      "epoch": 0.5425370082604073,
      "grad_norm": 9.707581520080566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28305158019065857,
      "logits/rejected": 0.19298692047595978,
      "logps/chosen": -428.2606201171875,
      "logps/rejected": -379.2715148925781,
      "loss": 0.6786,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6056806445121765,
      "rewards/margins": 0.5915105938911438,
      "rewards/rejected": -1.1971911191940308,
      "step": 4146
    },
    {
      "epoch": 0.5426678661977591,
      "grad_norm": 7.944955348968506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28506961464881897,
      "logits/rejected": 0.32263991236686707,
      "logps/chosen": -651.7998046875,
      "logps/rejected": -633.717041015625,
      "loss": 0.6805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0980808213353157,
      "rewards/margins": 0.32637518644332886,
      "rewards/rejected": -0.42445603013038635,
      "step": 4147
    },
    {
      "epoch": 0.5427987241351108,
      "grad_norm": 6.502335548400879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1047079861164093,
      "logits/rejected": 0.04581402987241745,
      "logps/chosen": -393.04296875,
      "logps/rejected": -386.93316650390625,
      "loss": 0.4334,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.0341953039169312,
      "rewards/margins": 1.8638083934783936,
      "rewards/rejected": -0.8296129107475281,
      "step": 4148
    },
    {
      "epoch": 0.5429295820724626,
      "grad_norm": 6.930442810058594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31783944368362427,
      "logits/rejected": 0.4128294587135315,
      "logps/chosen": -562.9641723632812,
      "logps/rejected": -421.8157958984375,
      "loss": 0.424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19886034727096558,
      "rewards/margins": 1.4407271146774292,
      "rewards/rejected": -1.6395875215530396,
      "step": 4149
    },
    {
      "epoch": 0.5430604400098143,
      "grad_norm": 5.723570823669434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23162369430065155,
      "logits/rejected": -0.06597593426704407,
      "logps/chosen": -495.5889892578125,
      "logps/rejected": -412.6820068359375,
      "loss": 0.3841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2648550570011139,
      "rewards/margins": 1.1102771759033203,
      "rewards/rejected": -1.3751322031021118,
      "step": 4150
    },
    {
      "epoch": 0.5431912979471661,
      "grad_norm": 6.579124450683594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04985957220196724,
      "logits/rejected": -0.09075765311717987,
      "logps/chosen": -402.93048095703125,
      "logps/rejected": -348.0780944824219,
      "loss": 0.4165,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0038789696991443634,
      "rewards/margins": 1.2557687759399414,
      "rewards/rejected": -1.2518898248672485,
      "step": 4151
    },
    {
      "epoch": 0.5433221558845178,
      "grad_norm": 8.09062671661377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09851841628551483,
      "logits/rejected": -0.10914570838212967,
      "logps/chosen": -516.2882690429688,
      "logps/rejected": -427.1710510253906,
      "loss": 0.6409,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.155973419547081,
      "rewards/margins": 0.6551723480224609,
      "rewards/rejected": -0.49919891357421875,
      "step": 4152
    },
    {
      "epoch": 0.5434530138218696,
      "grad_norm": 7.787176609039307,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24014660716056824,
      "logits/rejected": 0.09084372222423553,
      "logps/chosen": -458.34521484375,
      "logps/rejected": -417.54034423828125,
      "loss": 0.4508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23875463008880615,
      "rewards/margins": 0.9451354146003723,
      "rewards/rejected": -1.1838901042938232,
      "step": 4153
    },
    {
      "epoch": 0.5435838717592214,
      "grad_norm": 6.385119915008545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2541452944278717,
      "logits/rejected": 0.304944783449173,
      "logps/chosen": -576.9692993164062,
      "logps/rejected": -544.4554443359375,
      "loss": 0.401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8147156238555908,
      "rewards/margins": 1.6693317890167236,
      "rewards/rejected": -0.8546162247657776,
      "step": 4154
    },
    {
      "epoch": 0.5437147296965732,
      "grad_norm": 7.80438232421875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22217050194740295,
      "logits/rejected": 0.25419723987579346,
      "logps/chosen": -284.34521484375,
      "logps/rejected": -338.85089111328125,
      "loss": 0.6212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3531806766986847,
      "rewards/margins": 0.36481213569641113,
      "rewards/rejected": -0.7179927825927734,
      "step": 4155
    },
    {
      "epoch": 0.5438455876339249,
      "grad_norm": 9.065359115600586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17259910702705383,
      "logits/rejected": 0.23242861032485962,
      "logps/chosen": -445.6199951171875,
      "logps/rejected": -472.1190185546875,
      "loss": 0.6566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4280857443809509,
      "rewards/margins": 0.5190927386283875,
      "rewards/rejected": -0.9471784234046936,
      "step": 4156
    },
    {
      "epoch": 0.5439764455712767,
      "grad_norm": 5.366570949554443,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21680575609207153,
      "logits/rejected": 0.10959623754024506,
      "logps/chosen": -557.4881591796875,
      "logps/rejected": -454.4649353027344,
      "loss": 0.3477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24155083298683167,
      "rewards/margins": 1.0716371536254883,
      "rewards/rejected": -0.8300862908363342,
      "step": 4157
    },
    {
      "epoch": 0.5441073035086285,
      "grad_norm": 5.959880828857422,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07223066687583923,
      "logits/rejected": 0.15758958458900452,
      "logps/chosen": -353.9967956542969,
      "logps/rejected": -365.555908203125,
      "loss": 0.3827,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0062874071300029755,
      "rewards/margins": 1.0896635055541992,
      "rewards/rejected": -1.0959508419036865,
      "step": 4158
    },
    {
      "epoch": 0.5442381614459803,
      "grad_norm": 6.084469318389893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07956534624099731,
      "logits/rejected": 0.1040542870759964,
      "logps/chosen": -326.07763671875,
      "logps/rejected": -514.101806640625,
      "loss": 0.3781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07205486297607422,
      "rewards/margins": 1.5479211807250977,
      "rewards/rejected": -1.4758663177490234,
      "step": 4159
    },
    {
      "epoch": 0.5443690193833319,
      "grad_norm": 10.73739242553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11962219327688217,
      "logits/rejected": -0.02211814373731613,
      "logps/chosen": -477.6033935546875,
      "logps/rejected": -497.17742919921875,
      "loss": 0.8805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0177021026611328,
      "rewards/margins": 0.29984474182128906,
      "rewards/rejected": -1.3175468444824219,
      "step": 4160
    },
    {
      "epoch": 0.5444998773206837,
      "grad_norm": 6.452970027923584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15254229307174683,
      "logits/rejected": 0.09974673390388489,
      "logps/chosen": -426.0968017578125,
      "logps/rejected": -367.54425048828125,
      "loss": 0.5601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15751782059669495,
      "rewards/margins": 0.4191516637802124,
      "rewards/rejected": -0.576669454574585,
      "step": 4161
    },
    {
      "epoch": 0.5446307352580355,
      "grad_norm": 7.289144992828369,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4011121988296509,
      "logits/rejected": 0.3705233931541443,
      "logps/chosen": -429.72015380859375,
      "logps/rejected": -307.007568359375,
      "loss": 0.6718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30942419171333313,
      "rewards/margins": 0.8347963094711304,
      "rewards/rejected": -0.5253720879554749,
      "step": 4162
    },
    {
      "epoch": 0.5447615931953873,
      "grad_norm": 7.176026344299316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15583473443984985,
      "logits/rejected": 0.21446388959884644,
      "logps/chosen": -474.17926025390625,
      "logps/rejected": -327.094482421875,
      "loss": 0.4681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08807124197483063,
      "rewards/margins": 1.163881540298462,
      "rewards/rejected": -1.2519527673721313,
      "step": 4163
    },
    {
      "epoch": 0.544892451132739,
      "grad_norm": 5.4354753494262695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010937228798866272,
      "logits/rejected": -0.08692001551389694,
      "logps/chosen": -547.3812255859375,
      "logps/rejected": -509.398681640625,
      "loss": 0.29,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40843284130096436,
      "rewards/margins": 1.3521125316619873,
      "rewards/rejected": -1.7605453729629517,
      "step": 4164
    },
    {
      "epoch": 0.5450233090700908,
      "grad_norm": 6.867290496826172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15320532023906708,
      "logits/rejected": -0.19847694039344788,
      "logps/chosen": -393.704345703125,
      "logps/rejected": -366.2513122558594,
      "loss": 0.4686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3253815174102783,
      "rewards/margins": 0.9477954506874084,
      "rewards/rejected": -1.2731770277023315,
      "step": 4165
    },
    {
      "epoch": 0.5451541670074426,
      "grad_norm": 8.044168472290039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028651133179664612,
      "logits/rejected": -0.03425714373588562,
      "logps/chosen": -394.14971923828125,
      "logps/rejected": -487.83660888671875,
      "loss": 0.8386,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6881790161132812,
      "rewards/margins": -0.1920124739408493,
      "rewards/rejected": -0.49616652727127075,
      "step": 4166
    },
    {
      "epoch": 0.5452850249447944,
      "grad_norm": 4.8143229484558105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06343800574541092,
      "logits/rejected": 0.14285917580127716,
      "logps/chosen": -295.4919738769531,
      "logps/rejected": -275.7881164550781,
      "loss": 0.3402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5756769180297852,
      "rewards/margins": 1.344991683959961,
      "rewards/rejected": -0.7693147659301758,
      "step": 4167
    },
    {
      "epoch": 0.545415882882146,
      "grad_norm": 7.015022277832031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029018033295869827,
      "logits/rejected": 0.11573269218206406,
      "logps/chosen": -599.8128662109375,
      "logps/rejected": -517.4719848632812,
      "loss": 0.4518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46426814794540405,
      "rewards/margins": 1.187365174293518,
      "rewards/rejected": -1.651633620262146,
      "step": 4168
    },
    {
      "epoch": 0.5455467408194978,
      "grad_norm": 8.355856895446777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017070606350898743,
      "logits/rejected": 0.19004207849502563,
      "logps/chosen": -392.3078918457031,
      "logps/rejected": -614.9826049804688,
      "loss": 0.6316,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17490404844284058,
      "rewards/margins": 0.6861101388931274,
      "rewards/rejected": -0.861014187335968,
      "step": 4169
    },
    {
      "epoch": 0.5456775987568496,
      "grad_norm": 7.461470603942871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06668785959482193,
      "logits/rejected": 0.032053545117378235,
      "logps/chosen": -293.3127746582031,
      "logps/rejected": -309.709716796875,
      "loss": 0.6814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7244923114776611,
      "rewards/margins": 0.7036254405975342,
      "rewards/rejected": -1.4281177520751953,
      "step": 4170
    },
    {
      "epoch": 0.5458084566942013,
      "grad_norm": 5.468849182128906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3332127332687378,
      "logits/rejected": 0.2112596333026886,
      "logps/chosen": -298.5372009277344,
      "logps/rejected": -379.9993896484375,
      "loss": 0.4218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07851751893758774,
      "rewards/margins": 1.4006202220916748,
      "rewards/rejected": -1.4791377782821655,
      "step": 4171
    },
    {
      "epoch": 0.5459393146315531,
      "grad_norm": 10.591900825500488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21588997542858124,
      "logits/rejected": 0.09614403545856476,
      "logps/chosen": -572.890625,
      "logps/rejected": -436.4750671386719,
      "loss": 0.8813,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2350805103778839,
      "rewards/margins": 1.0286805629730225,
      "rewards/rejected": -0.7936001420021057,
      "step": 4172
    },
    {
      "epoch": 0.5460701725689049,
      "grad_norm": 6.190197467803955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15110351145267487,
      "logits/rejected": 0.05079653114080429,
      "logps/chosen": -290.1070251464844,
      "logps/rejected": -205.4400634765625,
      "loss": 0.5187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2353334128856659,
      "rewards/margins": 0.8189367651939392,
      "rewards/rejected": -1.0542702674865723,
      "step": 4173
    },
    {
      "epoch": 0.5462010305062567,
      "grad_norm": 7.863356113433838,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0506177619099617,
      "logits/rejected": 0.09922026097774506,
      "logps/chosen": -269.73223876953125,
      "logps/rejected": -440.2435607910156,
      "loss": 0.5336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7073225378990173,
      "rewards/margins": 0.8027251958847046,
      "rewards/rejected": -1.5100477933883667,
      "step": 4174
    },
    {
      "epoch": 0.5463318884436084,
      "grad_norm": 5.790628910064697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14553974568843842,
      "logits/rejected": -0.2103625237941742,
      "logps/chosen": -413.545166015625,
      "logps/rejected": -647.1962280273438,
      "loss": 0.3302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3269956707954407,
      "rewards/margins": 1.2607086896896362,
      "rewards/rejected": -1.5877044200897217,
      "step": 4175
    },
    {
      "epoch": 0.5464627463809602,
      "grad_norm": 6.485198497772217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1009538322687149,
      "logits/rejected": 0.08271040767431259,
      "logps/chosen": -366.1214904785156,
      "logps/rejected": -330.8663024902344,
      "loss": 0.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12195761501789093,
      "rewards/margins": 0.8334133625030518,
      "rewards/rejected": -0.7114557027816772,
      "step": 4176
    },
    {
      "epoch": 0.5465936043183119,
      "grad_norm": 8.703751564025879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15979647636413574,
      "logits/rejected": 0.09532380849123001,
      "logps/chosen": -404.20025634765625,
      "logps/rejected": -406.20379638671875,
      "loss": 0.6478,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33230888843536377,
      "rewards/margins": 0.3976612985134125,
      "rewards/rejected": -0.7299700975418091,
      "step": 4177
    },
    {
      "epoch": 0.5467244622556637,
      "grad_norm": 5.570422649383545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02496415376663208,
      "logits/rejected": 0.04360591992735863,
      "logps/chosen": -567.3515014648438,
      "logps/rejected": -541.1102294921875,
      "loss": 0.2952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5405303835868835,
      "rewards/margins": 1.838180422782898,
      "rewards/rejected": -1.2976500988006592,
      "step": 4178
    },
    {
      "epoch": 0.5468553201930154,
      "grad_norm": 3.2104012966156006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27259016036987305,
      "logits/rejected": 0.16361722350120544,
      "logps/chosen": -359.5069274902344,
      "logps/rejected": -455.5123596191406,
      "loss": 0.2415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.741694450378418,
      "rewards/margins": 2.3438520431518555,
      "rewards/rejected": -1.602157711982727,
      "step": 4179
    },
    {
      "epoch": 0.5469861781303672,
      "grad_norm": 7.44834041595459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20988717675209045,
      "logits/rejected": 0.1491708755493164,
      "logps/chosen": -334.6052551269531,
      "logps/rejected": -390.0025634765625,
      "loss": 0.6298,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6088905334472656,
      "rewards/margins": 0.6186766028404236,
      "rewards/rejected": -1.227567195892334,
      "step": 4180
    },
    {
      "epoch": 0.547117036067719,
      "grad_norm": 10.790743827819824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22860562801361084,
      "logits/rejected": 0.2995001971721649,
      "logps/chosen": -423.6488342285156,
      "logps/rejected": -469.13751220703125,
      "loss": 0.6817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3422207236289978,
      "rewards/margins": 1.300074815750122,
      "rewards/rejected": -1.642295479774475,
      "step": 4181
    },
    {
      "epoch": 0.5472478940050708,
      "grad_norm": 7.062157154083252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00994675699621439,
      "logits/rejected": -0.06537850946187973,
      "logps/chosen": -592.392578125,
      "logps/rejected": -603.1322021484375,
      "loss": 0.3916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6437656879425049,
      "rewards/margins": 1.259395718574524,
      "rewards/rejected": -0.6156299114227295,
      "step": 4182
    },
    {
      "epoch": 0.5473787519424225,
      "grad_norm": 3.469238519668579,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32820460200309753,
      "logits/rejected": 0.06073570251464844,
      "logps/chosen": -729.227783203125,
      "logps/rejected": -359.58990478515625,
      "loss": 0.276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9547121524810791,
      "rewards/margins": 1.5997258424758911,
      "rewards/rejected": -0.645013689994812,
      "step": 4183
    },
    {
      "epoch": 0.5475096098797743,
      "grad_norm": 5.898248195648193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1428641676902771,
      "logits/rejected": 0.10466457158327103,
      "logps/chosen": -398.31097412109375,
      "logps/rejected": -321.24310302734375,
      "loss": 0.4676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.011088080704212189,
      "rewards/margins": 0.7164510488510132,
      "rewards/rejected": -0.7275391221046448,
      "step": 4184
    },
    {
      "epoch": 0.547640467817126,
      "grad_norm": 5.825472831726074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5603839755058289,
      "logits/rejected": 0.4466441571712494,
      "logps/chosen": -532.6995849609375,
      "logps/rejected": -575.954833984375,
      "loss": 0.4663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.613788366317749,
      "rewards/margins": 1.088175892829895,
      "rewards/rejected": -0.47438734769821167,
      "step": 4185
    },
    {
      "epoch": 0.5477713257544778,
      "grad_norm": 4.46580696105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18313223123550415,
      "logits/rejected": -0.10021091997623444,
      "logps/chosen": -632.037109375,
      "logps/rejected": -462.1910705566406,
      "loss": 0.2664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5417757034301758,
      "rewards/margins": 1.5928453207015991,
      "rewards/rejected": -2.1346211433410645,
      "step": 4186
    },
    {
      "epoch": 0.5479021836918295,
      "grad_norm": 8.819601058959961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11974470317363739,
      "logits/rejected": 0.01905553787946701,
      "logps/chosen": -489.64288330078125,
      "logps/rejected": -634.7940063476562,
      "loss": 0.6186,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08513239026069641,
      "rewards/margins": 0.6902544498443604,
      "rewards/rejected": -0.6051220297813416,
      "step": 4187
    },
    {
      "epoch": 0.5480330416291813,
      "grad_norm": 8.091569900512695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29362067580223083,
      "logits/rejected": 0.3821945786476135,
      "logps/chosen": -324.50714111328125,
      "logps/rejected": -419.4547119140625,
      "loss": 0.6329,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1475960910320282,
      "rewards/margins": 0.8498820066452026,
      "rewards/rejected": -0.702285885810852,
      "step": 4188
    },
    {
      "epoch": 0.5481638995665331,
      "grad_norm": 10.714837074279785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18636691570281982,
      "logits/rejected": 0.08970801532268524,
      "logps/chosen": -698.9031982421875,
      "logps/rejected": -635.2173461914062,
      "loss": 0.8394,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09566919505596161,
      "rewards/margins": -0.012772716581821442,
      "rewards/rejected": 0.10844192653894424,
      "step": 4189
    },
    {
      "epoch": 0.5482947575038849,
      "grad_norm": 3.7988128662109375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3957829177379608,
      "logits/rejected": 0.004569115117192268,
      "logps/chosen": -434.6929931640625,
      "logps/rejected": -384.00946044921875,
      "loss": 0.2878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19699572026729584,
      "rewards/margins": 2.014446496963501,
      "rewards/rejected": -1.817450761795044,
      "step": 4190
    },
    {
      "epoch": 0.5484256154412366,
      "grad_norm": 10.409854888916016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037401944398880005,
      "logits/rejected": 0.0773983746767044,
      "logps/chosen": -410.58697509765625,
      "logps/rejected": -662.5323486328125,
      "loss": 0.9058,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4422680139541626,
      "rewards/margins": 0.45486709475517273,
      "rewards/rejected": -0.8971351385116577,
      "step": 4191
    },
    {
      "epoch": 0.5485564733785884,
      "grad_norm": 4.994929313659668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14460325241088867,
      "logits/rejected": 0.1389007866382599,
      "logps/chosen": -321.17962646484375,
      "logps/rejected": -430.1304016113281,
      "loss": 0.3921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1421453356742859,
      "rewards/margins": 1.3355445861816406,
      "rewards/rejected": -1.4776899814605713,
      "step": 4192
    },
    {
      "epoch": 0.5486873313159402,
      "grad_norm": 4.06279182434082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16041292250156403,
      "logits/rejected": -0.008798161521553993,
      "logps/chosen": -558.9847412109375,
      "logps/rejected": -648.3676147460938,
      "loss": 0.2281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05972534418106079,
      "rewards/margins": 2.7292184829711914,
      "rewards/rejected": -2.6694931983947754,
      "step": 4193
    },
    {
      "epoch": 0.5488181892532918,
      "grad_norm": 5.629594802856445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028835315257310867,
      "logits/rejected": -0.19316422939300537,
      "logps/chosen": -417.9237365722656,
      "logps/rejected": -348.4542236328125,
      "loss": 0.4719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1926325112581253,
      "rewards/margins": 0.9213793277740479,
      "rewards/rejected": -0.7287468910217285,
      "step": 4194
    },
    {
      "epoch": 0.5489490471906436,
      "grad_norm": 7.851417064666748,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19486930966377258,
      "logits/rejected": -0.06883665919303894,
      "logps/chosen": -475.1856384277344,
      "logps/rejected": -385.2279968261719,
      "loss": 0.4243,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42466074228286743,
      "rewards/margins": 1.3918750286102295,
      "rewards/rejected": -1.8165357112884521,
      "step": 4195
    },
    {
      "epoch": 0.5490799051279954,
      "grad_norm": 7.280573844909668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18358656764030457,
      "logits/rejected": 0.16397890448570251,
      "logps/chosen": -511.4449157714844,
      "logps/rejected": -394.7331237792969,
      "loss": 0.3547,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11142904311418533,
      "rewards/margins": 1.1823350191116333,
      "rewards/rejected": -1.0709059238433838,
      "step": 4196
    },
    {
      "epoch": 0.5492107630653472,
      "grad_norm": 10.02498722076416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040324028581380844,
      "logits/rejected": 0.1782134473323822,
      "logps/chosen": -327.928466796875,
      "logps/rejected": -496.1053161621094,
      "loss": 1.0557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1078617572784424,
      "rewards/margins": -0.25609394907951355,
      "rewards/rejected": -0.8517677783966064,
      "step": 4197
    },
    {
      "epoch": 0.5493416210026989,
      "grad_norm": 8.846016883850098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05199979245662689,
      "logits/rejected": -0.1486709713935852,
      "logps/chosen": -461.0375061035156,
      "logps/rejected": -431.31964111328125,
      "loss": 0.6746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9653095006942749,
      "rewards/margins": 0.6585836410522461,
      "rewards/rejected": -1.623893141746521,
      "step": 4198
    },
    {
      "epoch": 0.5494724789400507,
      "grad_norm": 9.360754013061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07212384045124054,
      "logits/rejected": 0.042630258947610855,
      "logps/chosen": -675.9613037109375,
      "logps/rejected": -500.9245910644531,
      "loss": 0.7325,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41073572635650635,
      "rewards/margins": 0.5404263138771057,
      "rewards/rejected": -0.9511619806289673,
      "step": 4199
    },
    {
      "epoch": 0.5496033368774025,
      "grad_norm": 8.950902938842773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07845176756381989,
      "logits/rejected": 0.05157899484038353,
      "logps/chosen": -571.42236328125,
      "logps/rejected": -531.5234375,
      "loss": 0.4402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17042827606201172,
      "rewards/margins": 0.9838250875473022,
      "rewards/rejected": -0.8133968710899353,
      "step": 4200
    },
    {
      "epoch": 0.5497341948147543,
      "grad_norm": 6.235395908355713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05731761455535889,
      "logits/rejected": -0.03741438686847687,
      "logps/chosen": -508.32550048828125,
      "logps/rejected": -359.02496337890625,
      "loss": 0.5542,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.45085984468460083,
      "rewards/margins": 0.9955366849899292,
      "rewards/rejected": -0.5446767807006836,
      "step": 4201
    },
    {
      "epoch": 0.549865052752106,
      "grad_norm": 6.058699607849121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16366654634475708,
      "logits/rejected": 0.0386720709502697,
      "logps/chosen": -489.6322021484375,
      "logps/rejected": -421.7139892578125,
      "loss": 0.3849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.44715577363967896,
      "rewards/margins": 1.1842349767684937,
      "rewards/rejected": -1.6313908100128174,
      "step": 4202
    },
    {
      "epoch": 0.5499959106894577,
      "grad_norm": 8.045637130737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1669793277978897,
      "logits/rejected": 0.12448733299970627,
      "logps/chosen": -394.2527160644531,
      "logps/rejected": -485.7413635253906,
      "loss": 0.6966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24009189009666443,
      "rewards/margins": 1.0837159156799316,
      "rewards/rejected": -1.323807716369629,
      "step": 4203
    },
    {
      "epoch": 0.5501267686268095,
      "grad_norm": 4.202549457550049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2131274938583374,
      "logits/rejected": -0.0159560926258564,
      "logps/chosen": -499.684814453125,
      "logps/rejected": -421.1843566894531,
      "loss": 0.2559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07655295729637146,
      "rewards/margins": 1.722494125366211,
      "rewards/rejected": -1.645941138267517,
      "step": 4204
    },
    {
      "epoch": 0.5502576265641613,
      "grad_norm": 7.98906135559082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3948062062263489,
      "logits/rejected": 0.130782812833786,
      "logps/chosen": -440.2039489746094,
      "logps/rejected": -685.6841430664062,
      "loss": 0.6423,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21743927896022797,
      "rewards/margins": 0.738007664680481,
      "rewards/rejected": -0.520568311214447,
      "step": 4205
    },
    {
      "epoch": 0.550388484501513,
      "grad_norm": 8.328936576843262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10480034351348877,
      "logits/rejected": 0.14357492327690125,
      "logps/chosen": -503.8417663574219,
      "logps/rejected": -354.6410217285156,
      "loss": 0.5022,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.35628607869148254,
      "rewards/margins": 0.8031706809997559,
      "rewards/rejected": -0.44688454270362854,
      "step": 4206
    },
    {
      "epoch": 0.5505193424388648,
      "grad_norm": 4.905803203582764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02831418812274933,
      "logits/rejected": -0.06779278069734573,
      "logps/chosen": -500.01446533203125,
      "logps/rejected": -540.5305786132812,
      "loss": 0.3042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7349854111671448,
      "rewards/margins": 1.8367552757263184,
      "rewards/rejected": -1.1017698049545288,
      "step": 4207
    },
    {
      "epoch": 0.5506502003762166,
      "grad_norm": 6.826504230499268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04153072088956833,
      "logits/rejected": 0.1019335389137268,
      "logps/chosen": -565.8746337890625,
      "logps/rejected": -646.9144287109375,
      "loss": 0.4078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.365948885679245,
      "rewards/margins": 1.6417162418365479,
      "rewards/rejected": -2.007664918899536,
      "step": 4208
    },
    {
      "epoch": 0.5507810583135684,
      "grad_norm": 8.322428703308105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024206994101405144,
      "logits/rejected": -0.18914946913719177,
      "logps/chosen": -593.5953369140625,
      "logps/rejected": -491.8898010253906,
      "loss": 0.5424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.473853200674057,
      "rewards/margins": 1.8533841371536255,
      "rewards/rejected": -1.379530906677246,
      "step": 4209
    },
    {
      "epoch": 0.5509119162509201,
      "grad_norm": 4.24659538269043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43403637409210205,
      "logits/rejected": 0.2687692642211914,
      "logps/chosen": -348.43499755859375,
      "logps/rejected": -461.1222229003906,
      "loss": 0.3117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5160484313964844,
      "rewards/margins": 2.5546178817749023,
      "rewards/rejected": -3.0706660747528076,
      "step": 4210
    },
    {
      "epoch": 0.5510427741882719,
      "grad_norm": 9.813543319702148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2044607400894165,
      "logits/rejected": -0.051461659371852875,
      "logps/chosen": -467.30657958984375,
      "logps/rejected": -379.47509765625,
      "loss": 0.9804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.814980149269104,
      "rewards/margins": 0.23106591403484344,
      "rewards/rejected": -1.0460461378097534,
      "step": 4211
    },
    {
      "epoch": 0.5511736321256236,
      "grad_norm": 6.2316436767578125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14801615476608276,
      "logits/rejected": -0.04007154330611229,
      "logps/chosen": -396.8955078125,
      "logps/rejected": -336.6162109375,
      "loss": 0.4339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32536429166793823,
      "rewards/margins": 1.5638669729232788,
      "rewards/rejected": -1.8892313241958618,
      "step": 4212
    },
    {
      "epoch": 0.5513044900629753,
      "grad_norm": 6.754332542419434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.440336138010025,
      "logits/rejected": 0.5537797808647156,
      "logps/chosen": -402.4797668457031,
      "logps/rejected": -435.518310546875,
      "loss": 0.5463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.35227394104003906,
      "rewards/margins": 1.143233299255371,
      "rewards/rejected": -0.7909594178199768,
      "step": 4213
    },
    {
      "epoch": 0.5514353480003271,
      "grad_norm": 8.78382396697998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3171725869178772,
      "logits/rejected": 0.2300303727388382,
      "logps/chosen": -594.562744140625,
      "logps/rejected": -591.6936645507812,
      "loss": 0.5949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20810547471046448,
      "rewards/margins": 0.42820531129837036,
      "rewards/rejected": -0.6363108158111572,
      "step": 4214
    },
    {
      "epoch": 0.5515662059376789,
      "grad_norm": 4.937828540802002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09370917081832886,
      "logits/rejected": 0.04820523038506508,
      "logps/chosen": -402.6470947265625,
      "logps/rejected": -431.4454345703125,
      "loss": 0.3233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5264824032783508,
      "rewards/margins": 1.2860554456710815,
      "rewards/rejected": -0.7595730423927307,
      "step": 4215
    },
    {
      "epoch": 0.5516970638750307,
      "grad_norm": 10.249098777770996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5797385573387146,
      "logits/rejected": 0.48160460591316223,
      "logps/chosen": -486.147705078125,
      "logps/rejected": -737.4228515625,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2234155833721161,
      "rewards/margins": 0.1384492665529251,
      "rewards/rejected": -0.36186483502388,
      "step": 4216
    },
    {
      "epoch": 0.5518279218123824,
      "grad_norm": 6.8372483253479,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2152099311351776,
      "logits/rejected": -0.005746626295149326,
      "logps/chosen": -328.43365478515625,
      "logps/rejected": -465.87811279296875,
      "loss": 0.7239,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2775663137435913,
      "rewards/margins": 0.1894758939743042,
      "rewards/rejected": -0.4670422673225403,
      "step": 4217
    },
    {
      "epoch": 0.5519587797497342,
      "grad_norm": 9.727916717529297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012653671205043793,
      "logits/rejected": -0.03005758672952652,
      "logps/chosen": -385.05169677734375,
      "logps/rejected": -435.7074890136719,
      "loss": 0.5956,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6169065237045288,
      "rewards/margins": 0.6488680839538574,
      "rewards/rejected": -1.2657747268676758,
      "step": 4218
    },
    {
      "epoch": 0.552089637687086,
      "grad_norm": 5.450617790222168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30207428336143494,
      "logits/rejected": 0.10012572258710861,
      "logps/chosen": -532.5704956054688,
      "logps/rejected": -400.72064208984375,
      "loss": 0.3513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44654786586761475,
      "rewards/margins": 1.2851803302764893,
      "rewards/rejected": -0.8386324048042297,
      "step": 4219
    },
    {
      "epoch": 0.5522204956244378,
      "grad_norm": 10.3417329788208,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3210148811340332,
      "logits/rejected": 0.26018357276916504,
      "logps/chosen": -485.427001953125,
      "logps/rejected": -344.0124816894531,
      "loss": 0.7777,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1754017025232315,
      "rewards/margins": 0.16663458943367004,
      "rewards/rejected": -0.34203630685806274,
      "step": 4220
    },
    {
      "epoch": 0.5523513535617894,
      "grad_norm": 8.241795539855957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1054864451289177,
      "logits/rejected": 0.03303295373916626,
      "logps/chosen": -401.6647644042969,
      "logps/rejected": -479.40399169921875,
      "loss": 0.566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08121224492788315,
      "rewards/margins": 0.7653384804725647,
      "rewards/rejected": -0.6841262578964233,
      "step": 4221
    },
    {
      "epoch": 0.5524822114991412,
      "grad_norm": 5.784528732299805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022392019629478455,
      "logits/rejected": 0.04144753888249397,
      "logps/chosen": -550.085693359375,
      "logps/rejected": -508.2139587402344,
      "loss": 0.4943,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2923664450645447,
      "rewards/margins": 0.7748472094535828,
      "rewards/rejected": -1.0672136545181274,
      "step": 4222
    },
    {
      "epoch": 0.552613069436493,
      "grad_norm": 10.494733810424805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07683636993169785,
      "logits/rejected": 0.03475896269083023,
      "logps/chosen": -472.14117431640625,
      "logps/rejected": -505.59375,
      "loss": 0.6744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7109218835830688,
      "rewards/margins": 0.5759689211845398,
      "rewards/rejected": -1.2868907451629639,
      "step": 4223
    },
    {
      "epoch": 0.5527439273738448,
      "grad_norm": 10.947319030761719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1766563355922699,
      "logits/rejected": 0.33664262294769287,
      "logps/chosen": -618.30712890625,
      "logps/rejected": -514.7882080078125,
      "loss": 0.8987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47027474641799927,
      "rewards/margins": 0.20071619749069214,
      "rewards/rejected": -0.6709909439086914,
      "step": 4224
    },
    {
      "epoch": 0.5528747853111965,
      "grad_norm": 8.98931884765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024412114173173904,
      "logits/rejected": -0.28060540556907654,
      "logps/chosen": -528.6712646484375,
      "logps/rejected": -612.3349609375,
      "loss": 0.6025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4803600311279297,
      "rewards/margins": 1.0829222202301025,
      "rewards/rejected": -2.5632822513580322,
      "step": 4225
    },
    {
      "epoch": 0.5530056432485483,
      "grad_norm": 9.707247734069824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06760580837726593,
      "logits/rejected": 0.1967180073261261,
      "logps/chosen": -346.1123352050781,
      "logps/rejected": -470.20953369140625,
      "loss": 0.7195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.341508150100708,
      "rewards/margins": 0.23723506927490234,
      "rewards/rejected": -0.5787431597709656,
      "step": 4226
    },
    {
      "epoch": 0.5531365011859001,
      "grad_norm": 6.479390621185303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4029279053211212,
      "logits/rejected": 0.41623473167419434,
      "logps/chosen": -332.17730712890625,
      "logps/rejected": -323.66839599609375,
      "loss": 0.4769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5987050533294678,
      "rewards/margins": 0.9893665909767151,
      "rewards/rejected": -1.588071584701538,
      "step": 4227
    },
    {
      "epoch": 0.5532673591232519,
      "grad_norm": 15.60100269317627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10437431186437607,
      "logits/rejected": 0.11279501765966415,
      "logps/chosen": -475.3846740722656,
      "logps/rejected": -606.2360229492188,
      "loss": 0.9268,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7637962698936462,
      "rewards/margins": -0.2964004874229431,
      "rewards/rejected": -0.46739575266838074,
      "step": 4228
    },
    {
      "epoch": 0.5533982170606035,
      "grad_norm": 8.225934982299805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10733159631490707,
      "logits/rejected": -0.030152395367622375,
      "logps/chosen": -510.4732666015625,
      "logps/rejected": -414.93524169921875,
      "loss": 0.6323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012369737029075623,
      "rewards/margins": 0.448956698179245,
      "rewards/rejected": -0.4365869164466858,
      "step": 4229
    },
    {
      "epoch": 0.5535290749979553,
      "grad_norm": 6.844602108001709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20660075545310974,
      "logits/rejected": -0.0770227462053299,
      "logps/chosen": -623.5585327148438,
      "logps/rejected": -578.2275390625,
      "loss": 0.4991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00289878249168396,
      "rewards/margins": 0.7069633603096008,
      "rewards/rejected": -0.7040646076202393,
      "step": 4230
    },
    {
      "epoch": 0.5536599329353071,
      "grad_norm": 6.9152302742004395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15027619898319244,
      "logits/rejected": -0.2116786390542984,
      "logps/chosen": -393.3743591308594,
      "logps/rejected": -422.4627380371094,
      "loss": 0.4545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11079844832420349,
      "rewards/margins": 1.2055567502975464,
      "rewards/rejected": -1.094758152961731,
      "step": 4231
    },
    {
      "epoch": 0.5537907908726589,
      "grad_norm": 6.131142616271973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.033928919583559036,
      "logits/rejected": -0.13721583783626556,
      "logps/chosen": -346.69793701171875,
      "logps/rejected": -401.2104797363281,
      "loss": 0.4592,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2741609215736389,
      "rewards/margins": 0.795627772808075,
      "rewards/rejected": -1.0697886943817139,
      "step": 4232
    },
    {
      "epoch": 0.5539216488100106,
      "grad_norm": 4.965118885040283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23071955144405365,
      "logits/rejected": -0.3221770226955414,
      "logps/chosen": -491.2213439941406,
      "logps/rejected": -381.1206970214844,
      "loss": 0.3632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23615682125091553,
      "rewards/margins": 1.8388614654541016,
      "rewards/rejected": -1.602704644203186,
      "step": 4233
    },
    {
      "epoch": 0.5540525067473624,
      "grad_norm": 4.374371528625488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20694847404956818,
      "logits/rejected": -0.11949723958969116,
      "logps/chosen": -594.8677978515625,
      "logps/rejected": -371.6525573730469,
      "loss": 0.2718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19291703402996063,
      "rewards/margins": 1.9118938446044922,
      "rewards/rejected": -2.104810953140259,
      "step": 4234
    },
    {
      "epoch": 0.5541833646847142,
      "grad_norm": 9.987974166870117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040202341973781586,
      "logits/rejected": -0.09448850899934769,
      "logps/chosen": -402.0306091308594,
      "logps/rejected": -387.57659912109375,
      "loss": 0.7952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7974107265472412,
      "rewards/margins": 0.2958826422691345,
      "rewards/rejected": -1.0932934284210205,
      "step": 4235
    },
    {
      "epoch": 0.5543142226220659,
      "grad_norm": 10.206198692321777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05406041443347931,
      "logits/rejected": 0.09013119339942932,
      "logps/chosen": -461.9937744140625,
      "logps/rejected": -474.8896179199219,
      "loss": 0.599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0722790956497192,
      "rewards/margins": 0.6280909180641174,
      "rewards/rejected": -1.700370192527771,
      "step": 4236
    },
    {
      "epoch": 0.5544450805594177,
      "grad_norm": 5.122673034667969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13294991850852966,
      "logits/rejected": 0.1982688009738922,
      "logps/chosen": -464.55682373046875,
      "logps/rejected": -559.9547119140625,
      "loss": 0.3771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.432046115398407,
      "rewards/margins": 1.2064716815948486,
      "rewards/rejected": -0.7744255065917969,
      "step": 4237
    },
    {
      "epoch": 0.5545759384967694,
      "grad_norm": 5.202028274536133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12117300182580948,
      "logits/rejected": 0.13394011557102203,
      "logps/chosen": -486.98065185546875,
      "logps/rejected": -498.6509704589844,
      "loss": 0.3518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.43306371569633484,
      "rewards/margins": 1.2783467769622803,
      "rewards/rejected": -0.8452831506729126,
      "step": 4238
    },
    {
      "epoch": 0.5547067964341212,
      "grad_norm": 8.559374809265137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0626259595155716,
      "logits/rejected": 0.23562850058078766,
      "logps/chosen": -456.4559020996094,
      "logps/rejected": -407.9246520996094,
      "loss": 0.7054,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9810096025466919,
      "rewards/margins": 0.7811946868896484,
      "rewards/rejected": -1.7622040510177612,
      "step": 4239
    },
    {
      "epoch": 0.5548376543714729,
      "grad_norm": 5.28374719619751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04752136021852493,
      "logits/rejected": -0.040897443890571594,
      "logps/chosen": -368.70245361328125,
      "logps/rejected": -400.32000732421875,
      "loss": 0.2664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25727465748786926,
      "rewards/margins": 1.9902466535568237,
      "rewards/rejected": -2.247521162033081,
      "step": 4240
    },
    {
      "epoch": 0.5549685123088247,
      "grad_norm": 23.954364776611328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27037766575813293,
      "logits/rejected": 0.12201954424381256,
      "logps/chosen": -547.3409423828125,
      "logps/rejected": -425.95599365234375,
      "loss": 0.8831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2159643173217773,
      "rewards/margins": 0.5178978443145752,
      "rewards/rejected": -1.7338621616363525,
      "step": 4241
    },
    {
      "epoch": 0.5550993702461765,
      "grad_norm": 8.647283554077148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04966361075639725,
      "logits/rejected": -0.03553348034620285,
      "logps/chosen": -422.2603454589844,
      "logps/rejected": -574.9102172851562,
      "loss": 0.4319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4440050423145294,
      "rewards/margins": 1.013157606124878,
      "rewards/rejected": -1.457162618637085,
      "step": 4242
    },
    {
      "epoch": 0.5552302281835283,
      "grad_norm": 8.292332649230957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1986435055732727,
      "logits/rejected": 0.03614158183336258,
      "logps/chosen": -437.0731201171875,
      "logps/rejected": -463.0977478027344,
      "loss": 0.467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13344936072826385,
      "rewards/margins": 1.2534022331237793,
      "rewards/rejected": -1.1199527978897095,
      "step": 4243
    },
    {
      "epoch": 0.55536108612088,
      "grad_norm": 9.562610626220703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07202142477035522,
      "logits/rejected": -0.06961983442306519,
      "logps/chosen": -447.2210693359375,
      "logps/rejected": -640.5150756835938,
      "loss": 0.8524,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4443971514701843,
      "rewards/margins": 0.11240805685520172,
      "rewards/rejected": -0.5568051934242249,
      "step": 4244
    },
    {
      "epoch": 0.5554919440582318,
      "grad_norm": 8.231084823608398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20173333585262299,
      "logits/rejected": -0.35302239656448364,
      "logps/chosen": -369.5256042480469,
      "logps/rejected": -380.84747314453125,
      "loss": 0.6774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.168047308921814,
      "rewards/margins": 0.47412109375,
      "rewards/rejected": -1.642168402671814,
      "step": 4245
    },
    {
      "epoch": 0.5556228019955836,
      "grad_norm": 3.765611171722412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08192630112171173,
      "logits/rejected": -0.10567966848611832,
      "logps/chosen": -339.4432373046875,
      "logps/rejected": -320.69598388671875,
      "loss": 0.4575,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3011239767074585,
      "rewards/margins": 1.5192979574203491,
      "rewards/rejected": -1.8204219341278076,
      "step": 4246
    },
    {
      "epoch": 0.5557536599329354,
      "grad_norm": 6.522780418395996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18774080276489258,
      "logits/rejected": -0.055229395627975464,
      "logps/chosen": -635.7645874023438,
      "logps/rejected": -587.4517211914062,
      "loss": 0.4491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5903135538101196,
      "rewards/margins": 0.9252637624740601,
      "rewards/rejected": -1.5155773162841797,
      "step": 4247
    },
    {
      "epoch": 0.555884517870287,
      "grad_norm": 7.218527317047119,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07264912128448486,
      "logits/rejected": 0.005153834819793701,
      "logps/chosen": -342.5221252441406,
      "logps/rejected": -357.452880859375,
      "loss": 0.5391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11633414030075073,
      "rewards/margins": 0.9322279691696167,
      "rewards/rejected": -1.0485620498657227,
      "step": 4248
    },
    {
      "epoch": 0.5560153758076388,
      "grad_norm": 7.033857822418213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20450691878795624,
      "logits/rejected": -0.16395282745361328,
      "logps/chosen": -431.526611328125,
      "logps/rejected": -514.915771484375,
      "loss": 0.3574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37913569808006287,
      "rewards/margins": 1.5295761823654175,
      "rewards/rejected": -1.9087119102478027,
      "step": 4249
    },
    {
      "epoch": 0.5561462337449906,
      "grad_norm": 3.1289072036743164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023615825921297073,
      "logits/rejected": 0.16740885376930237,
      "logps/chosen": -320.7900085449219,
      "logps/rejected": -279.4931640625,
      "loss": 0.2222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15037670731544495,
      "rewards/margins": 1.5491584539413452,
      "rewards/rejected": -1.3987817764282227,
      "step": 4250
    },
    {
      "epoch": 0.5562770916823424,
      "grad_norm": 5.307892322540283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09972923249006271,
      "logits/rejected": -0.0854976624250412,
      "logps/chosen": -403.3809814453125,
      "logps/rejected": -347.1517028808594,
      "loss": 0.4148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27952709794044495,
      "rewards/margins": 0.8838891983032227,
      "rewards/rejected": -1.1634161472320557,
      "step": 4251
    },
    {
      "epoch": 0.5564079496196941,
      "grad_norm": 8.778883934020996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2386280596256256,
      "logits/rejected": 0.2073906660079956,
      "logps/chosen": -705.6326904296875,
      "logps/rejected": -465.785400390625,
      "loss": 0.6135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7388246655464172,
      "rewards/margins": 0.6340188980102539,
      "rewards/rejected": -1.3728435039520264,
      "step": 4252
    },
    {
      "epoch": 0.5565388075570459,
      "grad_norm": 9.828642845153809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014888245612382889,
      "logits/rejected": 0.08498228341341019,
      "logps/chosen": -435.86737060546875,
      "logps/rejected": -335.9167175292969,
      "loss": 0.5308,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3855897784233093,
      "rewards/margins": 0.9836021661758423,
      "rewards/rejected": -0.598012387752533,
      "step": 4253
    },
    {
      "epoch": 0.5566696654943977,
      "grad_norm": 5.986912250518799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1782461404800415,
      "logits/rejected": 0.1597500741481781,
      "logps/chosen": -332.43597412109375,
      "logps/rejected": -382.01788330078125,
      "loss": 0.5129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.651662290096283,
      "rewards/margins": 0.7420373558998108,
      "rewards/rejected": -1.3936996459960938,
      "step": 4254
    },
    {
      "epoch": 0.5568005234317495,
      "grad_norm": 4.688689231872559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043084993958473206,
      "logits/rejected": -0.1137164756655693,
      "logps/chosen": -590.6571655273438,
      "logps/rejected": -405.2556457519531,
      "loss": 0.427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10958976298570633,
      "rewards/margins": 1.274266004562378,
      "rewards/rejected": -1.3838557004928589,
      "step": 4255
    },
    {
      "epoch": 0.5569313813691011,
      "grad_norm": 4.063174247741699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14921270310878754,
      "logits/rejected": -0.2715473473072052,
      "logps/chosen": -378.71319580078125,
      "logps/rejected": -365.5414123535156,
      "loss": 0.3231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5027917623519897,
      "rewards/margins": 1.1615701913833618,
      "rewards/rejected": -0.6587784290313721,
      "step": 4256
    },
    {
      "epoch": 0.5570622393064529,
      "grad_norm": 6.002463340759277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15701468288898468,
      "logits/rejected": 0.05462899059057236,
      "logps/chosen": -472.9847106933594,
      "logps/rejected": -436.5339050292969,
      "loss": 0.4293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2775346636772156,
      "rewards/margins": 1.314008116722107,
      "rewards/rejected": -1.5915427207946777,
      "step": 4257
    },
    {
      "epoch": 0.5571930972438047,
      "grad_norm": 7.443337440490723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34038323163986206,
      "logits/rejected": 0.1726572960615158,
      "logps/chosen": -635.1826782226562,
      "logps/rejected": -447.1782531738281,
      "loss": 0.6168,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2213977873325348,
      "rewards/margins": 0.3133041560649872,
      "rewards/rejected": -0.0919063538312912,
      "step": 4258
    },
    {
      "epoch": 0.5573239551811564,
      "grad_norm": 11.022841453552246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17791765928268433,
      "logits/rejected": 4.51505184173584e-05,
      "logps/chosen": -346.3025817871094,
      "logps/rejected": -280.71844482421875,
      "loss": 1.1145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8613025546073914,
      "rewards/margins": -0.1539541333913803,
      "rewards/rejected": -0.7073484659194946,
      "step": 4259
    },
    {
      "epoch": 0.5574548131185082,
      "grad_norm": 5.120370388031006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013894413597881794,
      "logits/rejected": -0.13429516553878784,
      "logps/chosen": -525.3660888671875,
      "logps/rejected": -388.5669860839844,
      "loss": 0.3846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5742328763008118,
      "rewards/margins": 1.3186854124069214,
      "rewards/rejected": -0.7444525957107544,
      "step": 4260
    },
    {
      "epoch": 0.55758567105586,
      "grad_norm": 4.9877448081970215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12067703902721405,
      "logits/rejected": -0.08066947758197784,
      "logps/chosen": -376.174072265625,
      "logps/rejected": -310.56365966796875,
      "loss": 0.4077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4468267560005188,
      "rewards/margins": 1.0188860893249512,
      "rewards/rejected": -1.4657129049301147,
      "step": 4261
    },
    {
      "epoch": 0.5577165289932118,
      "grad_norm": 8.107110023498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14632344245910645,
      "logits/rejected": -0.05510853976011276,
      "logps/chosen": -274.4664001464844,
      "logps/rejected": -299.8466491699219,
      "loss": 0.6376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48302191495895386,
      "rewards/margins": 0.6640942096710205,
      "rewards/rejected": -1.1471160650253296,
      "step": 4262
    },
    {
      "epoch": 0.5578473869305635,
      "grad_norm": 7.286829948425293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12128462642431259,
      "logits/rejected": 0.4445403218269348,
      "logps/chosen": -380.5635070800781,
      "logps/rejected": -438.767578125,
      "loss": 0.5806,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14697131514549255,
      "rewards/margins": 1.0430645942687988,
      "rewards/rejected": -0.8960932493209839,
      "step": 4263
    },
    {
      "epoch": 0.5579782448679153,
      "grad_norm": 6.632074356079102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12471174448728561,
      "logits/rejected": -0.11266929656267166,
      "logps/chosen": -302.0420227050781,
      "logps/rejected": -234.66644287109375,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43786850571632385,
      "rewards/margins": 0.5785235166549683,
      "rewards/rejected": -1.0163919925689697,
      "step": 4264
    },
    {
      "epoch": 0.558109102805267,
      "grad_norm": 6.124933242797852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24594146013259888,
      "logits/rejected": 0.1378641277551651,
      "logps/chosen": -385.15411376953125,
      "logps/rejected": -297.75225830078125,
      "loss": 0.5329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13077183067798615,
      "rewards/margins": 0.3985849618911743,
      "rewards/rejected": -0.5293567776679993,
      "step": 4265
    },
    {
      "epoch": 0.5582399607426188,
      "grad_norm": 10.169050216674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14984065294265747,
      "logits/rejected": -0.3677710294723511,
      "logps/chosen": -410.34234619140625,
      "logps/rejected": -366.12811279296875,
      "loss": 0.729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1805052757263184,
      "rewards/margins": 0.410652756690979,
      "rewards/rejected": -1.5911579132080078,
      "step": 4266
    },
    {
      "epoch": 0.5583708186799705,
      "grad_norm": 5.819157123565674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1666424721479416,
      "logits/rejected": -0.013710454106330872,
      "logps/chosen": -457.3708190917969,
      "logps/rejected": -487.303466796875,
      "loss": 0.427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7277679443359375,
      "rewards/margins": 0.9316810965538025,
      "rewards/rejected": -1.6594491004943848,
      "step": 4267
    },
    {
      "epoch": 0.5585016766173223,
      "grad_norm": 7.491123199462891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10147465765476227,
      "logits/rejected": 0.13800965249538422,
      "logps/chosen": -378.2271728515625,
      "logps/rejected": -272.8604736328125,
      "loss": 0.6969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6174707412719727,
      "rewards/margins": 0.42963528633117676,
      "rewards/rejected": -1.0471060276031494,
      "step": 4268
    },
    {
      "epoch": 0.5586325345546741,
      "grad_norm": 8.447287559509277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03427135944366455,
      "logits/rejected": -0.014358580112457275,
      "logps/chosen": -413.4486999511719,
      "logps/rejected": -416.2360534667969,
      "loss": 0.6373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8612756729125977,
      "rewards/margins": 0.6653755903244019,
      "rewards/rejected": -1.52665114402771,
      "step": 4269
    },
    {
      "epoch": 0.5587633924920259,
      "grad_norm": 17.449275970458984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.033170200884342194,
      "logits/rejected": 0.14400166273117065,
      "logps/chosen": -578.670654296875,
      "logps/rejected": -390.2532043457031,
      "loss": 0.9051,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6540748476982117,
      "rewards/margins": -0.005855560302734375,
      "rewards/rejected": -0.6482193470001221,
      "step": 4270
    },
    {
      "epoch": 0.5588942504293776,
      "grad_norm": 6.473407745361328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25850585103034973,
      "logits/rejected": -0.10629119724035263,
      "logps/chosen": -409.2802429199219,
      "logps/rejected": -431.7716979980469,
      "loss": 0.5074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.25337696075439453,
      "rewards/margins": 0.8955827355384827,
      "rewards/rejected": -0.6422058343887329,
      "step": 4271
    },
    {
      "epoch": 0.5590251083667294,
      "grad_norm": 4.880181789398193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09433788806200027,
      "logits/rejected": 0.033462561666965485,
      "logps/chosen": -485.14898681640625,
      "logps/rejected": -464.33489990234375,
      "loss": 0.3961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11611068248748779,
      "rewards/margins": 0.9251493215560913,
      "rewards/rejected": -1.041260004043579,
      "step": 4272
    },
    {
      "epoch": 0.5591559663040812,
      "grad_norm": 6.766836643218994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3207123577594757,
      "logits/rejected": -0.3522118628025055,
      "logps/chosen": -534.0713500976562,
      "logps/rejected": -484.5774230957031,
      "loss": 0.4008,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1696462780237198,
      "rewards/margins": 0.8969587683677673,
      "rewards/rejected": -1.0666050910949707,
      "step": 4273
    },
    {
      "epoch": 0.559286824241433,
      "grad_norm": 7.41032600402832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24506807327270508,
      "logits/rejected": 0.4114471971988678,
      "logps/chosen": -533.8095092773438,
      "logps/rejected": -497.62335205078125,
      "loss": 0.5564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10234664380550385,
      "rewards/margins": 0.9754606485366821,
      "rewards/rejected": -0.8731139898300171,
      "step": 4274
    },
    {
      "epoch": 0.5594176821787846,
      "grad_norm": 4.613040447235107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10850602388381958,
      "logits/rejected": -0.05665212869644165,
      "logps/chosen": -497.4219055175781,
      "logps/rejected": -304.4919738769531,
      "loss": 0.2933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7459593415260315,
      "rewards/margins": 1.7129632234573364,
      "rewards/rejected": -0.9670039415359497,
      "step": 4275
    },
    {
      "epoch": 0.5595485401161364,
      "grad_norm": 6.3422956466674805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14815247058868408,
      "logits/rejected": 0.1704827845096588,
      "logps/chosen": -341.524658203125,
      "logps/rejected": -369.65057373046875,
      "loss": 0.538,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4158298671245575,
      "rewards/margins": 0.7511717081069946,
      "rewards/rejected": -1.1670016050338745,
      "step": 4276
    },
    {
      "epoch": 0.5596793980534882,
      "grad_norm": 3.1795995235443115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2407117784023285,
      "logits/rejected": 0.2558801472187042,
      "logps/chosen": -445.93475341796875,
      "logps/rejected": -477.81317138671875,
      "loss": 0.1873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8338215351104736,
      "rewards/margins": 2.060114860534668,
      "rewards/rejected": -1.2262933254241943,
      "step": 4277
    },
    {
      "epoch": 0.5598102559908399,
      "grad_norm": 7.603362083435059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09251246601343155,
      "logits/rejected": -0.14694072306156158,
      "logps/chosen": -415.6129150390625,
      "logps/rejected": -384.72296142578125,
      "loss": 0.5438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24567623436450958,
      "rewards/margins": 0.6124988794326782,
      "rewards/rejected": -0.36682260036468506,
      "step": 4278
    },
    {
      "epoch": 0.5599411139281917,
      "grad_norm": 3.2164177894592285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06036530062556267,
      "logits/rejected": -0.08740527927875519,
      "logps/chosen": -312.3843994140625,
      "logps/rejected": -393.8319091796875,
      "loss": 0.2894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3616759181022644,
      "rewards/margins": 1.7489182949066162,
      "rewards/rejected": -2.1105942726135254,
      "step": 4279
    },
    {
      "epoch": 0.5600719718655435,
      "grad_norm": 5.598560333251953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26735323667526245,
      "logits/rejected": -0.05057232826948166,
      "logps/chosen": -300.28338623046875,
      "logps/rejected": -324.5455017089844,
      "loss": 0.3967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.050812751054763794,
      "rewards/margins": 1.2343006134033203,
      "rewards/rejected": -1.2851134538650513,
      "step": 4280
    },
    {
      "epoch": 0.5602028298028953,
      "grad_norm": 6.6203999519348145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12447462975978851,
      "logits/rejected": -0.07777541875839233,
      "logps/chosen": -286.36102294921875,
      "logps/rejected": -393.5289306640625,
      "loss": 0.5035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25797128677368164,
      "rewards/margins": 0.8148964643478394,
      "rewards/rejected": -1.0728678703308105,
      "step": 4281
    },
    {
      "epoch": 0.5603336877402469,
      "grad_norm": 4.550718784332275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12784698605537415,
      "logits/rejected": 0.005683455616235733,
      "logps/chosen": -561.5994262695312,
      "logps/rejected": -443.15032958984375,
      "loss": 0.2139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18574903905391693,
      "rewards/margins": 1.5946648120880127,
      "rewards/rejected": -1.4089157581329346,
      "step": 4282
    },
    {
      "epoch": 0.5604645456775987,
      "grad_norm": 10.395858764648438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2751643657684326,
      "logits/rejected": -0.02743426337838173,
      "logps/chosen": -543.0592041015625,
      "logps/rejected": -307.72259521484375,
      "loss": 0.7729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6905010342597961,
      "rewards/margins": 0.62823086977005,
      "rewards/rejected": -1.3187319040298462,
      "step": 4283
    },
    {
      "epoch": 0.5605954036149505,
      "grad_norm": 5.214422225952148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22563104331493378,
      "logits/rejected": -0.32568085193634033,
      "logps/chosen": -260.79595947265625,
      "logps/rejected": -349.99859619140625,
      "loss": 0.3826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.562838613986969,
      "rewards/margins": 1.67349374294281,
      "rewards/rejected": -2.2363321781158447,
      "step": 4284
    },
    {
      "epoch": 0.5607262615523023,
      "grad_norm": 5.916724681854248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4556805491447449,
      "logits/rejected": 0.24016818404197693,
      "logps/chosen": -475.82208251953125,
      "logps/rejected": -331.4732666015625,
      "loss": 0.3961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6446536779403687,
      "rewards/margins": 1.0375546216964722,
      "rewards/rejected": -0.39290085434913635,
      "step": 4285
    },
    {
      "epoch": 0.560857119489654,
      "grad_norm": 9.348791122436523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15391218662261963,
      "logits/rejected": 0.2092115581035614,
      "logps/chosen": -668.2300415039062,
      "logps/rejected": -433.4834899902344,
      "loss": 0.6078,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06281282007694244,
      "rewards/margins": 0.6906356811523438,
      "rewards/rejected": -0.7534485459327698,
      "step": 4286
    },
    {
      "epoch": 0.5609879774270058,
      "grad_norm": 7.812342643737793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06553468108177185,
      "logits/rejected": -0.08051969110965729,
      "logps/chosen": -358.5746765136719,
      "logps/rejected": -506.62213134765625,
      "loss": 0.5482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4335803985595703,
      "rewards/margins": 0.835791826248169,
      "rewards/rejected": -1.2693722248077393,
      "step": 4287
    },
    {
      "epoch": 0.5611188353643576,
      "grad_norm": 7.010796070098877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01816222071647644,
      "logits/rejected": -0.06865834444761276,
      "logps/chosen": -290.3932189941406,
      "logps/rejected": -274.62640380859375,
      "loss": 0.6514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7581723928451538,
      "rewards/margins": 0.5366607904434204,
      "rewards/rejected": -1.2948331832885742,
      "step": 4288
    },
    {
      "epoch": 0.5612496933017094,
      "grad_norm": 7.31654167175293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08304565399885178,
      "logits/rejected": -0.1655275821685791,
      "logps/chosen": -397.2030029296875,
      "logps/rejected": -447.0581359863281,
      "loss": 0.5447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3502744734287262,
      "rewards/margins": 1.2224955558776855,
      "rewards/rejected": -1.5727699995040894,
      "step": 4289
    },
    {
      "epoch": 0.561380551239061,
      "grad_norm": 4.919383525848389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1048748791217804,
      "logits/rejected": -0.1102994829416275,
      "logps/chosen": -392.08660888671875,
      "logps/rejected": -302.0462646484375,
      "loss": 0.3263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0209629088640213,
      "rewards/margins": 1.194722294807434,
      "rewards/rejected": -1.1737593412399292,
      "step": 4290
    },
    {
      "epoch": 0.5615114091764128,
      "grad_norm": 6.944817543029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3130534589290619,
      "logits/rejected": 0.1293654441833496,
      "logps/chosen": -464.60076904296875,
      "logps/rejected": -455.4176025390625,
      "loss": 0.6449,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013255100697278976,
      "rewards/margins": 0.6570714712142944,
      "rewards/rejected": -0.6703266501426697,
      "step": 4291
    },
    {
      "epoch": 0.5616422671137646,
      "grad_norm": 4.912593364715576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5319744348526001,
      "logits/rejected": 0.3943987786769867,
      "logps/chosen": -313.1297912597656,
      "logps/rejected": -308.5615539550781,
      "loss": 0.3573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5255568027496338,
      "rewards/margins": 1.1600232124328613,
      "rewards/rejected": -0.634466290473938,
      "step": 4292
    },
    {
      "epoch": 0.5617731250511164,
      "grad_norm": 6.78180456161499,
      "learning_rate": 5e-06,
      "logits/chosen": 0.115048848092556,
      "logits/rejected": -0.0320795439183712,
      "logps/chosen": -593.4240112304688,
      "logps/rejected": -363.31988525390625,
      "loss": 0.5711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.845186710357666,
      "rewards/margins": 0.7333674430847168,
      "rewards/rejected": 0.11181925237178802,
      "step": 4293
    },
    {
      "epoch": 0.5619039829884681,
      "grad_norm": 10.413872718811035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24295642971992493,
      "logits/rejected": 0.2293591946363449,
      "logps/chosen": -461.7116394042969,
      "logps/rejected": -433.00433349609375,
      "loss": 1.0235,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0297174453735352,
      "rewards/margins": -0.3622843027114868,
      "rewards/rejected": -0.6674330830574036,
      "step": 4294
    },
    {
      "epoch": 0.5620348409258199,
      "grad_norm": 5.687106132507324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1782870590686798,
      "logits/rejected": 0.15309585630893707,
      "logps/chosen": -368.15313720703125,
      "logps/rejected": -456.8480224609375,
      "loss": 0.3513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2010311484336853,
      "rewards/margins": 1.118334412574768,
      "rewards/rejected": -1.3193655014038086,
      "step": 4295
    },
    {
      "epoch": 0.5621656988631717,
      "grad_norm": 7.485562801361084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022123301401734352,
      "logits/rejected": -0.05929802358150482,
      "logps/chosen": -482.0489807128906,
      "logps/rejected": -470.4240417480469,
      "loss": 0.6242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17042005062103271,
      "rewards/margins": 0.7176070809364319,
      "rewards/rejected": -0.5471870303153992,
      "step": 4296
    },
    {
      "epoch": 0.5622965568005235,
      "grad_norm": 6.905355453491211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18428851664066315,
      "logits/rejected": 0.2352139949798584,
      "logps/chosen": -452.0885314941406,
      "logps/rejected": -408.02783203125,
      "loss": 0.3954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38308602571487427,
      "rewards/margins": 0.9277311563491821,
      "rewards/rejected": -0.5446451306343079,
      "step": 4297
    },
    {
      "epoch": 0.5624274147378752,
      "grad_norm": 7.501194477081299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28782033920288086,
      "logits/rejected": 0.028212644159793854,
      "logps/chosen": -495.96075439453125,
      "logps/rejected": -532.1075439453125,
      "loss": 0.6791,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2133886218070984,
      "rewards/margins": 0.5678155422210693,
      "rewards/rejected": -0.35442695021629333,
      "step": 4298
    },
    {
      "epoch": 0.562558272675227,
      "grad_norm": 6.114475250244141,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020472988486289978,
      "logits/rejected": -0.03777429461479187,
      "logps/chosen": -492.1839599609375,
      "logps/rejected": -503.2395324707031,
      "loss": 0.4735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7717012763023376,
      "rewards/margins": 1.4065399169921875,
      "rewards/rejected": -0.6348387598991394,
      "step": 4299
    },
    {
      "epoch": 0.5626891306125787,
      "grad_norm": 5.399203777313232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28776687383651733,
      "logits/rejected": -0.0068655796349048615,
      "logps/chosen": -433.2244567871094,
      "logps/rejected": -398.6508483886719,
      "loss": 0.3534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24367906153202057,
      "rewards/margins": 1.6049482822418213,
      "rewards/rejected": -1.8486272096633911,
      "step": 4300
    },
    {
      "epoch": 0.5628199885499304,
      "grad_norm": 6.258527755737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3684427738189697,
      "logits/rejected": 0.22196123003959656,
      "logps/chosen": -372.43988037109375,
      "logps/rejected": -516.280029296875,
      "loss": 0.4807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43871140480041504,
      "rewards/margins": 2.0586347579956055,
      "rewards/rejected": -1.61992347240448,
      "step": 4301
    },
    {
      "epoch": 0.5629508464872822,
      "grad_norm": 6.598745346069336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1251615434885025,
      "logits/rejected": 0.24547997117042542,
      "logps/chosen": -346.001708984375,
      "logps/rejected": -374.88873291015625,
      "loss": 0.6559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4491729736328125,
      "rewards/margins": 0.6831084489822388,
      "rewards/rejected": -1.1322814226150513,
      "step": 4302
    },
    {
      "epoch": 0.563081704424634,
      "grad_norm": 7.076175689697266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01854386180639267,
      "logits/rejected": 0.2603481411933899,
      "logps/chosen": -360.71014404296875,
      "logps/rejected": -263.09368896484375,
      "loss": 0.6287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3159072995185852,
      "rewards/margins": 0.6469253897666931,
      "rewards/rejected": -0.9628326892852783,
      "step": 4303
    },
    {
      "epoch": 0.5632125623619858,
      "grad_norm": 6.680875778198242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.342093288898468,
      "logits/rejected": 0.2382616400718689,
      "logps/chosen": -493.35321044921875,
      "logps/rejected": -529.9555053710938,
      "loss": 0.4252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5311118960380554,
      "rewards/margins": 0.9208229780197144,
      "rewards/rejected": -1.451934814453125,
      "step": 4304
    },
    {
      "epoch": 0.5633434202993375,
      "grad_norm": 6.989284992218018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013512506149709225,
      "logits/rejected": -0.16380853950977325,
      "logps/chosen": -545.6475830078125,
      "logps/rejected": -340.74267578125,
      "loss": 0.4959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23926952481269836,
      "rewards/margins": 0.9472634196281433,
      "rewards/rejected": -0.7079939246177673,
      "step": 4305
    },
    {
      "epoch": 0.5634742782366893,
      "grad_norm": 7.670062065124512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22632282972335815,
      "logits/rejected": 0.13378441333770752,
      "logps/chosen": -581.6865844726562,
      "logps/rejected": -438.4363098144531,
      "loss": 0.607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5248609781265259,
      "rewards/margins": 0.7469465732574463,
      "rewards/rejected": -0.22208555042743683,
      "step": 4306
    },
    {
      "epoch": 0.5636051361740411,
      "grad_norm": 5.35919189453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18144989013671875,
      "logits/rejected": 0.11667902767658234,
      "logps/chosen": -510.5314636230469,
      "logps/rejected": -404.3238525390625,
      "loss": 0.1818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3184508085250854,
      "rewards/margins": 2.220564365386963,
      "rewards/rejected": -0.902113676071167,
      "step": 4307
    },
    {
      "epoch": 0.5637359941113929,
      "grad_norm": 10.458356857299805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24369414150714874,
      "logits/rejected": 0.15624530613422394,
      "logps/chosen": -594.805908203125,
      "logps/rejected": -484.1850891113281,
      "loss": 0.7323,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.37422364950180054,
      "rewards/margins": 0.504801332950592,
      "rewards/rejected": -0.13057763874530792,
      "step": 4308
    },
    {
      "epoch": 0.5638668520487445,
      "grad_norm": 6.803004264831543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031226053833961487,
      "logits/rejected": 0.03224635124206543,
      "logps/chosen": -452.365966796875,
      "logps/rejected": -420.2256774902344,
      "loss": 0.51,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4542592763900757,
      "rewards/margins": 0.6630663871765137,
      "rewards/rejected": -0.2088070809841156,
      "step": 4309
    },
    {
      "epoch": 0.5639977099860963,
      "grad_norm": 7.873541355133057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02273215726017952,
      "logits/rejected": -0.051289092749357224,
      "logps/chosen": -431.3680419921875,
      "logps/rejected": -435.4382019042969,
      "loss": 0.721,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18480543792247772,
      "rewards/margins": 0.31125715374946594,
      "rewards/rejected": -0.496062695980072,
      "step": 4310
    },
    {
      "epoch": 0.5641285679234481,
      "grad_norm": 5.95481014251709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07698416709899902,
      "logits/rejected": 0.1274680495262146,
      "logps/chosen": -521.2218627929688,
      "logps/rejected": -429.89813232421875,
      "loss": 0.4401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6328478455543518,
      "rewards/margins": 1.0563818216323853,
      "rewards/rejected": -0.4235340356826782,
      "step": 4311
    },
    {
      "epoch": 0.5642594258607999,
      "grad_norm": 6.520013809204102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004551418125629425,
      "logits/rejected": -0.39853641390800476,
      "logps/chosen": -372.219970703125,
      "logps/rejected": -315.5715026855469,
      "loss": 0.5641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.511020302772522,
      "rewards/margins": 0.5774852633476257,
      "rewards/rejected": -1.0885056257247925,
      "step": 4312
    },
    {
      "epoch": 0.5643902837981516,
      "grad_norm": 10.640777587890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22636249661445618,
      "logits/rejected": 0.06576384603977203,
      "logps/chosen": -578.1351928710938,
      "logps/rejected": -506.18109130859375,
      "loss": 0.6399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25196725130081177,
      "rewards/margins": 0.8800866007804871,
      "rewards/rejected": -1.1320538520812988,
      "step": 4313
    },
    {
      "epoch": 0.5645211417355034,
      "grad_norm": 7.319074630737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2535260319709778,
      "logits/rejected": 0.10896279662847519,
      "logps/chosen": -405.95819091796875,
      "logps/rejected": -539.206298828125,
      "loss": 0.4493,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3677072525024414,
      "rewards/margins": 1.1787331104278564,
      "rewards/rejected": -0.8110257983207703,
      "step": 4314
    },
    {
      "epoch": 0.5646519996728552,
      "grad_norm": 6.344320297241211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04082515835762024,
      "logits/rejected": 0.04768931120634079,
      "logps/chosen": -553.64794921875,
      "logps/rejected": -426.15966796875,
      "loss": 0.6517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6936476230621338,
      "rewards/margins": 1.308455228805542,
      "rewards/rejected": -0.6148074865341187,
      "step": 4315
    },
    {
      "epoch": 0.564782857610207,
      "grad_norm": 7.571831703186035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.033450648188591,
      "logits/rejected": 0.05309056118130684,
      "logps/chosen": -514.6046752929688,
      "logps/rejected": -557.39794921875,
      "loss": 0.6662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13123837113380432,
      "rewards/margins": 0.7251247763633728,
      "rewards/rejected": -0.5938863754272461,
      "step": 4316
    },
    {
      "epoch": 0.5649137155475586,
      "grad_norm": 3.4844799041748047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10463591665029526,
      "logits/rejected": -0.14647775888442993,
      "logps/chosen": -337.90185546875,
      "logps/rejected": -353.3503112792969,
      "loss": 0.3132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7242687940597534,
      "rewards/margins": 1.2783015966415405,
      "rewards/rejected": -0.5540329217910767,
      "step": 4317
    },
    {
      "epoch": 0.5650445734849104,
      "grad_norm": 7.697333335876465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18623660504817963,
      "logits/rejected": 0.05231056362390518,
      "logps/chosen": -433.89324951171875,
      "logps/rejected": -389.07354736328125,
      "loss": 0.5386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4774377942085266,
      "rewards/margins": 0.7609564661979675,
      "rewards/rejected": -0.28351864218711853,
      "step": 4318
    },
    {
      "epoch": 0.5651754314222622,
      "grad_norm": 5.97929048538208,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39087343215942383,
      "logits/rejected": 0.3260934352874756,
      "logps/chosen": -522.859375,
      "logps/rejected": -473.43426513671875,
      "loss": 0.3813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1745960712432861,
      "rewards/margins": 1.1673632860183716,
      "rewards/rejected": 0.007232666015625,
      "step": 4319
    },
    {
      "epoch": 0.565306289359614,
      "grad_norm": 5.140893459320068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053568027913570404,
      "logits/rejected": -0.1313818395137787,
      "logps/chosen": -377.1411437988281,
      "logps/rejected": -525.1780395507812,
      "loss": 0.2768,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5013529062271118,
      "rewards/margins": 1.7526129484176636,
      "rewards/rejected": -1.2512600421905518,
      "step": 4320
    },
    {
      "epoch": 0.5654371472969657,
      "grad_norm": 10.444482803344727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2974144518375397,
      "logits/rejected": 0.009281566366553307,
      "logps/chosen": -399.3327331542969,
      "logps/rejected": -337.9353942871094,
      "loss": 0.7958,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18161489069461823,
      "rewards/margins": 0.3002583086490631,
      "rewards/rejected": -0.1186433881521225,
      "step": 4321
    },
    {
      "epoch": 0.5655680052343175,
      "grad_norm": 9.973433494567871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.051432251930236816,
      "logits/rejected": -0.1241627037525177,
      "logps/chosen": -321.9255676269531,
      "logps/rejected": -271.3014831542969,
      "loss": 0.8168,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14216041564941406,
      "rewards/margins": -0.02751270681619644,
      "rewards/rejected": 0.1696731448173523,
      "step": 4322
    },
    {
      "epoch": 0.5656988631716693,
      "grad_norm": 6.349825859069824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2898058295249939,
      "logits/rejected": 0.007013857364654541,
      "logps/chosen": -454.2799377441406,
      "logps/rejected": -418.3074645996094,
      "loss": 0.5573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.248679518699646,
      "rewards/margins": 0.7831933498382568,
      "rewards/rejected": -0.5345138311386108,
      "step": 4323
    },
    {
      "epoch": 0.565829721109021,
      "grad_norm": 10.078292846679688,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21016699075698853,
      "logits/rejected": -0.016589362174272537,
      "logps/chosen": -541.7772216796875,
      "logps/rejected": -451.85662841796875,
      "loss": 0.8886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1900488138198853,
      "rewards/margins": 0.38399049639701843,
      "rewards/rejected": -1.574039340019226,
      "step": 4324
    },
    {
      "epoch": 0.5659605790463728,
      "grad_norm": 8.4751558303833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06349121779203415,
      "logits/rejected": -0.014367522671818733,
      "logps/chosen": -437.0877685546875,
      "logps/rejected": -430.87054443359375,
      "loss": 0.7659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5498793125152588,
      "rewards/margins": 0.6222220063209534,
      "rewards/rejected": -1.1721012592315674,
      "step": 4325
    },
    {
      "epoch": 0.5660914369837245,
      "grad_norm": 10.578458786010742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0796755850315094,
      "logits/rejected": 0.13082414865493774,
      "logps/chosen": -481.783935546875,
      "logps/rejected": -427.4758605957031,
      "loss": 0.6955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6686971783638,
      "rewards/margins": 0.5092357397079468,
      "rewards/rejected": -1.1779329776763916,
      "step": 4326
    },
    {
      "epoch": 0.5662222949210763,
      "grad_norm": 4.342067241668701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21709251403808594,
      "logits/rejected": -0.06766930222511292,
      "logps/chosen": -552.480224609375,
      "logps/rejected": -447.07879638671875,
      "loss": 0.249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8101760745048523,
      "rewards/margins": 1.629324197769165,
      "rewards/rejected": -0.819148063659668,
      "step": 4327
    },
    {
      "epoch": 0.566353152858428,
      "grad_norm": 8.946096420288086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21278458833694458,
      "logits/rejected": 0.0289623960852623,
      "logps/chosen": -537.3705444335938,
      "logps/rejected": -588.7981567382812,
      "loss": 0.7883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4966014623641968,
      "rewards/margins": 0.5554848909378052,
      "rewards/rejected": -1.0520864725112915,
      "step": 4328
    },
    {
      "epoch": 0.5664840107957798,
      "grad_norm": 7.626243591308594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07144525647163391,
      "logits/rejected": 0.12659674882888794,
      "logps/chosen": -560.8046264648438,
      "logps/rejected": -558.6094360351562,
      "loss": 0.5618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4090646803379059,
      "rewards/margins": 0.8460233807563782,
      "rewards/rejected": -0.4369587302207947,
      "step": 4329
    },
    {
      "epoch": 0.5666148687331316,
      "grad_norm": 6.356847763061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0369495190680027,
      "logits/rejected": 0.0843500941991806,
      "logps/chosen": -490.06072998046875,
      "logps/rejected": -596.82275390625,
      "loss": 0.3868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13384738564491272,
      "rewards/margins": 1.4722328186035156,
      "rewards/rejected": -1.3383854627609253,
      "step": 4330
    },
    {
      "epoch": 0.5667457266704834,
      "grad_norm": 4.290992259979248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12145476043224335,
      "logits/rejected": 0.17265650629997253,
      "logps/chosen": -428.15008544921875,
      "logps/rejected": -555.7005615234375,
      "loss": 0.2876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8986433148384094,
      "rewards/margins": 1.6655734777450562,
      "rewards/rejected": -0.7669301629066467,
      "step": 4331
    },
    {
      "epoch": 0.5668765846078351,
      "grad_norm": 7.352603912353516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.053820300847291946,
      "logits/rejected": 0.18953461945056915,
      "logps/chosen": -340.3323974609375,
      "logps/rejected": -416.33380126953125,
      "loss": 0.5507,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031502336263656616,
      "rewards/margins": 0.5057664513587952,
      "rewards/rejected": -0.537268877029419,
      "step": 4332
    },
    {
      "epoch": 0.5670074425451869,
      "grad_norm": 6.696225643157959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07503513991832733,
      "logits/rejected": -0.3032512664794922,
      "logps/chosen": -400.80316162109375,
      "logps/rejected": -415.5622253417969,
      "loss": 0.4363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03077620267868042,
      "rewards/margins": 1.0507416725158691,
      "rewards/rejected": -1.0199655294418335,
      "step": 4333
    },
    {
      "epoch": 0.5671383004825387,
      "grad_norm": 11.999016761779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037051670253276825,
      "logits/rejected": -0.045180466026067734,
      "logps/chosen": -398.13043212890625,
      "logps/rejected": -370.4194030761719,
      "loss": 1.0781,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003269195556640625,
      "rewards/margins": -0.4258726239204407,
      "rewards/rejected": 0.4291417598724365,
      "step": 4334
    },
    {
      "epoch": 0.5672691584198905,
      "grad_norm": 8.744828224182129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08673524111509323,
      "logits/rejected": 0.15694263577461243,
      "logps/chosen": -509.86932373046875,
      "logps/rejected": -554.7520751953125,
      "loss": 0.6614,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4919136166572571,
      "rewards/margins": 0.5770221948623657,
      "rewards/rejected": -1.068935751914978,
      "step": 4335
    },
    {
      "epoch": 0.5674000163572421,
      "grad_norm": 6.118948459625244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10474035143852234,
      "logits/rejected": 0.21914871037006378,
      "logps/chosen": -299.260986328125,
      "logps/rejected": -424.65203857421875,
      "loss": 0.3174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5689197778701782,
      "rewards/margins": 1.5920052528381348,
      "rewards/rejected": -2.1609249114990234,
      "step": 4336
    },
    {
      "epoch": 0.5675308742945939,
      "grad_norm": 7.671947479248047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19786103069782257,
      "logits/rejected": -0.03911525756120682,
      "logps/chosen": -493.2462463378906,
      "logps/rejected": -417.2444152832031,
      "loss": 0.6788,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0239307880401611,
      "rewards/margins": 0.8750239610671997,
      "rewards/rejected": 0.14890670776367188,
      "step": 4337
    },
    {
      "epoch": 0.5676617322319457,
      "grad_norm": 7.124945163726807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1442943811416626,
      "logits/rejected": -0.1759311854839325,
      "logps/chosen": -261.9645080566406,
      "logps/rejected": -310.65728759765625,
      "loss": 0.398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.015278458595275879,
      "rewards/margins": 2.028160572052002,
      "rewards/rejected": -2.0434391498565674,
      "step": 4338
    },
    {
      "epoch": 0.5677925901692975,
      "grad_norm": 5.276613235473633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024498505517840385,
      "logits/rejected": 0.05279053747653961,
      "logps/chosen": -764.8916625976562,
      "logps/rejected": -505.91107177734375,
      "loss": 0.4116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5464174151420593,
      "rewards/margins": 0.8981788754463196,
      "rewards/rejected": -0.35176149010658264,
      "step": 4339
    },
    {
      "epoch": 0.5679234481066492,
      "grad_norm": 11.300434112548828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0956055149435997,
      "logits/rejected": -0.04456159472465515,
      "logps/chosen": -435.7861328125,
      "logps/rejected": -286.4173583984375,
      "loss": 0.9072,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4479145109653473,
      "rewards/margins": 0.023535311222076416,
      "rewards/rejected": -0.4714498817920685,
      "step": 4340
    },
    {
      "epoch": 0.568054306044001,
      "grad_norm": 8.14682674407959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.038845375180244446,
      "logits/rejected": -0.10995031893253326,
      "logps/chosen": -519.0794677734375,
      "logps/rejected": -633.168212890625,
      "loss": 0.6097,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0414680652320385,
      "rewards/margins": 0.8580532670021057,
      "rewards/rejected": -0.8995212316513062,
      "step": 4341
    },
    {
      "epoch": 0.5681851639813528,
      "grad_norm": 5.101573467254639,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020870987325906754,
      "logits/rejected": -0.07131703943014145,
      "logps/chosen": -454.3227233886719,
      "logps/rejected": -445.3592529296875,
      "loss": 0.2795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9678995609283447,
      "rewards/margins": 2.1297945976257324,
      "rewards/rejected": -1.1618951559066772,
      "step": 4342
    },
    {
      "epoch": 0.5683160219187046,
      "grad_norm": 11.001199722290039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03842553496360779,
      "logits/rejected": -0.10632862895727158,
      "logps/chosen": -462.9061584472656,
      "logps/rejected": -548.6246948242188,
      "loss": 0.7634,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3134802281856537,
      "rewards/margins": 0.16359898447990417,
      "rewards/rejected": -0.47707921266555786,
      "step": 4343
    },
    {
      "epoch": 0.5684468798560562,
      "grad_norm": 6.297797679901123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08472494035959244,
      "logits/rejected": 0.05880090594291687,
      "logps/chosen": -449.376953125,
      "logps/rejected": -572.1928100585938,
      "loss": 0.4743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21738795936107635,
      "rewards/margins": 1.3341492414474487,
      "rewards/rejected": -1.1167612075805664,
      "step": 4344
    },
    {
      "epoch": 0.568577737793408,
      "grad_norm": 6.300515174865723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26308348774909973,
      "logits/rejected": 0.18026301264762878,
      "logps/chosen": -582.6630249023438,
      "logps/rejected": -602.3853759765625,
      "loss": 0.3461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.367430329322815,
      "rewards/margins": 1.682641625404358,
      "rewards/rejected": -0.31521111726760864,
      "step": 4345
    },
    {
      "epoch": 0.5687085957307598,
      "grad_norm": 6.413415431976318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1037769690155983,
      "logits/rejected": -0.013785980641841888,
      "logps/chosen": -458.255859375,
      "logps/rejected": -502.62689208984375,
      "loss": 0.3646,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2042408287525177,
      "rewards/margins": 1.1694704294204712,
      "rewards/rejected": -1.3737112283706665,
      "step": 4346
    },
    {
      "epoch": 0.5688394536681115,
      "grad_norm": 9.728543281555176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20169781148433685,
      "logits/rejected": 0.21825583279132843,
      "logps/chosen": -374.8797912597656,
      "logps/rejected": -447.9250793457031,
      "loss": 0.7141,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00955219566822052,
      "rewards/margins": 0.3676426410675049,
      "rewards/rejected": -0.3580904006958008,
      "step": 4347
    },
    {
      "epoch": 0.5689703116054633,
      "grad_norm": 9.689957618713379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06930588185787201,
      "logits/rejected": -0.006145812571048737,
      "logps/chosen": -431.7065124511719,
      "logps/rejected": -352.8758850097656,
      "loss": 0.7773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.44558316469192505,
      "rewards/margins": 0.220911905169487,
      "rewards/rejected": -0.6664950251579285,
      "step": 4348
    },
    {
      "epoch": 0.5691011695428151,
      "grad_norm": 5.884007930755615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06965004652738571,
      "logits/rejected": 0.02440664730966091,
      "logps/chosen": -409.447998046875,
      "logps/rejected": -473.3993225097656,
      "loss": 0.461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011784940958023071,
      "rewards/margins": 0.9285438656806946,
      "rewards/rejected": -0.9167588949203491,
      "step": 4349
    },
    {
      "epoch": 0.5692320274801669,
      "grad_norm": 7.937836170196533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18338607251644135,
      "logits/rejected": -0.10488241910934448,
      "logps/chosen": -496.04095458984375,
      "logps/rejected": -394.19281005859375,
      "loss": 0.597,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44114378094673157,
      "rewards/margins": 0.6577686071395874,
      "rewards/rejected": -1.0989124774932861,
      "step": 4350
    },
    {
      "epoch": 0.5693628854175186,
      "grad_norm": 7.111383438110352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08446208387613297,
      "logits/rejected": 0.027852389961481094,
      "logps/chosen": -510.8214111328125,
      "logps/rejected": -541.4663696289062,
      "loss": 0.4685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22179032862186432,
      "rewards/margins": 0.7630851864814758,
      "rewards/rejected": -0.984875500202179,
      "step": 4351
    },
    {
      "epoch": 0.5694937433548704,
      "grad_norm": 6.300292015075684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16559021174907684,
      "logits/rejected": 0.184803768992424,
      "logps/chosen": -603.6947021484375,
      "logps/rejected": -473.473388671875,
      "loss": 0.4375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09495735168457031,
      "rewards/margins": 0.9094968438148499,
      "rewards/rejected": -1.004454255104065,
      "step": 4352
    },
    {
      "epoch": 0.5696246012922221,
      "grad_norm": 4.179262638092041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1526617705821991,
      "logits/rejected": 0.07030138373374939,
      "logps/chosen": -482.2750549316406,
      "logps/rejected": -471.87506103515625,
      "loss": 0.2602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3527641296386719,
      "rewards/margins": 1.8247586488723755,
      "rewards/rejected": -1.471994400024414,
      "step": 4353
    },
    {
      "epoch": 0.5697554592295739,
      "grad_norm": 6.065284252166748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09850618243217468,
      "logits/rejected": -0.07545247673988342,
      "logps/chosen": -499.9378662109375,
      "logps/rejected": -530.0597534179688,
      "loss": 0.425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2368936836719513,
      "rewards/margins": 1.509339690208435,
      "rewards/rejected": -1.746233344078064,
      "step": 4354
    },
    {
      "epoch": 0.5698863171669256,
      "grad_norm": 5.672552585601807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3268025517463684,
      "logits/rejected": 0.05113327503204346,
      "logps/chosen": -440.1416320800781,
      "logps/rejected": -468.944580078125,
      "loss": 0.517,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2608869671821594,
      "rewards/margins": 1.1557645797729492,
      "rewards/rejected": -1.4166514873504639,
      "step": 4355
    },
    {
      "epoch": 0.5700171751042774,
      "grad_norm": 4.949565410614014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1436794102191925,
      "logits/rejected": 0.07114341109991074,
      "logps/chosen": -443.04010009765625,
      "logps/rejected": -422.6698303222656,
      "loss": 0.4781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0767677053809166,
      "rewards/margins": 0.7711114883422852,
      "rewards/rejected": -0.8478792905807495,
      "step": 4356
    },
    {
      "epoch": 0.5701480330416292,
      "grad_norm": 10.406388282775879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.184735506772995,
      "logits/rejected": 0.16173626482486725,
      "logps/chosen": -366.8902282714844,
      "logps/rejected": -447.72857666015625,
      "loss": 0.8783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6628902554512024,
      "rewards/margins": 0.11488364636898041,
      "rewards/rejected": 0.5480066537857056,
      "step": 4357
    },
    {
      "epoch": 0.570278890978981,
      "grad_norm": 6.2603678703308105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24489866197109222,
      "logits/rejected": 0.20753231644630432,
      "logps/chosen": -524.3968505859375,
      "logps/rejected": -465.1226806640625,
      "loss": 0.4412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.44824928045272827,
      "rewards/margins": 0.8130972385406494,
      "rewards/rejected": -0.36484798789024353,
      "step": 4358
    },
    {
      "epoch": 0.5704097489163327,
      "grad_norm": 6.538767337799072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06575517356395721,
      "logits/rejected": 0.0980394259095192,
      "logps/chosen": -645.0032958984375,
      "logps/rejected": -508.75531005859375,
      "loss": 0.366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25171488523483276,
      "rewards/margins": 1.5667680501937866,
      "rewards/rejected": -1.8184829950332642,
      "step": 4359
    },
    {
      "epoch": 0.5705406068536845,
      "grad_norm": 5.5272722244262695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09985283762216568,
      "logits/rejected": -0.307992160320282,
      "logps/chosen": -428.13726806640625,
      "logps/rejected": -385.49786376953125,
      "loss": 0.4038,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6381847858428955,
      "rewards/margins": 1.086144208908081,
      "rewards/rejected": -1.724328875541687,
      "step": 4360
    },
    {
      "epoch": 0.5706714647910363,
      "grad_norm": 7.856658458709717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1272149682044983,
      "logits/rejected": 0.13112732768058777,
      "logps/chosen": -279.40997314453125,
      "logps/rejected": -370.32525634765625,
      "loss": 0.6608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6075478792190552,
      "rewards/margins": 0.49502304196357727,
      "rewards/rejected": -1.1025710105895996,
      "step": 4361
    },
    {
      "epoch": 0.570802322728388,
      "grad_norm": 5.246636390686035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2777431309223175,
      "logits/rejected": 0.41049930453300476,
      "logps/chosen": -372.25701904296875,
      "logps/rejected": -520.8966064453125,
      "loss": 0.3221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2736193537712097,
      "rewards/margins": 1.343881368637085,
      "rewards/rejected": -1.6175007820129395,
      "step": 4362
    },
    {
      "epoch": 0.5709331806657397,
      "grad_norm": 7.210010528564453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18664129078388214,
      "logits/rejected": -0.1659214347600937,
      "logps/chosen": -393.48394775390625,
      "logps/rejected": -388.83441162109375,
      "loss": 0.4132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3681718707084656,
      "rewards/margins": 0.8412094712257385,
      "rewards/rejected": -1.209381341934204,
      "step": 4363
    },
    {
      "epoch": 0.5710640386030915,
      "grad_norm": 5.700775146484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3430328369140625,
      "logits/rejected": -0.22037170827388763,
      "logps/chosen": -423.822509765625,
      "logps/rejected": -418.1829833984375,
      "loss": 0.3685,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7135643362998962,
      "rewards/margins": 1.077294945716858,
      "rewards/rejected": -0.3637305498123169,
      "step": 4364
    },
    {
      "epoch": 0.5711948965404433,
      "grad_norm": 9.629378318786621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1415701061487198,
      "logits/rejected": -0.496137797832489,
      "logps/chosen": -459.271728515625,
      "logps/rejected": -426.13543701171875,
      "loss": 0.7868,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6893301606178284,
      "rewards/margins": 0.4886636435985565,
      "rewards/rejected": -1.1779937744140625,
      "step": 4365
    },
    {
      "epoch": 0.571325754477795,
      "grad_norm": 4.104284286499023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16811791062355042,
      "logits/rejected": 0.04442153871059418,
      "logps/chosen": -552.3955078125,
      "logps/rejected": -607.9144897460938,
      "loss": 0.2291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2598772048950195,
      "rewards/margins": 1.583009958267212,
      "rewards/rejected": -0.3231326937675476,
      "step": 4366
    },
    {
      "epoch": 0.5714566124151468,
      "grad_norm": 9.231154441833496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29699400067329407,
      "logits/rejected": 0.16228172183036804,
      "logps/chosen": -597.1272583007812,
      "logps/rejected": -427.4100646972656,
      "loss": 0.5971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9665542840957642,
      "rewards/margins": 1.299962043762207,
      "rewards/rejected": -0.33340778946876526,
      "step": 4367
    },
    {
      "epoch": 0.5715874703524986,
      "grad_norm": 4.591809272766113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20336486399173737,
      "logits/rejected": -0.39912325143814087,
      "logps/chosen": -314.75457763671875,
      "logps/rejected": -467.5023498535156,
      "loss": 0.3371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08515510708093643,
      "rewards/margins": 1.1755025386810303,
      "rewards/rejected": -1.0903475284576416,
      "step": 4368
    },
    {
      "epoch": 0.5717183282898504,
      "grad_norm": 11.502029418945312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2881789803504944,
      "logits/rejected": -0.3210446834564209,
      "logps/chosen": -478.0562744140625,
      "logps/rejected": -371.4245910644531,
      "loss": 0.415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02378334477543831,
      "rewards/margins": 1.0881543159484863,
      "rewards/rejected": -1.0643709897994995,
      "step": 4369
    },
    {
      "epoch": 0.571849186227202,
      "grad_norm": 6.837249755859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11922064423561096,
      "logits/rejected": -0.17319358885288239,
      "logps/chosen": -356.64642333984375,
      "logps/rejected": -361.0991516113281,
      "loss": 0.4383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3388538360595703,
      "rewards/margins": 1.1451797485351562,
      "rewards/rejected": -0.8063259720802307,
      "step": 4370
    },
    {
      "epoch": 0.5719800441645538,
      "grad_norm": 7.047171592712402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014513107016682625,
      "logits/rejected": 0.1425464004278183,
      "logps/chosen": -294.4753723144531,
      "logps/rejected": -397.330078125,
      "loss": 0.7201,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11054688692092896,
      "rewards/margins": 0.12180731445550919,
      "rewards/rejected": -0.01126040518283844,
      "step": 4371
    },
    {
      "epoch": 0.5721109021019056,
      "grad_norm": 5.968695163726807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15692514181137085,
      "logits/rejected": 0.2638624608516693,
      "logps/chosen": -352.4793701171875,
      "logps/rejected": -418.89007568359375,
      "loss": 0.3936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5249658226966858,
      "rewards/margins": 1.0678333044052124,
      "rewards/rejected": -0.5428675413131714,
      "step": 4372
    },
    {
      "epoch": 0.5722417600392574,
      "grad_norm": 10.69293212890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0718521922826767,
      "logits/rejected": -0.023718034848570824,
      "logps/chosen": -536.7310791015625,
      "logps/rejected": -462.4100036621094,
      "loss": 0.7993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020931974053382874,
      "rewards/margins": 0.41692811250686646,
      "rewards/rejected": -0.39599609375,
      "step": 4373
    },
    {
      "epoch": 0.5723726179766091,
      "grad_norm": 4.537838459014893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21408775448799133,
      "logits/rejected": -0.07303513586521149,
      "logps/chosen": -322.430419921875,
      "logps/rejected": -297.6800231933594,
      "loss": 0.4502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23784007132053375,
      "rewards/margins": 1.0629265308380127,
      "rewards/rejected": -1.3007667064666748,
      "step": 4374
    },
    {
      "epoch": 0.5725034759139609,
      "grad_norm": 4.172410488128662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23293918371200562,
      "logits/rejected": 0.18845723569393158,
      "logps/chosen": -550.4859619140625,
      "logps/rejected": -470.5091247558594,
      "loss": 0.2456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6704069375991821,
      "rewards/margins": 1.8314714431762695,
      "rewards/rejected": -1.1610643863677979,
      "step": 4375
    },
    {
      "epoch": 0.5726343338513127,
      "grad_norm": 13.47460651397705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28161847591400146,
      "logits/rejected": 0.26486098766326904,
      "logps/chosen": -380.2185363769531,
      "logps/rejected": -362.380859375,
      "loss": 1.0424,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7746641635894775,
      "rewards/margins": 0.3486085534095764,
      "rewards/rejected": -1.1232726573944092,
      "step": 4376
    },
    {
      "epoch": 0.5727651917886645,
      "grad_norm": 4.400046348571777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.281809002161026,
      "logits/rejected": 0.22676023840904236,
      "logps/chosen": -363.38055419921875,
      "logps/rejected": -423.09637451171875,
      "loss": 0.3853,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01990509033203125,
      "rewards/margins": 1.0220584869384766,
      "rewards/rejected": -1.0419635772705078,
      "step": 4377
    },
    {
      "epoch": 0.5728960497260162,
      "grad_norm": 6.186535835266113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42485716938972473,
      "logits/rejected": -0.3264195919036865,
      "logps/chosen": -342.2067565917969,
      "logps/rejected": -402.56134033203125,
      "loss": 0.4037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18775655329227448,
      "rewards/margins": 0.9327191114425659,
      "rewards/rejected": -0.7449625730514526,
      "step": 4378
    },
    {
      "epoch": 0.573026907663368,
      "grad_norm": 5.067683219909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14660429954528809,
      "logits/rejected": -0.20431824028491974,
      "logps/chosen": -464.449462890625,
      "logps/rejected": -434.7910461425781,
      "loss": 0.4116,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6019365191459656,
      "rewards/margins": 1.3011469841003418,
      "rewards/rejected": -1.903083324432373,
      "step": 4379
    },
    {
      "epoch": 0.5731577656007197,
      "grad_norm": 10.027426719665527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38316506147384644,
      "logits/rejected": 0.22436988353729248,
      "logps/chosen": -370.11029052734375,
      "logps/rejected": -323.2906494140625,
      "loss": 0.9824,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12888450920581818,
      "rewards/margins": -0.13038389384746552,
      "rewards/rejected": 0.0014993660151958466,
      "step": 4380
    },
    {
      "epoch": 0.5732886235380715,
      "grad_norm": 8.405017852783203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2671787440776825,
      "logits/rejected": 0.38314926624298096,
      "logps/chosen": -428.6871032714844,
      "logps/rejected": -520.7830810546875,
      "loss": 0.6849,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5765685439109802,
      "rewards/margins": 0.4096149802207947,
      "rewards/rejected": 0.1669536530971527,
      "step": 4381
    },
    {
      "epoch": 0.5734194814754232,
      "grad_norm": 15.530986785888672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020896613597869873,
      "logits/rejected": -0.02588672749698162,
      "logps/chosen": -502.5,
      "logps/rejected": -389.1180725097656,
      "loss": 0.9449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3845823407173157,
      "rewards/margins": 0.7186672687530518,
      "rewards/rejected": -1.1032495498657227,
      "step": 4382
    },
    {
      "epoch": 0.573550339412775,
      "grad_norm": 9.917685508728027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03232505917549133,
      "logits/rejected": -0.3886786699295044,
      "logps/chosen": -443.9234619140625,
      "logps/rejected": -495.2652893066406,
      "loss": 0.6287,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8578633069992065,
      "rewards/margins": 0.2866363227367401,
      "rewards/rejected": -1.1444995403289795,
      "step": 4383
    },
    {
      "epoch": 0.5736811973501268,
      "grad_norm": 6.916865348815918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04742279276251793,
      "logits/rejected": -0.015588857233524323,
      "logps/chosen": -391.9617919921875,
      "logps/rejected": -347.3746032714844,
      "loss": 0.5454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2772918939590454,
      "rewards/margins": 0.6266320943832397,
      "rewards/rejected": -0.9039239883422852,
      "step": 4384
    },
    {
      "epoch": 0.5738120552874786,
      "grad_norm": 9.885397911071777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03388100862503052,
      "logits/rejected": 0.0029621869325637817,
      "logps/chosen": -498.8898010253906,
      "logps/rejected": -451.20281982421875,
      "loss": 0.7966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0120958089828491,
      "rewards/margins": 0.6154930591583252,
      "rewards/rejected": -1.6275889873504639,
      "step": 4385
    },
    {
      "epoch": 0.5739429132248303,
      "grad_norm": 6.450091361999512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11389977484941483,
      "logits/rejected": -0.009056039154529572,
      "logps/chosen": -297.14312744140625,
      "logps/rejected": -371.975830078125,
      "loss": 0.4304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7926521301269531,
      "rewards/margins": 1.1823639869689941,
      "rewards/rejected": -1.9750161170959473,
      "step": 4386
    },
    {
      "epoch": 0.5740737711621821,
      "grad_norm": 5.434131622314453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28057006001472473,
      "logits/rejected": 0.07890769839286804,
      "logps/chosen": -359.955078125,
      "logps/rejected": -490.6778564453125,
      "loss": 0.3562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.010970771312713623,
      "rewards/margins": 1.4968372583389282,
      "rewards/rejected": -1.507807970046997,
      "step": 4387
    },
    {
      "epoch": 0.5742046290995338,
      "grad_norm": 4.9364495277404785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25914859771728516,
      "logits/rejected": 0.09887057542800903,
      "logps/chosen": -308.78753662109375,
      "logps/rejected": -312.6427001953125,
      "loss": 0.4508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2746616303920746,
      "rewards/margins": 1.0801056623458862,
      "rewards/rejected": -0.8054440021514893,
      "step": 4388
    },
    {
      "epoch": 0.5743354870368855,
      "grad_norm": 5.637602806091309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2781887352466583,
      "logits/rejected": 0.1484033316373825,
      "logps/chosen": -465.0191345214844,
      "logps/rejected": -369.4205017089844,
      "loss": 0.4199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7580782175064087,
      "rewards/margins": 1.6926133632659912,
      "rewards/rejected": -0.9345353841781616,
      "step": 4389
    },
    {
      "epoch": 0.5744663449742373,
      "grad_norm": 9.327970504760742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029857587069272995,
      "logits/rejected": -0.11156708747148514,
      "logps/chosen": -588.1970825195312,
      "logps/rejected": -464.6968688964844,
      "loss": 0.6661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3282497525215149,
      "rewards/margins": 0.6292023658752441,
      "rewards/rejected": -0.300952672958374,
      "step": 4390
    },
    {
      "epoch": 0.5745972029115891,
      "grad_norm": 6.18925666809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05438558757305145,
      "logits/rejected": 0.10315904766321182,
      "logps/chosen": -396.08184814453125,
      "logps/rejected": -433.66973876953125,
      "loss": 0.3543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8183922171592712,
      "rewards/margins": 1.8221114873886108,
      "rewards/rejected": -1.0037193298339844,
      "step": 4391
    },
    {
      "epoch": 0.5747280608489409,
      "grad_norm": 5.6591973304748535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3722381889820099,
      "logits/rejected": 0.017185427248477936,
      "logps/chosen": -451.4142761230469,
      "logps/rejected": -435.4598388671875,
      "loss": 0.3518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24319075047969818,
      "rewards/margins": 1.7769598960876465,
      "rewards/rejected": -1.533769130706787,
      "step": 4392
    },
    {
      "epoch": 0.5748589187862926,
      "grad_norm": 5.749161243438721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005810577422380447,
      "logits/rejected": -0.09631403535604477,
      "logps/chosen": -308.09588623046875,
      "logps/rejected": -367.2371520996094,
      "loss": 0.5829,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3227083384990692,
      "rewards/margins": 0.6261637210845947,
      "rewards/rejected": -0.9488720297813416,
      "step": 4393
    },
    {
      "epoch": 0.5749897767236444,
      "grad_norm": 15.741021156311035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11125065386295319,
      "logits/rejected": -0.13233478367328644,
      "logps/chosen": -678.1805419921875,
      "logps/rejected": -499.08233642578125,
      "loss": 1.2749,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.046361178159713745,
      "rewards/margins": 0.18251018226146698,
      "rewards/rejected": -0.13614903390407562,
      "step": 4394
    },
    {
      "epoch": 0.5751206346609962,
      "grad_norm": 7.166994571685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14607679843902588,
      "logits/rejected": 0.0012860223650932312,
      "logps/chosen": -337.738525390625,
      "logps/rejected": -398.7633056640625,
      "loss": 0.54,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21969452500343323,
      "rewards/margins": 0.8435186147689819,
      "rewards/rejected": -1.0632131099700928,
      "step": 4395
    },
    {
      "epoch": 0.575251492598348,
      "grad_norm": 4.982906341552734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14072544872760773,
      "logits/rejected": 0.19016876816749573,
      "logps/chosen": -444.114990234375,
      "logps/rejected": -456.5875244140625,
      "loss": 0.4001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.17849101126194,
      "rewards/margins": 1.1759716272354126,
      "rewards/rejected": -0.9974806308746338,
      "step": 4396
    },
    {
      "epoch": 0.5753823505356996,
      "grad_norm": 6.438525676727295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2813202738761902,
      "logits/rejected": 0.12022646516561508,
      "logps/chosen": -542.7820434570312,
      "logps/rejected": -478.60870361328125,
      "loss": 0.3609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23549042642116547,
      "rewards/margins": 1.2338974475860596,
      "rewards/rejected": -0.9984070062637329,
      "step": 4397
    },
    {
      "epoch": 0.5755132084730514,
      "grad_norm": 2.6504883766174316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08307000994682312,
      "logits/rejected": -0.0961804911494255,
      "logps/chosen": -647.8794555664062,
      "logps/rejected": -402.218994140625,
      "loss": 0.1226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5931603908538818,
      "rewards/margins": 2.6075165271759033,
      "rewards/rejected": -1.014356017112732,
      "step": 4398
    },
    {
      "epoch": 0.5756440664104032,
      "grad_norm": 9.447592735290527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09567306190729141,
      "logits/rejected": 0.08184406906366348,
      "logps/chosen": -434.70233154296875,
      "logps/rejected": -500.32763671875,
      "loss": 0.7245,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.27628955245018005,
      "rewards/margins": 0.06584171950817108,
      "rewards/rejected": 0.21044787764549255,
      "step": 4399
    },
    {
      "epoch": 0.575774924347755,
      "grad_norm": 10.060586929321289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09628201276063919,
      "logits/rejected": 0.21484844386577606,
      "logps/chosen": -432.687744140625,
      "logps/rejected": -412.3296203613281,
      "loss": 0.5194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4358137249946594,
      "rewards/margins": 0.9106130599975586,
      "rewards/rejected": -1.3464268445968628,
      "step": 4400
    },
    {
      "epoch": 0.5759057822851067,
      "grad_norm": 4.140223026275635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0663522332906723,
      "logits/rejected": 0.3059459328651428,
      "logps/chosen": -464.348876953125,
      "logps/rejected": -506.6949157714844,
      "loss": 0.2446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2553130090236664,
      "rewards/margins": 1.5431464910507202,
      "rewards/rejected": -1.2878334522247314,
      "step": 4401
    },
    {
      "epoch": 0.5760366402224585,
      "grad_norm": 7.031292915344238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25181832909584045,
      "logits/rejected": 0.30205994844436646,
      "logps/chosen": -380.1806640625,
      "logps/rejected": -401.12384033203125,
      "loss": 0.5295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011699095368385315,
      "rewards/margins": 1.068850040435791,
      "rewards/rejected": -1.0805492401123047,
      "step": 4402
    },
    {
      "epoch": 0.5761674981598103,
      "grad_norm": 5.435790538787842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05219317227602005,
      "logits/rejected": 0.2613343298435211,
      "logps/chosen": -289.9814147949219,
      "logps/rejected": -452.1685791015625,
      "loss": 0.4648,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06357134878635406,
      "rewards/margins": 1.0739468336105347,
      "rewards/rejected": -1.0103753805160522,
      "step": 4403
    },
    {
      "epoch": 0.5762983560971621,
      "grad_norm": 10.310633659362793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06223384663462639,
      "logits/rejected": -0.032356634736061096,
      "logps/chosen": -367.83428955078125,
      "logps/rejected": -368.35247802734375,
      "loss": 0.8322,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24433231353759766,
      "rewards/margins": 0.595043957233429,
      "rewards/rejected": -0.3507115840911865,
      "step": 4404
    },
    {
      "epoch": 0.5764292140345137,
      "grad_norm": 6.05494499206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21726977825164795,
      "logits/rejected": 0.1906416267156601,
      "logps/chosen": -392.36920166015625,
      "logps/rejected": -384.7066650390625,
      "loss": 0.5216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34928399324417114,
      "rewards/margins": 0.6834777593612671,
      "rewards/rejected": -1.0327616930007935,
      "step": 4405
    },
    {
      "epoch": 0.5765600719718655,
      "grad_norm": 9.162229537963867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16036637127399445,
      "logits/rejected": 0.25717678666114807,
      "logps/chosen": -435.0717468261719,
      "logps/rejected": -353.5243835449219,
      "loss": 0.7316,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5899227261543274,
      "rewards/margins": 0.5518728494644165,
      "rewards/rejected": -1.1417955160140991,
      "step": 4406
    },
    {
      "epoch": 0.5766909299092173,
      "grad_norm": 10.345392227172852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04356587678194046,
      "logits/rejected": 0.14483192563056946,
      "logps/chosen": -379.7355041503906,
      "logps/rejected": -509.38543701171875,
      "loss": 0.9729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1616758406162262,
      "rewards/margins": 0.0683843195438385,
      "rewards/rejected": -0.23006021976470947,
      "step": 4407
    },
    {
      "epoch": 0.5768217878465691,
      "grad_norm": 5.313080310821533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15057219564914703,
      "logits/rejected": 0.050064656883478165,
      "logps/chosen": -510.8956604003906,
      "logps/rejected": -308.7060546875,
      "loss": 0.3419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2774713933467865,
      "rewards/margins": 1.2450063228607178,
      "rewards/rejected": -0.9675350785255432,
      "step": 4408
    },
    {
      "epoch": 0.5769526457839208,
      "grad_norm": 6.628969669342041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4236578941345215,
      "logits/rejected": 0.40268537402153015,
      "logps/chosen": -347.5051574707031,
      "logps/rejected": -388.0155029296875,
      "loss": 0.6718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18216899037361145,
      "rewards/margins": 0.35709837079048157,
      "rewards/rejected": -0.539267361164093,
      "step": 4409
    },
    {
      "epoch": 0.5770835037212726,
      "grad_norm": 8.352394104003906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10935163497924805,
      "logits/rejected": -0.09712938219308853,
      "logps/chosen": -376.9943542480469,
      "logps/rejected": -433.670166015625,
      "loss": 0.3789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2935144305229187,
      "rewards/margins": 1.1207094192504883,
      "rewards/rejected": -0.8271949291229248,
      "step": 4410
    },
    {
      "epoch": 0.5772143616586244,
      "grad_norm": 4.543122291564941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21701529622077942,
      "logits/rejected": -0.044691115617752075,
      "logps/chosen": -362.1165771484375,
      "logps/rejected": -331.7664794921875,
      "loss": 0.2078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.022112667560577393,
      "rewards/margins": 2.101407051086426,
      "rewards/rejected": -2.079294443130493,
      "step": 4411
    },
    {
      "epoch": 0.5773452195959761,
      "grad_norm": 6.718711853027344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3053642511367798,
      "logits/rejected": 0.2943095564842224,
      "logps/chosen": -581.43310546875,
      "logps/rejected": -578.579833984375,
      "loss": 0.461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24718570709228516,
      "rewards/margins": 0.9592088460922241,
      "rewards/rejected": -0.712023138999939,
      "step": 4412
    },
    {
      "epoch": 0.5774760775333279,
      "grad_norm": 9.208613395690918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024524765089154243,
      "logits/rejected": -0.0038909390568733215,
      "logps/chosen": -368.890625,
      "logps/rejected": -477.44989013671875,
      "loss": 0.7051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7250194549560547,
      "rewards/margins": 0.33043792843818665,
      "rewards/rejected": -1.055457353591919,
      "step": 4413
    },
    {
      "epoch": 0.5776069354706796,
      "grad_norm": 8.836542129516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21755336225032806,
      "logits/rejected": 0.05075180530548096,
      "logps/chosen": -557.2137451171875,
      "logps/rejected": -546.0657348632812,
      "loss": 0.6118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29618167877197266,
      "rewards/margins": 0.9171379804611206,
      "rewards/rejected": -1.2133197784423828,
      "step": 4414
    },
    {
      "epoch": 0.5777377934080314,
      "grad_norm": 9.38068675994873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08435488492250443,
      "logits/rejected": -0.10741220414638519,
      "logps/chosen": -470.3620910644531,
      "logps/rejected": -423.4917907714844,
      "loss": 0.8197,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9133745431900024,
      "rewards/margins": -0.16607913374900818,
      "rewards/rejected": -0.7472953796386719,
      "step": 4415
    },
    {
      "epoch": 0.5778686513453831,
      "grad_norm": 5.806862831115723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10768433660268784,
      "logits/rejected": -0.24114738404750824,
      "logps/chosen": -309.83917236328125,
      "logps/rejected": -262.13134765625,
      "loss": 0.572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23230688273906708,
      "rewards/margins": 0.9618910551071167,
      "rewards/rejected": -1.1941978931427002,
      "step": 4416
    },
    {
      "epoch": 0.5779995092827349,
      "grad_norm": 8.001609802246094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17751361429691315,
      "logits/rejected": 0.15019658207893372,
      "logps/chosen": -578.8466186523438,
      "logps/rejected": -465.6894836425781,
      "loss": 0.6509,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.43281590938568115,
      "rewards/margins": 0.3618510663509369,
      "rewards/rejected": 0.07096481323242188,
      "step": 4417
    },
    {
      "epoch": 0.5781303672200867,
      "grad_norm": 6.563497066497803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0665653795003891,
      "logits/rejected": 0.07587766647338867,
      "logps/chosen": -803.2548217773438,
      "logps/rejected": -617.861083984375,
      "loss": 0.4258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23648911714553833,
      "rewards/margins": 0.8741430640220642,
      "rewards/rejected": -0.6376539468765259,
      "step": 4418
    },
    {
      "epoch": 0.5782612251574385,
      "grad_norm": 6.331382751464844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0010648872703313828,
      "logits/rejected": 0.015897037461400032,
      "logps/chosen": -314.71478271484375,
      "logps/rejected": -337.1977844238281,
      "loss": 0.5366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.028653431683778763,
      "rewards/margins": 0.7700155973434448,
      "rewards/rejected": -0.7413622736930847,
      "step": 4419
    },
    {
      "epoch": 0.5783920830947902,
      "grad_norm": 5.5608296394348145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023950807750225067,
      "logits/rejected": -0.05146664008498192,
      "logps/chosen": -462.59417724609375,
      "logps/rejected": -320.35943603515625,
      "loss": 0.3871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42679575085639954,
      "rewards/margins": 1.2331624031066895,
      "rewards/rejected": -0.8063668012619019,
      "step": 4420
    },
    {
      "epoch": 0.578522941032142,
      "grad_norm": 6.798379421234131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08720365166664124,
      "logits/rejected": -0.125665545463562,
      "logps/chosen": -379.9069519042969,
      "logps/rejected": -347.62689208984375,
      "loss": 0.512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2884323298931122,
      "rewards/margins": 1.3756576776504517,
      "rewards/rejected": -1.087225317955017,
      "step": 4421
    },
    {
      "epoch": 0.5786537989694938,
      "grad_norm": 6.869431972503662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09109555184841156,
      "logits/rejected": 0.03469955921173096,
      "logps/chosen": -334.63360595703125,
      "logps/rejected": -299.54296875,
      "loss": 0.6039,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.058781206607818604,
      "rewards/margins": 0.5967779755592346,
      "rewards/rejected": -0.6555591821670532,
      "step": 4422
    },
    {
      "epoch": 0.5787846569068456,
      "grad_norm": 7.162740707397461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07894794642925262,
      "logits/rejected": 0.03769553080201149,
      "logps/chosen": -352.9894104003906,
      "logps/rejected": -406.18426513671875,
      "loss": 0.6068,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18264789879322052,
      "rewards/margins": 0.8058741092681885,
      "rewards/rejected": -0.6232261657714844,
      "step": 4423
    },
    {
      "epoch": 0.5789155148441972,
      "grad_norm": 3.6081507205963135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05820826068520546,
      "logits/rejected": -0.3028391897678375,
      "logps/chosen": -475.64154052734375,
      "logps/rejected": -479.58203125,
      "loss": 0.199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2629367709159851,
      "rewards/margins": 2.0213279724121094,
      "rewards/rejected": -2.28426456451416,
      "step": 4424
    },
    {
      "epoch": 0.579046372781549,
      "grad_norm": 5.465828895568848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3629409074783325,
      "logits/rejected": -0.07732996344566345,
      "logps/chosen": -317.82952880859375,
      "logps/rejected": -492.95318603515625,
      "loss": 0.4533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3817669153213501,
      "rewards/margins": 1.243707299232483,
      "rewards/rejected": -1.6254740953445435,
      "step": 4425
    },
    {
      "epoch": 0.5791772307189008,
      "grad_norm": 5.9153666496276855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15781782567501068,
      "logits/rejected": 0.312601774930954,
      "logps/chosen": -615.003173828125,
      "logps/rejected": -435.01593017578125,
      "loss": 0.5169,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07502689957618713,
      "rewards/margins": 0.7499439716339111,
      "rewards/rejected": -0.8249708414077759,
      "step": 4426
    },
    {
      "epoch": 0.5793080886562526,
      "grad_norm": 6.976050853729248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19885802268981934,
      "logits/rejected": -0.2337084412574768,
      "logps/chosen": -464.5548095703125,
      "logps/rejected": -543.3902587890625,
      "loss": 0.4518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19533175230026245,
      "rewards/margins": 1.243621826171875,
      "rewards/rejected": -1.4389535188674927,
      "step": 4427
    },
    {
      "epoch": 0.5794389465936043,
      "grad_norm": 7.904623508453369,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00935564935207367,
      "logits/rejected": 0.17352482676506042,
      "logps/chosen": -406.53765869140625,
      "logps/rejected": -459.7051696777344,
      "loss": 0.74,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5774990320205688,
      "rewards/margins": 0.2047080546617508,
      "rewards/rejected": -0.7822070717811584,
      "step": 4428
    },
    {
      "epoch": 0.5795698045309561,
      "grad_norm": 6.608576774597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06843234598636627,
      "logits/rejected": -0.049615681171417236,
      "logps/chosen": -460.68536376953125,
      "logps/rejected": -442.9314270019531,
      "loss": 0.4469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21918807923793793,
      "rewards/margins": 0.8758829832077026,
      "rewards/rejected": -1.0950709581375122,
      "step": 4429
    },
    {
      "epoch": 0.5797006624683079,
      "grad_norm": 7.523776054382324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039099402725696564,
      "logits/rejected": -0.02092531882226467,
      "logps/chosen": -356.6699523925781,
      "logps/rejected": -360.9055480957031,
      "loss": 0.6253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16345861554145813,
      "rewards/margins": 0.35000282526016235,
      "rewards/rejected": -0.5134615302085876,
      "step": 4430
    },
    {
      "epoch": 0.5798315204056596,
      "grad_norm": 6.488447189331055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12127915769815445,
      "logits/rejected": -0.00904767494648695,
      "logps/chosen": -450.9768981933594,
      "logps/rejected": -446.21575927734375,
      "loss": 0.3934,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05927640199661255,
      "rewards/margins": 0.9328196048736572,
      "rewards/rejected": -0.992095947265625,
      "step": 4431
    },
    {
      "epoch": 0.5799623783430113,
      "grad_norm": 8.746419906616211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3768056035041809,
      "logits/rejected": -0.3485667109489441,
      "logps/chosen": -457.3494567871094,
      "logps/rejected": -299.8113708496094,
      "loss": 0.5317,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.645336925983429,
      "rewards/margins": 1.5304044485092163,
      "rewards/rejected": -2.175741195678711,
      "step": 4432
    },
    {
      "epoch": 0.5800932362803631,
      "grad_norm": 7.693070888519287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09056543558835983,
      "logits/rejected": -0.18716199696063995,
      "logps/chosen": -521.02734375,
      "logps/rejected": -390.66998291015625,
      "loss": 0.4796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18469008803367615,
      "rewards/margins": 0.8186023831367493,
      "rewards/rejected": -0.6339122653007507,
      "step": 4433
    },
    {
      "epoch": 0.5802240942177149,
      "grad_norm": 10.577165603637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0031847432255744934,
      "logits/rejected": -0.010266272351145744,
      "logps/chosen": -410.5765686035156,
      "logps/rejected": -426.39019775390625,
      "loss": 0.7068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3984655737876892,
      "rewards/margins": 0.8610395789146423,
      "rewards/rejected": -1.2595051527023315,
      "step": 4434
    },
    {
      "epoch": 0.5803549521550666,
      "grad_norm": 6.533537864685059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15550771355628967,
      "logits/rejected": -0.012954454869031906,
      "logps/chosen": -395.86810302734375,
      "logps/rejected": -365.4862060546875,
      "loss": 0.4289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8425453305244446,
      "rewards/margins": 0.9124569892883301,
      "rewards/rejected": -1.755002498626709,
      "step": 4435
    },
    {
      "epoch": 0.5804858100924184,
      "grad_norm": 6.472026824951172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18169721961021423,
      "logits/rejected": -0.005983076989650726,
      "logps/chosen": -461.4132385253906,
      "logps/rejected": -496.3671875,
      "loss": 0.4908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20728819072246552,
      "rewards/margins": 1.1824367046356201,
      "rewards/rejected": -0.9751484394073486,
      "step": 4436
    },
    {
      "epoch": 0.5806166680297702,
      "grad_norm": 5.667995452880859,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1646270453929901,
      "logits/rejected": 0.13462714850902557,
      "logps/chosen": -531.6087646484375,
      "logps/rejected": -455.48736572265625,
      "loss": 0.3705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.39110150933265686,
      "rewards/margins": 1.4898039102554321,
      "rewards/rejected": -1.0987024307250977,
      "step": 4437
    },
    {
      "epoch": 0.580747525967122,
      "grad_norm": 12.597085952758789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23440435528755188,
      "logits/rejected": 0.3551473617553711,
      "logps/chosen": -366.08441162109375,
      "logps/rejected": -378.28887939453125,
      "loss": 0.8833,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1147661209106445,
      "rewards/margins": 0.24216321110725403,
      "rewards/rejected": -1.3569293022155762,
      "step": 4438
    },
    {
      "epoch": 0.5808783839044737,
      "grad_norm": 13.750414848327637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04437238723039627,
      "logits/rejected": 0.0036449190229177475,
      "logps/chosen": -559.7577514648438,
      "logps/rejected": -459.20928955078125,
      "loss": 1.0294,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2210952639579773,
      "rewards/margins": 0.004647016525268555,
      "rewards/rejected": -0.22574235498905182,
      "step": 4439
    },
    {
      "epoch": 0.5810092418418255,
      "grad_norm": 7.263567924499512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011786222457885742,
      "logits/rejected": -0.02734633907675743,
      "logps/chosen": -404.91015625,
      "logps/rejected": -449.81146240234375,
      "loss": 0.6311,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.28762421011924744,
      "rewards/margins": 0.3809904158115387,
      "rewards/rejected": -0.09336626529693604,
      "step": 4440
    },
    {
      "epoch": 0.5811400997791772,
      "grad_norm": 9.016863822937012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31032103300094604,
      "logits/rejected": -0.27117466926574707,
      "logps/chosen": -715.6618041992188,
      "logps/rejected": -706.4808959960938,
      "loss": 0.6217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3468689024448395,
      "rewards/margins": 0.7226371765136719,
      "rewards/rejected": -0.3757683038711548,
      "step": 4441
    },
    {
      "epoch": 0.581270957716529,
      "grad_norm": 5.020709037780762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28908050060272217,
      "logits/rejected": 0.14373517036437988,
      "logps/chosen": -487.42413330078125,
      "logps/rejected": -472.2495422363281,
      "loss": 0.32,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9253612756729126,
      "rewards/margins": 1.7658072710037231,
      "rewards/rejected": -0.8404461145401001,
      "step": 4442
    },
    {
      "epoch": 0.5814018156538807,
      "grad_norm": 7.093387126922607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19478048384189606,
      "logits/rejected": -0.05556826293468475,
      "logps/chosen": -549.2081298828125,
      "logps/rejected": -451.3977966308594,
      "loss": 0.5736,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4379434585571289,
      "rewards/margins": 0.5912183523178101,
      "rewards/rejected": -0.15327489376068115,
      "step": 4443
    },
    {
      "epoch": 0.5815326735912325,
      "grad_norm": 9.701013565063477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14245188236236572,
      "logits/rejected": 0.1404820829629898,
      "logps/chosen": -378.51812744140625,
      "logps/rejected": -329.1903991699219,
      "loss": 0.9174,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29853934049606323,
      "rewards/margins": 0.4245615005493164,
      "rewards/rejected": -0.7231009006500244,
      "step": 4444
    },
    {
      "epoch": 0.5816635315285843,
      "grad_norm": 8.625126838684082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1650274097919464,
      "logits/rejected": 0.2764318883419037,
      "logps/chosen": -892.1474609375,
      "logps/rejected": -678.3497924804688,
      "loss": 0.804,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1278068572282791,
      "rewards/margins": 0.9480065107345581,
      "rewards/rejected": -1.0758132934570312,
      "step": 4445
    },
    {
      "epoch": 0.5817943894659361,
      "grad_norm": 5.321598529815674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1966952681541443,
      "logits/rejected": -0.10185759514570236,
      "logps/chosen": -479.69561767578125,
      "logps/rejected": -520.6985473632812,
      "loss": 0.365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10358849167823792,
      "rewards/margins": 0.8788030743598938,
      "rewards/rejected": -0.775214672088623,
      "step": 4446
    },
    {
      "epoch": 0.5819252474032878,
      "grad_norm": 7.1539788246154785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40888017416000366,
      "logits/rejected": 0.33989417552948,
      "logps/chosen": -438.39508056640625,
      "logps/rejected": -398.937255859375,
      "loss": 0.5433,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.4371439218521118,
      "rewards/margins": 0.6888735294342041,
      "rewards/rejected": -0.2517296075820923,
      "step": 4447
    },
    {
      "epoch": 0.5820561053406396,
      "grad_norm": 14.778773307800293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08426453173160553,
      "logits/rejected": 0.13647915422916412,
      "logps/chosen": -538.4850463867188,
      "logps/rejected": -340.3357238769531,
      "loss": 0.9501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5782337188720703,
      "rewards/margins": 0.199440598487854,
      "rewards/rejected": -0.7776743173599243,
      "step": 4448
    },
    {
      "epoch": 0.5821869632779914,
      "grad_norm": 8.659967422485352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2595014274120331,
      "logits/rejected": -0.1734534502029419,
      "logps/chosen": -453.61468505859375,
      "logps/rejected": -452.3670959472656,
      "loss": 0.5515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5457569360733032,
      "rewards/margins": 1.18889582157135,
      "rewards/rejected": -1.7346527576446533,
      "step": 4449
    },
    {
      "epoch": 0.5823178212153431,
      "grad_norm": 8.807026863098145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10891026258468628,
      "logits/rejected": 0.11972953379154205,
      "logps/chosen": -464.54071044921875,
      "logps/rejected": -470.82781982421875,
      "loss": 0.6391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04210171103477478,
      "rewards/margins": 1.013582468032837,
      "rewards/rejected": -0.9714807271957397,
      "step": 4450
    },
    {
      "epoch": 0.5824486791526948,
      "grad_norm": 8.61467456817627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14986516535282135,
      "logits/rejected": 0.2523643672466278,
      "logps/chosen": -441.05419921875,
      "logps/rejected": -510.1300354003906,
      "loss": 0.5995,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30157434940338135,
      "rewards/margins": 0.7412002086639404,
      "rewards/rejected": -1.0427746772766113,
      "step": 4451
    },
    {
      "epoch": 0.5825795370900466,
      "grad_norm": 6.206505298614502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2275751531124115,
      "logits/rejected": 0.2042853832244873,
      "logps/chosen": -434.27069091796875,
      "logps/rejected": -395.80816650390625,
      "loss": 0.617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.007495120167732239,
      "rewards/margins": 0.4674520492553711,
      "rewards/rejected": -0.47494715452194214,
      "step": 4452
    },
    {
      "epoch": 0.5827103950273984,
      "grad_norm": 6.038500785827637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20007477700710297,
      "logits/rejected": 0.002866249531507492,
      "logps/chosen": -471.9100341796875,
      "logps/rejected": -335.8797302246094,
      "loss": 0.4395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17929667234420776,
      "rewards/margins": 0.9249091148376465,
      "rewards/rejected": -0.745612382888794,
      "step": 4453
    },
    {
      "epoch": 0.5828412529647501,
      "grad_norm": 5.85737419128418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17312006652355194,
      "logits/rejected": 0.18204635381698608,
      "logps/chosen": -462.188720703125,
      "logps/rejected": -388.46636962890625,
      "loss": 0.4769,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.649334192276001,
      "rewards/margins": 0.8815351128578186,
      "rewards/rejected": -0.2322009652853012,
      "step": 4454
    },
    {
      "epoch": 0.5829721109021019,
      "grad_norm": 5.049839496612549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2469889521598816,
      "logits/rejected": 0.11268092691898346,
      "logps/chosen": -596.9266967773438,
      "logps/rejected": -504.4207763671875,
      "loss": 0.2784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24615365266799927,
      "rewards/margins": 1.303839921951294,
      "rewards/rejected": -1.5499935150146484,
      "step": 4455
    },
    {
      "epoch": 0.5831029688394537,
      "grad_norm": 5.178932189941406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024074003100395203,
      "logits/rejected": 0.1294144243001938,
      "logps/chosen": -385.5692138671875,
      "logps/rejected": -409.7689514160156,
      "loss": 0.3699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6010701060295105,
      "rewards/margins": 1.139482021331787,
      "rewards/rejected": -1.7405519485473633,
      "step": 4456
    },
    {
      "epoch": 0.5832338267768055,
      "grad_norm": 6.875326156616211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004520148038864136,
      "logits/rejected": -0.08766083419322968,
      "logps/chosen": -493.0220031738281,
      "logps/rejected": -528.5584716796875,
      "loss": 0.3549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29455435276031494,
      "rewards/margins": 1.3337304592132568,
      "rewards/rejected": -1.0391762256622314,
      "step": 4457
    },
    {
      "epoch": 0.5833646847141571,
      "grad_norm": 7.864147663116455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5200667381286621,
      "logits/rejected": 0.18955828249454498,
      "logps/chosen": -421.5340576171875,
      "logps/rejected": -387.8937683105469,
      "loss": 0.5389,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1499742567539215,
      "rewards/margins": 1.2609244585037231,
      "rewards/rejected": -1.4108988046646118,
      "step": 4458
    },
    {
      "epoch": 0.5834955426515089,
      "grad_norm": 7.365645885467529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08843129873275757,
      "logits/rejected": 0.024950038641691208,
      "logps/chosen": -560.0213012695312,
      "logps/rejected": -449.755126953125,
      "loss": 0.5106,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6374093890190125,
      "rewards/margins": 1.0271708965301514,
      "rewards/rejected": -0.3897615373134613,
      "step": 4459
    },
    {
      "epoch": 0.5836264005888607,
      "grad_norm": 5.420416831970215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11166393756866455,
      "logits/rejected": 0.17438076436519623,
      "logps/chosen": -389.41436767578125,
      "logps/rejected": -518.0889282226562,
      "loss": 0.3896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6355229020118713,
      "rewards/margins": 1.069948434829712,
      "rewards/rejected": -1.7054712772369385,
      "step": 4460
    },
    {
      "epoch": 0.5837572585262125,
      "grad_norm": 9.381007194519043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12341956794261932,
      "logits/rejected": -0.0418526753783226,
      "logps/chosen": -475.6053771972656,
      "logps/rejected": -363.05145263671875,
      "loss": 0.6707,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6757925748825073,
      "rewards/margins": 0.3848383128643036,
      "rewards/rejected": -1.0606309175491333,
      "step": 4461
    },
    {
      "epoch": 0.5838881164635642,
      "grad_norm": 5.40436315536499,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04172869026660919,
      "logits/rejected": -0.29609793424606323,
      "logps/chosen": -477.6370849609375,
      "logps/rejected": -305.8764343261719,
      "loss": 0.3756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0489448606967926,
      "rewards/margins": 1.053305983543396,
      "rewards/rejected": -1.1022508144378662,
      "step": 4462
    },
    {
      "epoch": 0.584018974400916,
      "grad_norm": 4.335102081298828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40893423557281494,
      "logits/rejected": 0.185848206281662,
      "logps/chosen": -391.0132141113281,
      "logps/rejected": -361.26605224609375,
      "loss": 0.2377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20423126220703125,
      "rewards/margins": 1.6728894710540771,
      "rewards/rejected": -1.468658208847046,
      "step": 4463
    },
    {
      "epoch": 0.5841498323382678,
      "grad_norm": 9.265341758728027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1311943531036377,
      "logits/rejected": -0.0070802196860313416,
      "logps/chosen": -429.909912109375,
      "logps/rejected": -453.5067138671875,
      "loss": 0.7492,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5816871523857117,
      "rewards/margins": 1.4929516315460205,
      "rewards/rejected": -0.9112644195556641,
      "step": 4464
    },
    {
      "epoch": 0.5842806902756196,
      "grad_norm": 7.1394944190979,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25734102725982666,
      "logits/rejected": -0.014890288934111595,
      "logps/chosen": -387.57659912109375,
      "logps/rejected": -613.2168579101562,
      "loss": 0.3698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5136287212371826,
      "rewards/margins": 1.736650824546814,
      "rewards/rejected": -3.250279664993286,
      "step": 4465
    },
    {
      "epoch": 0.5844115482129713,
      "grad_norm": 7.150721073150635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.051611319184303284,
      "logits/rejected": -0.0774594247341156,
      "logps/chosen": -418.313232421875,
      "logps/rejected": -479.4666442871094,
      "loss": 0.452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12897413969039917,
      "rewards/margins": 1.933597207069397,
      "rewards/rejected": -1.8046231269836426,
      "step": 4466
    },
    {
      "epoch": 0.584542406150323,
      "grad_norm": 8.335064888000488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33952081203460693,
      "logits/rejected": 0.4880217909812927,
      "logps/chosen": -471.17926025390625,
      "logps/rejected": -403.3446350097656,
      "loss": 0.6731,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.17545069754123688,
      "rewards/margins": 0.49370160698890686,
      "rewards/rejected": -0.3182508647441864,
      "step": 4467
    },
    {
      "epoch": 0.5846732640876748,
      "grad_norm": 10.129524230957031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10741152614355087,
      "logits/rejected": -0.23368433117866516,
      "logps/chosen": -383.646484375,
      "logps/rejected": -458.5565185546875,
      "loss": 0.7485,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9996337890625,
      "rewards/margins": 0.10408592224121094,
      "rewards/rejected": -1.103719711303711,
      "step": 4468
    },
    {
      "epoch": 0.5848041220250266,
      "grad_norm": 8.924967765808105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18292038142681122,
      "logits/rejected": 0.16842472553253174,
      "logps/chosen": -656.5530395507812,
      "logps/rejected": -509.1221008300781,
      "loss": 0.7467,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11615011096000671,
      "rewards/margins": 0.2605322003364563,
      "rewards/rejected": -0.3766822814941406,
      "step": 4469
    },
    {
      "epoch": 0.5849349799623783,
      "grad_norm": 6.607978343963623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17945869266986847,
      "logits/rejected": -0.24352306127548218,
      "logps/chosen": -499.0778503417969,
      "logps/rejected": -465.6231689453125,
      "loss": 0.5387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20084743201732635,
      "rewards/margins": 1.0286386013031006,
      "rewards/rejected": -0.827791154384613,
      "step": 4470
    },
    {
      "epoch": 0.5850658378997301,
      "grad_norm": 3.63261079788208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037438806146383286,
      "logits/rejected": 0.07371509075164795,
      "logps/chosen": -474.72735595703125,
      "logps/rejected": -393.1755676269531,
      "loss": 0.2606,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34215009212493896,
      "rewards/margins": 2.284261465072632,
      "rewards/rejected": -1.9421111345291138,
      "step": 4471
    },
    {
      "epoch": 0.5851966958370819,
      "grad_norm": 6.207993507385254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24476784467697144,
      "logits/rejected": 0.22486096620559692,
      "logps/chosen": -493.0168762207031,
      "logps/rejected": -327.78448486328125,
      "loss": 0.3761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5566568374633789,
      "rewards/margins": 1.1840591430664062,
      "rewards/rejected": -0.6274023652076721,
      "step": 4472
    },
    {
      "epoch": 0.5853275537744337,
      "grad_norm": 10.526629447937012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002257697284221649,
      "logits/rejected": 0.06928402185440063,
      "logps/chosen": -535.9401245117188,
      "logps/rejected": -579.7333984375,
      "loss": 0.9435,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7116148471832275,
      "rewards/margins": 0.09134204685688019,
      "rewards/rejected": -0.8029567003250122,
      "step": 4473
    },
    {
      "epoch": 0.5854584117117854,
      "grad_norm": 5.790347099304199,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19559486210346222,
      "logits/rejected": -0.07781368494033813,
      "logps/chosen": -304.642822265625,
      "logps/rejected": -342.35260009765625,
      "loss": 0.4966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5338396430015564,
      "rewards/margins": 1.020417332649231,
      "rewards/rejected": -1.554256796836853,
      "step": 4474
    },
    {
      "epoch": 0.5855892696491372,
      "grad_norm": 11.10711669921875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21626794338226318,
      "logits/rejected": 0.2160826325416565,
      "logps/chosen": -582.8387451171875,
      "logps/rejected": -510.6940612792969,
      "loss": 0.6441,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8314990997314453,
      "rewards/margins": 0.37140733003616333,
      "rewards/rejected": -1.2029063701629639,
      "step": 4475
    },
    {
      "epoch": 0.585720127586489,
      "grad_norm": 8.133526802062988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0001338571310043335,
      "logits/rejected": 0.10363911092281342,
      "logps/chosen": -309.1501159667969,
      "logps/rejected": -296.2469482421875,
      "loss": 0.9216,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021929334849119186,
      "rewards/margins": -0.0675586462020874,
      "rewards/rejected": 0.04562931880354881,
      "step": 4476
    },
    {
      "epoch": 0.5858509855238406,
      "grad_norm": 7.119701385498047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06845645606517792,
      "logits/rejected": -0.26923343539237976,
      "logps/chosen": -435.9452819824219,
      "logps/rejected": -315.1371154785156,
      "loss": 0.6404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35407885909080505,
      "rewards/margins": 0.34885650873184204,
      "rewards/rejected": -0.7029354572296143,
      "step": 4477
    },
    {
      "epoch": 0.5859818434611924,
      "grad_norm": 8.695340156555176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20190733671188354,
      "logits/rejected": 0.025440514087677002,
      "logps/chosen": -539.9921875,
      "logps/rejected": -593.715087890625,
      "loss": 0.4462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20010223984718323,
      "rewards/margins": 1.0657298564910889,
      "rewards/rejected": -1.2658321857452393,
      "step": 4478
    },
    {
      "epoch": 0.5861127013985442,
      "grad_norm": 7.271799564361572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07331150770187378,
      "logits/rejected": 0.03316154330968857,
      "logps/chosen": -678.9203491210938,
      "logps/rejected": -578.9415283203125,
      "loss": 0.4506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7416889667510986,
      "rewards/margins": 0.9153751730918884,
      "rewards/rejected": -1.6570641994476318,
      "step": 4479
    },
    {
      "epoch": 0.586243559335896,
      "grad_norm": 5.380494117736816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08636398613452911,
      "logits/rejected": -0.20988279581069946,
      "logps/chosen": -330.1031494140625,
      "logps/rejected": -401.99517822265625,
      "loss": 0.4394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.287692666053772,
      "rewards/margins": 1.3715888261795044,
      "rewards/rejected": -1.6592814922332764,
      "step": 4480
    },
    {
      "epoch": 0.5863744172732477,
      "grad_norm": 6.604220867156982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2983258366584778,
      "logits/rejected": 0.44123584032058716,
      "logps/chosen": -509.9517822265625,
      "logps/rejected": -589.4859619140625,
      "loss": 0.3574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9847934246063232,
      "rewards/margins": 1.4280617237091064,
      "rewards/rejected": -2.4128551483154297,
      "step": 4481
    },
    {
      "epoch": 0.5865052752105995,
      "grad_norm": 6.305936813354492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18567709624767303,
      "logits/rejected": -0.17362838983535767,
      "logps/chosen": -397.4803771972656,
      "logps/rejected": -406.9776916503906,
      "loss": 0.437,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9097940921783447,
      "rewards/margins": 1.023508906364441,
      "rewards/rejected": -1.9333031177520752,
      "step": 4482
    },
    {
      "epoch": 0.5866361331479513,
      "grad_norm": 10.240352630615234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08807158470153809,
      "logits/rejected": 0.07451668381690979,
      "logps/chosen": -621.613037109375,
      "logps/rejected": -587.0546264648438,
      "loss": 0.7168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8784874081611633,
      "rewards/margins": 0.4066879153251648,
      "rewards/rejected": -1.2851753234863281,
      "step": 4483
    },
    {
      "epoch": 0.5867669910853031,
      "grad_norm": 4.817002773284912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4780483841896057,
      "logits/rejected": 0.3178609311580658,
      "logps/chosen": -607.029296875,
      "logps/rejected": -570.284423828125,
      "loss": 0.3055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04119589924812317,
      "rewards/margins": 1.406419038772583,
      "rewards/rejected": -1.3652231693267822,
      "step": 4484
    },
    {
      "epoch": 0.5868978490226547,
      "grad_norm": 6.933516025543213,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1929152011871338,
      "logits/rejected": 0.13723628222942352,
      "logps/chosen": -560.5018920898438,
      "logps/rejected": -457.98406982421875,
      "loss": 0.4286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27292484045028687,
      "rewards/margins": 0.8529157638549805,
      "rewards/rejected": -1.1258405447006226,
      "step": 4485
    },
    {
      "epoch": 0.5870287069600065,
      "grad_norm": 7.25601053237915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021277768537402153,
      "logits/rejected": -0.08078770339488983,
      "logps/chosen": -481.8434753417969,
      "logps/rejected": -330.4462890625,
      "loss": 0.5671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4287809431552887,
      "rewards/margins": 0.8609235286712646,
      "rewards/rejected": -1.2897045612335205,
      "step": 4486
    },
    {
      "epoch": 0.5871595648973583,
      "grad_norm": 4.782864093780518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09322278201580048,
      "logits/rejected": 0.27851319313049316,
      "logps/chosen": -422.36083984375,
      "logps/rejected": -352.48822021484375,
      "loss": 0.3727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3381061851978302,
      "rewards/margins": 1.0450388193130493,
      "rewards/rejected": -1.3831449747085571,
      "step": 4487
    },
    {
      "epoch": 0.5872904228347101,
      "grad_norm": 6.057911396026611,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08610605448484421,
      "logits/rejected": -0.12085453420877457,
      "logps/chosen": -588.94482421875,
      "logps/rejected": -544.4385986328125,
      "loss": 0.4794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00664215674623847,
      "rewards/margins": 1.00798761844635,
      "rewards/rejected": -1.0013455152511597,
      "step": 4488
    },
    {
      "epoch": 0.5874212807720618,
      "grad_norm": 4.4717817306518555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08572040498256683,
      "logits/rejected": -0.10147000849246979,
      "logps/chosen": -366.3417053222656,
      "logps/rejected": -505.43048095703125,
      "loss": 0.359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3447149395942688,
      "rewards/margins": 1.3108049631118774,
      "rewards/rejected": -1.655519962310791,
      "step": 4489
    },
    {
      "epoch": 0.5875521387094136,
      "grad_norm": 5.803943634033203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16093222796916962,
      "logits/rejected": 0.17073644697666168,
      "logps/chosen": -532.5478515625,
      "logps/rejected": -380.563232421875,
      "loss": 0.3104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.32311978936195374,
      "rewards/margins": 1.512003779411316,
      "rewards/rejected": -1.1888840198516846,
      "step": 4490
    },
    {
      "epoch": 0.5876829966467654,
      "grad_norm": 5.27443265914917,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2215680330991745,
      "logits/rejected": -0.2510039806365967,
      "logps/chosen": -711.43505859375,
      "logps/rejected": -446.51507568359375,
      "loss": 0.3003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38433951139450073,
      "rewards/margins": 1.697866678237915,
      "rewards/rejected": -2.0822062492370605,
      "step": 4491
    },
    {
      "epoch": 0.5878138545841172,
      "grad_norm": 7.64578104019165,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27193233370780945,
      "logits/rejected": 0.3044489324092865,
      "logps/chosen": -533.084228515625,
      "logps/rejected": -483.471435546875,
      "loss": 0.8702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5825150012969971,
      "rewards/margins": 0.32000732421875,
      "rewards/rejected": -0.9025223851203918,
      "step": 4492
    },
    {
      "epoch": 0.5879447125214688,
      "grad_norm": 6.46121072769165,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1763303279876709,
      "logits/rejected": 0.03773249685764313,
      "logps/chosen": -493.3939208984375,
      "logps/rejected": -505.18438720703125,
      "loss": 0.3753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.885447084903717,
      "rewards/margins": 0.9045385122299194,
      "rewards/rejected": -1.7899856567382812,
      "step": 4493
    },
    {
      "epoch": 0.5880755704588206,
      "grad_norm": 7.399581432342529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03537824749946594,
      "logits/rejected": 0.002079203724861145,
      "logps/chosen": -558.447509765625,
      "logps/rejected": -620.9356689453125,
      "loss": 0.3558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09520018845796585,
      "rewards/margins": 1.3512938022613525,
      "rewards/rejected": -1.2560936212539673,
      "step": 4494
    },
    {
      "epoch": 0.5882064283961724,
      "grad_norm": 8.949361801147461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08111737668514252,
      "logits/rejected": -0.008573412895202637,
      "logps/chosen": -357.8263244628906,
      "logps/rejected": -377.73175048828125,
      "loss": 0.4744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4028509259223938,
      "rewards/margins": 1.574477195739746,
      "rewards/rejected": -1.9773281812667847,
      "step": 4495
    },
    {
      "epoch": 0.5883372863335242,
      "grad_norm": 9.077364921569824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03505193069577217,
      "logits/rejected": 0.07686743140220642,
      "logps/chosen": -335.3731384277344,
      "logps/rejected": -348.40045166015625,
      "loss": 0.6609,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0156499147415161,
      "rewards/margins": 0.220029816031456,
      "rewards/rejected": -1.2356796264648438,
      "step": 4496
    },
    {
      "epoch": 0.5884681442708759,
      "grad_norm": 9.682958602905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07674793899059296,
      "logits/rejected": -0.34226465225219727,
      "logps/chosen": -497.22235107421875,
      "logps/rejected": -391.99810791015625,
      "loss": 0.7043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6320589780807495,
      "rewards/margins": 0.6616231203079224,
      "rewards/rejected": -1.2936820983886719,
      "step": 4497
    },
    {
      "epoch": 0.5885990022082277,
      "grad_norm": 8.460598945617676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05503658577799797,
      "logits/rejected": -0.08370332419872284,
      "logps/chosen": -532.5116577148438,
      "logps/rejected": -412.75738525390625,
      "loss": 0.5852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9622836709022522,
      "rewards/margins": 0.7045673727989197,
      "rewards/rejected": -1.6668510437011719,
      "step": 4498
    },
    {
      "epoch": 0.5887298601455795,
      "grad_norm": 6.45050573348999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08087208867073059,
      "logits/rejected": 0.07246308028697968,
      "logps/chosen": -447.4291687011719,
      "logps/rejected": -391.1727294921875,
      "loss": 0.5343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23775026202201843,
      "rewards/margins": 0.8886271119117737,
      "rewards/rejected": -1.1263773441314697,
      "step": 4499
    },
    {
      "epoch": 0.5888607180829312,
      "grad_norm": 6.551695346832275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14583580195903778,
      "logits/rejected": -0.14538398385047913,
      "logps/chosen": -601.2091064453125,
      "logps/rejected": -475.59442138671875,
      "loss": 0.463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23459169268608093,
      "rewards/margins": 1.4338955879211426,
      "rewards/rejected": -1.1993038654327393,
      "step": 4500
    },
    {
      "epoch": 0.588991576020283,
      "grad_norm": 3.332625389099121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15434002876281738,
      "logits/rejected": 0.11054962873458862,
      "logps/chosen": -606.604248046875,
      "logps/rejected": -532.857666015625,
      "loss": 0.2478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7898443341255188,
      "rewards/margins": 2.1133873462677,
      "rewards/rejected": -1.3235430717468262,
      "step": 4501
    },
    {
      "epoch": 0.5891224339576348,
      "grad_norm": 6.379399299621582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24837839603424072,
      "logits/rejected": 0.17653173208236694,
      "logps/chosen": -492.1379699707031,
      "logps/rejected": -431.29107666015625,
      "loss": 0.4441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09612694382667542,
      "rewards/margins": 1.0003535747528076,
      "rewards/rejected": -1.0964804887771606,
      "step": 4502
    },
    {
      "epoch": 0.5892532918949865,
      "grad_norm": 7.997769832611084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06026273965835571,
      "logits/rejected": 0.07507874071598053,
      "logps/chosen": -416.7864990234375,
      "logps/rejected": -357.345703125,
      "loss": 0.5842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4915477931499481,
      "rewards/margins": 0.5250930786132812,
      "rewards/rejected": -1.0166409015655518,
      "step": 4503
    },
    {
      "epoch": 0.5893841498323382,
      "grad_norm": 9.569464683532715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22399424016475677,
      "logits/rejected": -0.03902774304151535,
      "logps/chosen": -444.2427062988281,
      "logps/rejected": -318.5441589355469,
      "loss": 0.5944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2603965997695923,
      "rewards/margins": 0.5395368337631226,
      "rewards/rejected": -1.7999334335327148,
      "step": 4504
    },
    {
      "epoch": 0.58951500776969,
      "grad_norm": 8.73320198059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051113568246364594,
      "logits/rejected": -0.12924626469612122,
      "logps/chosen": -504.0015563964844,
      "logps/rejected": -356.3511962890625,
      "loss": 0.8318,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8708746433258057,
      "rewards/margins": 0.27222198247909546,
      "rewards/rejected": -1.1430965662002563,
      "step": 4505
    },
    {
      "epoch": 0.5896458657070418,
      "grad_norm": 6.452402591705322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06800410896539688,
      "logits/rejected": 0.06633789837360382,
      "logps/chosen": -347.5942687988281,
      "logps/rejected": -324.3720703125,
      "loss": 0.4488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25324854254722595,
      "rewards/margins": 1.150817632675171,
      "rewards/rejected": -1.4040660858154297,
      "step": 4506
    },
    {
      "epoch": 0.5897767236443936,
      "grad_norm": 3.152622938156128,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21187281608581543,
      "logits/rejected": 0.17612536251544952,
      "logps/chosen": -585.136474609375,
      "logps/rejected": -511.66796875,
      "loss": 0.1501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41408810019493103,
      "rewards/margins": 2.2239131927490234,
      "rewards/rejected": -1.80982506275177,
      "step": 4507
    },
    {
      "epoch": 0.5899075815817453,
      "grad_norm": 4.781600475311279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043061815202236176,
      "logits/rejected": -0.032628633081912994,
      "logps/chosen": -523.4913330078125,
      "logps/rejected": -480.8606872558594,
      "loss": 0.2515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25850963592529297,
      "rewards/margins": 1.7114487886428833,
      "rewards/rejected": -1.9699584245681763,
      "step": 4508
    },
    {
      "epoch": 0.5900384395190971,
      "grad_norm": 6.579290390014648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12854498624801636,
      "logits/rejected": 0.03333622217178345,
      "logps/chosen": -416.8779602050781,
      "logps/rejected": -361.25732421875,
      "loss": 0.4406,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06019517406821251,
      "rewards/margins": 1.12918221950531,
      "rewards/rejected": -1.1893773078918457,
      "step": 4509
    },
    {
      "epoch": 0.5901692974564489,
      "grad_norm": 3.865138530731201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11630603671073914,
      "logits/rejected": -0.029343344271183014,
      "logps/chosen": -425.22210693359375,
      "logps/rejected": -358.6182556152344,
      "loss": 0.2398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2674604654312134,
      "rewards/margins": 2.0131044387817383,
      "rewards/rejected": -1.745643973350525,
      "step": 4510
    },
    {
      "epoch": 0.5903001553938007,
      "grad_norm": 6.647284030914307,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15929022431373596,
      "logits/rejected": 0.30645930767059326,
      "logps/chosen": -648.123046875,
      "logps/rejected": -487.8653259277344,
      "loss": 0.4503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26754674315452576,
      "rewards/margins": 1.2510547637939453,
      "rewards/rejected": -1.518601655960083,
      "step": 4511
    },
    {
      "epoch": 0.5904310133311523,
      "grad_norm": 9.491320610046387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0006960108876228333,
      "logits/rejected": 0.13735553622245789,
      "logps/chosen": -316.25848388671875,
      "logps/rejected": -269.8904724121094,
      "loss": 0.7429,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8186922669410706,
      "rewards/margins": 0.22354929149150848,
      "rewards/rejected": -1.0422415733337402,
      "step": 4512
    },
    {
      "epoch": 0.5905618712685041,
      "grad_norm": 6.907168388366699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30129218101501465,
      "logits/rejected": 0.0863737091422081,
      "logps/chosen": -389.7978515625,
      "logps/rejected": -360.36956787109375,
      "loss": 0.3912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03292521834373474,
      "rewards/margins": 1.2315216064453125,
      "rewards/rejected": -1.198596477508545,
      "step": 4513
    },
    {
      "epoch": 0.5906927292058559,
      "grad_norm": 5.340584754943848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04007061570882797,
      "logits/rejected": 0.11749953776597977,
      "logps/chosen": -332.1024475097656,
      "logps/rejected": -388.5895080566406,
      "loss": 0.412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1607433408498764,
      "rewards/margins": 1.248246669769287,
      "rewards/rejected": -1.4089899063110352,
      "step": 4514
    },
    {
      "epoch": 0.5908235871432077,
      "grad_norm": 12.831077575683594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09450127184391022,
      "logits/rejected": -0.26049521565437317,
      "logps/chosen": -484.3626708984375,
      "logps/rejected": -527.0760498046875,
      "loss": 0.999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2953951358795166,
      "rewards/margins": -0.13930892944335938,
      "rewards/rejected": -1.1560862064361572,
      "step": 4515
    },
    {
      "epoch": 0.5909544450805594,
      "grad_norm": 9.390189170837402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.148127481341362,
      "logits/rejected": -0.20096619427204132,
      "logps/chosen": -580.785888671875,
      "logps/rejected": -458.6410217285156,
      "loss": 0.639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34055987000465393,
      "rewards/margins": 0.3573434054851532,
      "rewards/rejected": -0.6979032754898071,
      "step": 4516
    },
    {
      "epoch": 0.5910853030179112,
      "grad_norm": 9.097785949707031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028858531266450882,
      "logits/rejected": -0.08984369039535522,
      "logps/chosen": -531.2536010742188,
      "logps/rejected": -551.9054565429688,
      "loss": 0.644,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7223869562149048,
      "rewards/margins": 1.0681735277175903,
      "rewards/rejected": -1.7905604839324951,
      "step": 4517
    },
    {
      "epoch": 0.591216160955263,
      "grad_norm": 8.123184204101562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0989939346909523,
      "logits/rejected": -0.06214236468076706,
      "logps/chosen": -505.1116027832031,
      "logps/rejected": -383.78839111328125,
      "loss": 0.4559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2374858856201172,
      "rewards/margins": 1.0586416721343994,
      "rewards/rejected": -1.2961275577545166,
      "step": 4518
    },
    {
      "epoch": 0.5913470188926147,
      "grad_norm": 10.351593971252441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1218801811337471,
      "logits/rejected": -0.10838808119297028,
      "logps/chosen": -671.6885375976562,
      "logps/rejected": -483.5283508300781,
      "loss": 0.8193,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.058600619435310364,
      "rewards/margins": 0.6843145489692688,
      "rewards/rejected": -0.742915153503418,
      "step": 4519
    },
    {
      "epoch": 0.5914778768299664,
      "grad_norm": 12.708049774169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006892256438732147,
      "logits/rejected": 0.22661249339580536,
      "logps/chosen": -487.52777099609375,
      "logps/rejected": -498.3288269042969,
      "loss": 0.5723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48404690623283386,
      "rewards/margins": 0.8980201482772827,
      "rewards/rejected": -1.3820672035217285,
      "step": 4520
    },
    {
      "epoch": 0.5916087347673182,
      "grad_norm": 10.587611198425293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.256126344203949,
      "logits/rejected": 0.1694212406873703,
      "logps/chosen": -543.1246337890625,
      "logps/rejected": -441.11993408203125,
      "loss": 0.6369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30243510007858276,
      "rewards/margins": 0.8041057586669922,
      "rewards/rejected": -1.1065409183502197,
      "step": 4521
    },
    {
      "epoch": 0.59173959270467,
      "grad_norm": 7.035658836364746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30192720890045166,
      "logits/rejected": 0.12909796833992004,
      "logps/chosen": -476.54144287109375,
      "logps/rejected": -434.2010803222656,
      "loss": 0.6491,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3117198944091797,
      "rewards/margins": 0.9555903077125549,
      "rewards/rejected": -1.2673102617263794,
      "step": 4522
    },
    {
      "epoch": 0.5918704506420217,
      "grad_norm": 9.549766540527344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014637984335422516,
      "logits/rejected": -0.18231427669525146,
      "logps/chosen": -556.1389770507812,
      "logps/rejected": -353.3286437988281,
      "loss": 0.4843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4312509596347809,
      "rewards/margins": 1.262934923171997,
      "rewards/rejected": -1.6941858530044556,
      "step": 4523
    },
    {
      "epoch": 0.5920013085793735,
      "grad_norm": 5.492033958435059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3844425082206726,
      "logits/rejected": 0.20132650434970856,
      "logps/chosen": -581.4166259765625,
      "logps/rejected": -538.8214111328125,
      "loss": 0.3142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5187160968780518,
      "rewards/margins": 1.9968677759170532,
      "rewards/rejected": -1.478151559829712,
      "step": 4524
    },
    {
      "epoch": 0.5921321665167253,
      "grad_norm": 6.351065158843994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0509147047996521,
      "logits/rejected": 0.09108351171016693,
      "logps/chosen": -578.1046752929688,
      "logps/rejected": -677.5650024414062,
      "loss": 0.557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5290384888648987,
      "rewards/margins": 0.9455018639564514,
      "rewards/rejected": -1.47454035282135,
      "step": 4525
    },
    {
      "epoch": 0.5922630244540771,
      "grad_norm": 4.964465141296387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2204514741897583,
      "logits/rejected": 0.14711803197860718,
      "logps/chosen": -386.1958923339844,
      "logps/rejected": -359.87811279296875,
      "loss": 0.3355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5462145209312439,
      "rewards/margins": 1.083566665649414,
      "rewards/rejected": -1.6297812461853027,
      "step": 4526
    },
    {
      "epoch": 0.5923938823914288,
      "grad_norm": 10.082560539245605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30257144570350647,
      "logits/rejected": 0.2557573914527893,
      "logps/chosen": -359.8699645996094,
      "logps/rejected": -378.74127197265625,
      "loss": 0.8867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9344280958175659,
      "rewards/margins": 0.2159363329410553,
      "rewards/rejected": -1.1503643989562988,
      "step": 4527
    },
    {
      "epoch": 0.5925247403287806,
      "grad_norm": 5.762176990509033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36010634899139404,
      "logits/rejected": 0.0909319519996643,
      "logps/chosen": -368.23028564453125,
      "logps/rejected": -347.4913330078125,
      "loss": 0.4229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7508573532104492,
      "rewards/margins": 0.9615582227706909,
      "rewards/rejected": -1.7124156951904297,
      "step": 4528
    },
    {
      "epoch": 0.5926555982661323,
      "grad_norm": 8.341392517089844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1802869290113449,
      "logits/rejected": -0.04278239607810974,
      "logps/chosen": -421.41021728515625,
      "logps/rejected": -375.9812927246094,
      "loss": 0.6521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.440359890460968,
      "rewards/margins": 0.7517951726913452,
      "rewards/rejected": -1.192155122756958,
      "step": 4529
    },
    {
      "epoch": 0.5927864562034841,
      "grad_norm": 8.022316932678223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024624384939670563,
      "logits/rejected": 0.04085836932063103,
      "logps/chosen": -430.8270568847656,
      "logps/rejected": -385.01739501953125,
      "loss": 0.6247,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8561820983886719,
      "rewards/margins": 0.40119630098342896,
      "rewards/rejected": -1.257378339767456,
      "step": 4530
    },
    {
      "epoch": 0.5929173141408358,
      "grad_norm": 11.22177791595459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27141618728637695,
      "logits/rejected": -0.1872730255126953,
      "logps/chosen": -516.0452270507812,
      "logps/rejected": -367.1065368652344,
      "loss": 0.4759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19881325960159302,
      "rewards/margins": 1.4350383281707764,
      "rewards/rejected": -1.6338515281677246,
      "step": 4531
    },
    {
      "epoch": 0.5930481720781876,
      "grad_norm": 6.396149635314941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10519111901521683,
      "logits/rejected": 0.129411980509758,
      "logps/chosen": -468.1165466308594,
      "logps/rejected": -426.8905334472656,
      "loss": 0.3943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2072908580303192,
      "rewards/margins": 0.9404788613319397,
      "rewards/rejected": -1.147769808769226,
      "step": 4532
    },
    {
      "epoch": 0.5931790300155394,
      "grad_norm": 8.866060256958008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021603470668196678,
      "logits/rejected": 0.23753134906291962,
      "logps/chosen": -279.5908508300781,
      "logps/rejected": -314.3548278808594,
      "loss": 0.9135,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6583666205406189,
      "rewards/margins": -0.09663432091474533,
      "rewards/rejected": -0.561732292175293,
      "step": 4533
    },
    {
      "epoch": 0.5933098879528912,
      "grad_norm": 7.928110122680664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002295374870300293,
      "logits/rejected": -0.09863882511854172,
      "logps/chosen": -404.2963562011719,
      "logps/rejected": -458.0610046386719,
      "loss": 0.624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26129090785980225,
      "rewards/margins": 0.8230481147766113,
      "rewards/rejected": -1.0843390226364136,
      "step": 4534
    },
    {
      "epoch": 0.5934407458902429,
      "grad_norm": 7.613081932067871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14052216708660126,
      "logits/rejected": 0.08697521686553955,
      "logps/chosen": -496.0536193847656,
      "logps/rejected": -504.4363708496094,
      "loss": 0.5382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33670100569725037,
      "rewards/margins": 0.7322519421577454,
      "rewards/rejected": -1.068953037261963,
      "step": 4535
    },
    {
      "epoch": 0.5935716038275947,
      "grad_norm": 4.769434452056885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07645586133003235,
      "logits/rejected": -0.26820650696754456,
      "logps/chosen": -589.8675537109375,
      "logps/rejected": -385.4164733886719,
      "loss": 0.2233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3326786160469055,
      "rewards/margins": 2.3666462898254395,
      "rewards/rejected": -2.0339677333831787,
      "step": 4536
    },
    {
      "epoch": 0.5937024617649465,
      "grad_norm": 4.224803447723389,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2759115695953369,
      "logits/rejected": 0.28925174474716187,
      "logps/chosen": -369.8087158203125,
      "logps/rejected": -346.9665222167969,
      "loss": 0.2946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08149205148220062,
      "rewards/margins": 1.6778472661972046,
      "rewards/rejected": -1.7593393325805664,
      "step": 4537
    },
    {
      "epoch": 0.5938333197022982,
      "grad_norm": 9.392082214355469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24734653532505035,
      "logits/rejected": -0.16934217512607574,
      "logps/chosen": -425.2039794921875,
      "logps/rejected": -579.83154296875,
      "loss": 0.5576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6366525888442993,
      "rewards/margins": 0.9200557470321655,
      "rewards/rejected": -1.5567082166671753,
      "step": 4538
    },
    {
      "epoch": 0.5939641776396499,
      "grad_norm": 4.772029399871826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10958895087242126,
      "logits/rejected": 0.23936086893081665,
      "logps/chosen": -623.7127075195312,
      "logps/rejected": -474.69036865234375,
      "loss": 0.3564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5116450786590576,
      "rewards/margins": 1.0902676582336426,
      "rewards/rejected": -0.5786224603652954,
      "step": 4539
    },
    {
      "epoch": 0.5940950355770017,
      "grad_norm": 6.750232219696045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21257665753364563,
      "logits/rejected": 0.1751510202884674,
      "logps/chosen": -341.83251953125,
      "logps/rejected": -269.17242431640625,
      "loss": 0.4838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6467138528823853,
      "rewards/margins": 0.549252450466156,
      "rewards/rejected": -1.1959662437438965,
      "step": 4540
    },
    {
      "epoch": 0.5942258935143535,
      "grad_norm": 8.455721855163574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04541599750518799,
      "logits/rejected": 0.12917214632034302,
      "logps/chosen": -492.62408447265625,
      "logps/rejected": -416.86993408203125,
      "loss": 0.5897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16994133591651917,
      "rewards/margins": 0.5748251080513,
      "rewards/rejected": -0.4048837721347809,
      "step": 4541
    },
    {
      "epoch": 0.5943567514517052,
      "grad_norm": 6.579319477081299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11049152910709381,
      "logits/rejected": -0.05331948399543762,
      "logps/chosen": -413.9024658203125,
      "logps/rejected": -516.8486328125,
      "loss": 0.4477,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6980526447296143,
      "rewards/margins": 1.0314671993255615,
      "rewards/rejected": -1.7295198440551758,
      "step": 4542
    },
    {
      "epoch": 0.594487609389057,
      "grad_norm": 7.113679885864258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1928587555885315,
      "logits/rejected": 0.2558947801589966,
      "logps/chosen": -265.67694091796875,
      "logps/rejected": -339.1835632324219,
      "loss": 0.6277,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7457472085952759,
      "rewards/margins": 0.2617311477661133,
      "rewards/rejected": -1.0074783563613892,
      "step": 4543
    },
    {
      "epoch": 0.5946184673264088,
      "grad_norm": 4.960143089294434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037260010838508606,
      "logits/rejected": 0.0980038195848465,
      "logps/chosen": -402.94415283203125,
      "logps/rejected": -341.754150390625,
      "loss": 0.2821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18407556414604187,
      "rewards/margins": 1.633927583694458,
      "rewards/rejected": -1.8180031776428223,
      "step": 4544
    },
    {
      "epoch": 0.5947493252637606,
      "grad_norm": 7.886208534240723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08731591701507568,
      "logits/rejected": 0.027640845626592636,
      "logps/chosen": -389.03662109375,
      "logps/rejected": -515.7571411132812,
      "loss": 0.7593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2798175811767578,
      "rewards/margins": 0.31632572412490845,
      "rewards/rejected": -0.596143364906311,
      "step": 4545
    },
    {
      "epoch": 0.5948801832011122,
      "grad_norm": 7.931173801422119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12845563888549805,
      "logits/rejected": 0.1056012436747551,
      "logps/chosen": -439.3028564453125,
      "logps/rejected": -384.51788330078125,
      "loss": 0.6716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7118940353393555,
      "rewards/margins": 0.31588059663772583,
      "rewards/rejected": -1.0277745723724365,
      "step": 4546
    },
    {
      "epoch": 0.595011041138464,
      "grad_norm": 8.429553985595703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09524039924144745,
      "logits/rejected": 0.20131924748420715,
      "logps/chosen": -404.77838134765625,
      "logps/rejected": -367.1228332519531,
      "loss": 0.7846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7490278482437134,
      "rewards/margins": 0.21129971742630005,
      "rewards/rejected": -0.9603275656700134,
      "step": 4547
    },
    {
      "epoch": 0.5951418990758158,
      "grad_norm": 4.668543338775635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09984979778528214,
      "logits/rejected": -0.14733003079891205,
      "logps/chosen": -480.5153503417969,
      "logps/rejected": -532.6597900390625,
      "loss": 0.2707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2691338062286377,
      "rewards/margins": 1.625745415687561,
      "rewards/rejected": -1.8948793411254883,
      "step": 4548
    },
    {
      "epoch": 0.5952727570131676,
      "grad_norm": 5.644165515899658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2384970784187317,
      "logits/rejected": -0.191900834441185,
      "logps/chosen": -430.1407470703125,
      "logps/rejected": -331.25933837890625,
      "loss": 0.4825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07586001604795456,
      "rewards/margins": 0.5530976057052612,
      "rewards/rejected": -0.6289575695991516,
      "step": 4549
    },
    {
      "epoch": 0.5954036149505193,
      "grad_norm": 4.847588062286377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08338549733161926,
      "logits/rejected": -0.03340264782309532,
      "logps/chosen": -598.37890625,
      "logps/rejected": -499.6934814453125,
      "loss": 0.3736,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11940537393093109,
      "rewards/margins": 1.2772637605667114,
      "rewards/rejected": -1.15785813331604,
      "step": 4550
    },
    {
      "epoch": 0.5955344728878711,
      "grad_norm": 7.2680816650390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24564319849014282,
      "logits/rejected": 0.14817024767398834,
      "logps/chosen": -678.48681640625,
      "logps/rejected": -416.27557373046875,
      "loss": 0.427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6156280040740967,
      "rewards/margins": 1.2106437683105469,
      "rewards/rejected": -0.5950157642364502,
      "step": 4551
    },
    {
      "epoch": 0.5956653308252229,
      "grad_norm": 6.851123332977295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15866217017173767,
      "logits/rejected": -0.006702916696667671,
      "logps/chosen": -260.20086669921875,
      "logps/rejected": -376.1245422363281,
      "loss": 0.3263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36356469988822937,
      "rewards/margins": 1.268580436706543,
      "rewards/rejected": -1.6321451663970947,
      "step": 4552
    },
    {
      "epoch": 0.5957961887625747,
      "grad_norm": 2.859297513961792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15392443537712097,
      "logits/rejected": 0.05043896287679672,
      "logps/chosen": -452.88104248046875,
      "logps/rejected": -426.89764404296875,
      "loss": 0.1153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42010498046875,
      "rewards/margins": 2.5876049995422363,
      "rewards/rejected": -2.1675000190734863,
      "step": 4553
    },
    {
      "epoch": 0.5959270466999264,
      "grad_norm": 6.183025360107422,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043860889971256256,
      "logits/rejected": 0.0219135582447052,
      "logps/chosen": -456.39959716796875,
      "logps/rejected": -342.67022705078125,
      "loss": 0.4095,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12295800447463989,
      "rewards/margins": 0.9415954947471619,
      "rewards/rejected": -1.0645534992218018,
      "step": 4554
    },
    {
      "epoch": 0.5960579046372781,
      "grad_norm": 5.644517421722412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15371298789978027,
      "logits/rejected": -0.04233338683843613,
      "logps/chosen": -346.69110107421875,
      "logps/rejected": -413.53125,
      "loss": 0.4446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4096572995185852,
      "rewards/margins": 1.1684125661849976,
      "rewards/rejected": -1.5780699253082275,
      "step": 4555
    },
    {
      "epoch": 0.5961887625746299,
      "grad_norm": 9.433810234069824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07636980712413788,
      "logits/rejected": -0.0497334748506546,
      "logps/chosen": -669.7666015625,
      "logps/rejected": -465.5524597167969,
      "loss": 0.4309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6012278199195862,
      "rewards/margins": 1.2714331150054932,
      "rewards/rejected": -1.8726608753204346,
      "step": 4556
    },
    {
      "epoch": 0.5963196205119817,
      "grad_norm": 8.441250801086426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20226074755191803,
      "logits/rejected": -0.1465889811515808,
      "logps/chosen": -373.4985656738281,
      "logps/rejected": -472.7390441894531,
      "loss": 0.6588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7267072200775146,
      "rewards/margins": 0.28660234808921814,
      "rewards/rejected": -1.0133094787597656,
      "step": 4557
    },
    {
      "epoch": 0.5964504784493334,
      "grad_norm": 4.820387840270996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18000167608261108,
      "logits/rejected": 0.04404657706618309,
      "logps/chosen": -502.7071533203125,
      "logps/rejected": -478.4898986816406,
      "loss": 0.3614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3603954315185547,
      "rewards/margins": 1.4600275754928589,
      "rewards/rejected": -1.0996321439743042,
      "step": 4558
    },
    {
      "epoch": 0.5965813363866852,
      "grad_norm": 7.60563325881958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1420157253742218,
      "logits/rejected": 0.08652912825345993,
      "logps/chosen": -458.1829833984375,
      "logps/rejected": -433.7452087402344,
      "loss": 0.5097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09456119686365128,
      "rewards/margins": 0.5664543509483337,
      "rewards/rejected": -0.4718931317329407,
      "step": 4559
    },
    {
      "epoch": 0.596712194324037,
      "grad_norm": 7.3054938316345215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21492740511894226,
      "logits/rejected": 0.29800111055374146,
      "logps/chosen": -495.47955322265625,
      "logps/rejected": -457.02935791015625,
      "loss": 0.4926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8408210873603821,
      "rewards/margins": 0.7267968654632568,
      "rewards/rejected": -1.5676180124282837,
      "step": 4560
    },
    {
      "epoch": 0.5968430522613888,
      "grad_norm": 5.974444389343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0017660539597272873,
      "logits/rejected": -0.10922732949256897,
      "logps/chosen": -405.1540222167969,
      "logps/rejected": -320.8695983886719,
      "loss": 0.6952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5579990148544312,
      "rewards/margins": 0.2368035912513733,
      "rewards/rejected": -0.7948026657104492,
      "step": 4561
    },
    {
      "epoch": 0.5969739101987405,
      "grad_norm": 4.592033386230469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3122197985649109,
      "logits/rejected": 0.2752932906150818,
      "logps/chosen": -557.8154296875,
      "logps/rejected": -371.3345947265625,
      "loss": 0.2622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.36212313175201416,
      "rewards/margins": 1.7480871677398682,
      "rewards/rejected": -1.385964035987854,
      "step": 4562
    },
    {
      "epoch": 0.5971047681360923,
      "grad_norm": 6.716740131378174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.391169935464859,
      "logits/rejected": -0.236092209815979,
      "logps/chosen": -253.1143798828125,
      "logps/rejected": -365.87286376953125,
      "loss": 0.6225,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7171779870986938,
      "rewards/margins": 0.4379694163799286,
      "rewards/rejected": -1.1551474332809448,
      "step": 4563
    },
    {
      "epoch": 0.597235626073444,
      "grad_norm": 5.864742755889893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012635696679353714,
      "logits/rejected": -0.18857084214687347,
      "logps/chosen": -414.02276611328125,
      "logps/rejected": -488.6160888671875,
      "loss": 0.3553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8163042664527893,
      "rewards/margins": 1.2965357303619385,
      "rewards/rejected": -2.112840175628662,
      "step": 4564
    },
    {
      "epoch": 0.5973664840107957,
      "grad_norm": 8.166901588439941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09993499517440796,
      "logits/rejected": 0.09694056212902069,
      "logps/chosen": -355.1445007324219,
      "logps/rejected": -378.14154052734375,
      "loss": 0.5049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6859508752822876,
      "rewards/margins": 0.8828554153442383,
      "rewards/rejected": -1.5688062906265259,
      "step": 4565
    },
    {
      "epoch": 0.5974973419481475,
      "grad_norm": 5.786052227020264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.164771169424057,
      "logits/rejected": 0.01786050572991371,
      "logps/chosen": -385.786865234375,
      "logps/rejected": -299.8752136230469,
      "loss": 0.3847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1994699388742447,
      "rewards/margins": 1.0290770530700684,
      "rewards/rejected": -1.2285469770431519,
      "step": 4566
    },
    {
      "epoch": 0.5976281998854993,
      "grad_norm": 8.060834884643555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031978264451026917,
      "logits/rejected": 0.04148755222558975,
      "logps/chosen": -553.8953857421875,
      "logps/rejected": -443.89727783203125,
      "loss": 0.4272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5312411785125732,
      "rewards/margins": 1.2287746667861938,
      "rewards/rejected": -1.7600159645080566,
      "step": 4567
    },
    {
      "epoch": 0.5977590578228511,
      "grad_norm": 9.28225040435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017377406358718872,
      "logits/rejected": 0.07927384972572327,
      "logps/chosen": -595.39208984375,
      "logps/rejected": -602.9544677734375,
      "loss": 0.5549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.477935552597046,
      "rewards/margins": 0.4906315207481384,
      "rewards/rejected": -1.96856689453125,
      "step": 4568
    },
    {
      "epoch": 0.5978899157602028,
      "grad_norm": 7.199757099151611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19007840752601624,
      "logits/rejected": -0.25135019421577454,
      "logps/chosen": -407.49615478515625,
      "logps/rejected": -457.730224609375,
      "loss": 0.5785,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8950392007827759,
      "rewards/margins": 1.0564509630203247,
      "rewards/rejected": -1.9514901638031006,
      "step": 4569
    },
    {
      "epoch": 0.5980207736975546,
      "grad_norm": 7.774134635925293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16686993837356567,
      "logits/rejected": -0.12687353789806366,
      "logps/chosen": -410.8333740234375,
      "logps/rejected": -424.3594055175781,
      "loss": 0.7105,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9234063625335693,
      "rewards/margins": 0.571495771408081,
      "rewards/rejected": -1.49490225315094,
      "step": 4570
    },
    {
      "epoch": 0.5981516316349064,
      "grad_norm": 7.935460567474365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0033270902931690216,
      "logits/rejected": 0.09604819118976593,
      "logps/chosen": -429.421630859375,
      "logps/rejected": -348.46038818359375,
      "loss": 0.5918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8062235116958618,
      "rewards/margins": 0.7396449446678162,
      "rewards/rejected": -1.5458685159683228,
      "step": 4571
    },
    {
      "epoch": 0.5982824895722582,
      "grad_norm": 6.548283100128174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042867839336395264,
      "logits/rejected": -0.056475237011909485,
      "logps/chosen": -547.4647216796875,
      "logps/rejected": -472.2287292480469,
      "loss": 0.4427,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31632959842681885,
      "rewards/margins": 0.8247036933898926,
      "rewards/rejected": -1.1410331726074219,
      "step": 4572
    },
    {
      "epoch": 0.5984133475096098,
      "grad_norm": 8.328643798828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15888378024101257,
      "logits/rejected": 0.2001248002052307,
      "logps/chosen": -421.750244140625,
      "logps/rejected": -490.93798828125,
      "loss": 0.5278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23990194499492645,
      "rewards/margins": 0.60472571849823,
      "rewards/rejected": -0.8446276187896729,
      "step": 4573
    },
    {
      "epoch": 0.5985442054469616,
      "grad_norm": 6.1141839027404785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2271001935005188,
      "logits/rejected": -0.03762246295809746,
      "logps/chosen": -300.7955627441406,
      "logps/rejected": -370.2254333496094,
      "loss": 0.4386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24058552086353302,
      "rewards/margins": 0.8239431977272034,
      "rewards/rejected": -1.0645287036895752,
      "step": 4574
    },
    {
      "epoch": 0.5986750633843134,
      "grad_norm": 5.170656681060791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16696122288703918,
      "logits/rejected": 0.051282621920108795,
      "logps/chosen": -388.24285888671875,
      "logps/rejected": -448.9576110839844,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7470443844795227,
      "rewards/margins": 0.8738216161727905,
      "rewards/rejected": -1.620866060256958,
      "step": 4575
    },
    {
      "epoch": 0.5988059213216652,
      "grad_norm": 10.641396522521973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13206161558628082,
      "logits/rejected": 0.1843731701374054,
      "logps/chosen": -552.60546875,
      "logps/rejected": -417.8511962890625,
      "loss": 0.72,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4234766960144043,
      "rewards/margins": 0.29291579127311707,
      "rewards/rejected": -1.7163923978805542,
      "step": 4576
    },
    {
      "epoch": 0.5989367792590169,
      "grad_norm": 4.839348793029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24682363867759705,
      "logits/rejected": -0.02920297160744667,
      "logps/chosen": -478.9351806640625,
      "logps/rejected": -534.4886474609375,
      "loss": 0.3551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43635958433151245,
      "rewards/margins": 1.2639994621276855,
      "rewards/rejected": -1.7003588676452637,
      "step": 4577
    },
    {
      "epoch": 0.5990676371963687,
      "grad_norm": 4.916832447052002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17802654206752777,
      "logits/rejected": 0.23457495868206024,
      "logps/chosen": -511.9441223144531,
      "logps/rejected": -400.7259216308594,
      "loss": 0.3148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46110421419143677,
      "rewards/margins": 1.5384479761123657,
      "rewards/rejected": -1.9995523691177368,
      "step": 4578
    },
    {
      "epoch": 0.5991984951337205,
      "grad_norm": 11.42037296295166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08027257025241852,
      "logits/rejected": 0.07946458458900452,
      "logps/chosen": -654.82080078125,
      "logps/rejected": -851.0776977539062,
      "loss": 0.7032,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2446770668029785,
      "rewards/margins": 0.6475638151168823,
      "rewards/rejected": -1.8922408819198608,
      "step": 4579
    },
    {
      "epoch": 0.5993293530710723,
      "grad_norm": 7.972971439361572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046484485268592834,
      "logits/rejected": -0.036953411996364594,
      "logps/chosen": -389.97552490234375,
      "logps/rejected": -403.65179443359375,
      "loss": 0.805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8078689575195312,
      "rewards/margins": 0.3349027633666992,
      "rewards/rejected": -1.1427717208862305,
      "step": 4580
    },
    {
      "epoch": 0.599460211008424,
      "grad_norm": 6.380738258361816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13575445115566254,
      "logits/rejected": -0.324155330657959,
      "logps/chosen": -525.2904052734375,
      "logps/rejected": -361.2447509765625,
      "loss": 0.5967,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4030742645263672,
      "rewards/margins": 0.5216072201728821,
      "rewards/rejected": -1.9246814250946045,
      "step": 4581
    },
    {
      "epoch": 0.5995910689457757,
      "grad_norm": 10.740571975708008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06095859035849571,
      "logits/rejected": 0.07557934522628784,
      "logps/chosen": -610.7192993164062,
      "logps/rejected": -645.3073120117188,
      "loss": 0.7724,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8035674095153809,
      "rewards/margins": 0.3645452857017517,
      "rewards/rejected": -1.1681126356124878,
      "step": 4582
    },
    {
      "epoch": 0.5997219268831275,
      "grad_norm": 12.343718528747559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2927248775959015,
      "logits/rejected": -0.3102589249610901,
      "logps/chosen": -355.626953125,
      "logps/rejected": -361.0464782714844,
      "loss": 1.2028,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3690259456634521,
      "rewards/margins": 0.17354106903076172,
      "rewards/rejected": -1.5425670146942139,
      "step": 4583
    },
    {
      "epoch": 0.5998527848204792,
      "grad_norm": 7.158392906188965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13772405683994293,
      "logits/rejected": -0.09608373790979385,
      "logps/chosen": -364.6054992675781,
      "logps/rejected": -468.7045593261719,
      "loss": 0.4667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5453645586967468,
      "rewards/margins": 1.3363316059112549,
      "rewards/rejected": -1.8816962242126465,
      "step": 4584
    },
    {
      "epoch": 0.599983642757831,
      "grad_norm": 6.046929359436035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.263484388589859,
      "logits/rejected": 0.29652610421180725,
      "logps/chosen": -401.9200134277344,
      "logps/rejected": -415.1975402832031,
      "loss": 0.3916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9200947284698486,
      "rewards/margins": 1.0255616903305054,
      "rewards/rejected": -1.945656418800354,
      "step": 4585
    },
    {
      "epoch": 0.6001145006951828,
      "grad_norm": 6.352313041687012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09496505558490753,
      "logits/rejected": 0.09566859900951385,
      "logps/chosen": -538.2947387695312,
      "logps/rejected": -498.7166442871094,
      "loss": 0.3612,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09298667311668396,
      "rewards/margins": 2.187580108642578,
      "rewards/rejected": -2.280566692352295,
      "step": 4586
    },
    {
      "epoch": 0.6002453586325346,
      "grad_norm": 12.35444164276123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06446675956249237,
      "logits/rejected": -0.015928447246551514,
      "logps/chosen": -523.623291015625,
      "logps/rejected": -501.5708312988281,
      "loss": 0.9514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7504627704620361,
      "rewards/margins": 0.039088428020477295,
      "rewards/rejected": -0.7895511388778687,
      "step": 4587
    },
    {
      "epoch": 0.6003762165698863,
      "grad_norm": 7.528454303741455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017762716859579086,
      "logits/rejected": 0.011459402740001678,
      "logps/chosen": -371.0293273925781,
      "logps/rejected": -594.4754028320312,
      "loss": 0.6174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1504995822906494,
      "rewards/margins": 0.3619661331176758,
      "rewards/rejected": -1.5124657154083252,
      "step": 4588
    },
    {
      "epoch": 0.6005070745072381,
      "grad_norm": 5.60279655456543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08755025267601013,
      "logits/rejected": -0.031626179814338684,
      "logps/chosen": -395.7994384765625,
      "logps/rejected": -402.0927429199219,
      "loss": 0.5098,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3168767988681793,
      "rewards/margins": 0.8335992693901062,
      "rewards/rejected": -1.150476098060608,
      "step": 4589
    },
    {
      "epoch": 0.6006379324445899,
      "grad_norm": 6.084801197052002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16744175553321838,
      "logits/rejected": 0.1321820169687271,
      "logps/chosen": -447.7573547363281,
      "logps/rejected": -487.82843017578125,
      "loss": 0.372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5114038586616516,
      "rewards/margins": 1.6494364738464355,
      "rewards/rejected": -2.1608402729034424,
      "step": 4590
    },
    {
      "epoch": 0.6007687903819416,
      "grad_norm": 5.785313606262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018769575282931328,
      "logits/rejected": -0.08359625935554504,
      "logps/chosen": -305.986572265625,
      "logps/rejected": -375.7022399902344,
      "loss": 0.4215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8588886260986328,
      "rewards/margins": 1.139642596244812,
      "rewards/rejected": -1.9985313415527344,
      "step": 4591
    },
    {
      "epoch": 0.6008996483192933,
      "grad_norm": 4.112054347991943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2836913466453552,
      "logits/rejected": 0.04855824261903763,
      "logps/chosen": -500.5427551269531,
      "logps/rejected": -426.2720947265625,
      "loss": 0.2465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07076680660247803,
      "rewards/margins": 1.8147014379501343,
      "rewards/rejected": -1.7439346313476562,
      "step": 4592
    },
    {
      "epoch": 0.6010305062566451,
      "grad_norm": 6.728342056274414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30484670400619507,
      "logits/rejected": 0.26024389266967773,
      "logps/chosen": -468.4007263183594,
      "logps/rejected": -405.18896484375,
      "loss": 0.5568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.754254162311554,
      "rewards/margins": 0.5077069997787476,
      "rewards/rejected": -1.2619612216949463,
      "step": 4593
    },
    {
      "epoch": 0.6011613641939969,
      "grad_norm": 8.677298545837402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043297454714775085,
      "logits/rejected": -0.014431416988372803,
      "logps/chosen": -516.0234375,
      "logps/rejected": -468.4006652832031,
      "loss": 0.6343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0365527868270874,
      "rewards/margins": 0.9104294776916504,
      "rewards/rejected": -1.9469821453094482,
      "step": 4594
    },
    {
      "epoch": 0.6012922221313487,
      "grad_norm": 4.875173568725586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19108593463897705,
      "logits/rejected": -0.16792011260986328,
      "logps/chosen": -447.28338623046875,
      "logps/rejected": -469.8545837402344,
      "loss": 0.3357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5177823901176453,
      "rewards/margins": 2.012209177017212,
      "rewards/rejected": -2.529991388320923,
      "step": 4595
    },
    {
      "epoch": 0.6014230800687004,
      "grad_norm": 6.02802038192749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20115070044994354,
      "logits/rejected": -0.10630284249782562,
      "logps/chosen": -345.9999694824219,
      "logps/rejected": -389.9862976074219,
      "loss": 0.3955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.881696879863739,
      "rewards/margins": 1.0117228031158447,
      "rewards/rejected": -1.8934197425842285,
      "step": 4596
    },
    {
      "epoch": 0.6015539380060522,
      "grad_norm": 9.440348625183105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21327883005142212,
      "logits/rejected": 0.238061785697937,
      "logps/chosen": -527.191650390625,
      "logps/rejected": -403.9090576171875,
      "loss": 0.4886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9652458429336548,
      "rewards/margins": 0.7240973711013794,
      "rewards/rejected": -1.6893433332443237,
      "step": 4597
    },
    {
      "epoch": 0.601684795943404,
      "grad_norm": 6.358195781707764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26955315470695496,
      "logits/rejected": 0.28277087211608887,
      "logps/chosen": -626.0521240234375,
      "logps/rejected": -497.0141296386719,
      "loss": 0.3881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6320843696594238,
      "rewards/margins": 1.3881583213806152,
      "rewards/rejected": -2.020242691040039,
      "step": 4598
    },
    {
      "epoch": 0.6018156538807558,
      "grad_norm": 8.081968307495117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12113428860902786,
      "logits/rejected": 0.11394261568784714,
      "logps/chosen": -297.8846130371094,
      "logps/rejected": -448.95648193359375,
      "loss": 0.57,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6231928467750549,
      "rewards/margins": 0.481022447347641,
      "rewards/rejected": -1.1042152643203735,
      "step": 4599
    },
    {
      "epoch": 0.6019465118181074,
      "grad_norm": 9.067615509033203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1341148167848587,
      "logits/rejected": 0.06291352212429047,
      "logps/chosen": -382.873779296875,
      "logps/rejected": -446.3058166503906,
      "loss": 0.8734,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.411591649055481,
      "rewards/margins": -0.10330237448215485,
      "rewards/rejected": -1.3082894086837769,
      "step": 4600
    },
    {
      "epoch": 0.6020773697554592,
      "grad_norm": 7.004837512969971,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1106063723564148,
      "logits/rejected": 0.14129851758480072,
      "logps/chosen": -357.29083251953125,
      "logps/rejected": -465.81787109375,
      "loss": 0.5497,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3551023602485657,
      "rewards/margins": 0.7192636728286743,
      "rewards/rejected": -1.0743659734725952,
      "step": 4601
    },
    {
      "epoch": 0.602208227692811,
      "grad_norm": 10.463698387145996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27933430671691895,
      "logits/rejected": -0.22220151126384735,
      "logps/chosen": -415.71246337890625,
      "logps/rejected": -450.956298828125,
      "loss": 0.7977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.567267894744873,
      "rewards/margins": 0.9291736483573914,
      "rewards/rejected": -2.496441602706909,
      "step": 4602
    },
    {
      "epoch": 0.6023390856301628,
      "grad_norm": 5.679585933685303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17726916074752808,
      "logits/rejected": 0.08202946186065674,
      "logps/chosen": -475.65997314453125,
      "logps/rejected": -470.1593322753906,
      "loss": 0.3742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.35833680629730225,
      "rewards/margins": 2.107879161834717,
      "rewards/rejected": -1.7495423555374146,
      "step": 4603
    },
    {
      "epoch": 0.6024699435675145,
      "grad_norm": 8.056109428405762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18192309141159058,
      "logits/rejected": 0.06237697973847389,
      "logps/chosen": -537.7605590820312,
      "logps/rejected": -665.280517578125,
      "loss": 0.5698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7664005160331726,
      "rewards/margins": 1.0057973861694336,
      "rewards/rejected": -1.772197961807251,
      "step": 4604
    },
    {
      "epoch": 0.6026008015048663,
      "grad_norm": 3.910339832305908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28377392888069153,
      "logits/rejected": 0.034229815006256104,
      "logps/chosen": -633.337646484375,
      "logps/rejected": -501.9658203125,
      "loss": 0.2255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0901014506816864,
      "rewards/margins": 1.6717216968536377,
      "rewards/rejected": -1.5816203355789185,
      "step": 4605
    },
    {
      "epoch": 0.6027316594422181,
      "grad_norm": 5.3127617835998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0024042949080467224,
      "logits/rejected": 0.06545744091272354,
      "logps/chosen": -568.398193359375,
      "logps/rejected": -555.5812377929688,
      "loss": 0.3414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.007043659687042236,
      "rewards/margins": 1.249110221862793,
      "rewards/rejected": -1.2561538219451904,
      "step": 4606
    },
    {
      "epoch": 0.6028625173795698,
      "grad_norm": 10.420169830322266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2090182900428772,
      "logits/rejected": 0.36209163069725037,
      "logps/chosen": -352.13037109375,
      "logps/rejected": -379.05780029296875,
      "loss": 0.6813,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5501994490623474,
      "rewards/margins": 0.6574177145957947,
      "rewards/rejected": -1.2076172828674316,
      "step": 4607
    },
    {
      "epoch": 0.6029933753169215,
      "grad_norm": 4.555394172668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10717304050922394,
      "logits/rejected": 0.04539984092116356,
      "logps/chosen": -512.7962036132812,
      "logps/rejected": -429.58984375,
      "loss": 0.3498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.799720048904419,
      "rewards/margins": 1.3265657424926758,
      "rewards/rejected": -2.1262857913970947,
      "step": 4608
    },
    {
      "epoch": 0.6031242332542733,
      "grad_norm": 10.580704689025879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037794385105371475,
      "logits/rejected": -0.2803505063056946,
      "logps/chosen": -553.3582763671875,
      "logps/rejected": -428.5145568847656,
      "loss": 0.7585,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9984129071235657,
      "rewards/margins": 1.1553083658218384,
      "rewards/rejected": -2.153721332550049,
      "step": 4609
    },
    {
      "epoch": 0.6032550911916251,
      "grad_norm": 4.946013450622559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004544474184513092,
      "logits/rejected": -0.10179604589939117,
      "logps/chosen": -596.48095703125,
      "logps/rejected": -433.652099609375,
      "loss": 0.2542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7836622595787048,
      "rewards/margins": 2.2214176654815674,
      "rewards/rejected": -3.005079984664917,
      "step": 4610
    },
    {
      "epoch": 0.6033859491289768,
      "grad_norm": 4.206801891326904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3345716595649719,
      "logits/rejected": -0.13606825470924377,
      "logps/chosen": -403.7757568359375,
      "logps/rejected": -375.1391906738281,
      "loss": 0.3435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02783621847629547,
      "rewards/margins": 1.023314118385315,
      "rewards/rejected": -0.9954779148101807,
      "step": 4611
    },
    {
      "epoch": 0.6035168070663286,
      "grad_norm": 5.636113166809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3428823947906494,
      "logits/rejected": -0.47962403297424316,
      "logps/chosen": -509.07745361328125,
      "logps/rejected": -495.16900634765625,
      "loss": 0.3005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8366260528564453,
      "rewards/margins": 1.2585198879241943,
      "rewards/rejected": -2.0951461791992188,
      "step": 4612
    },
    {
      "epoch": 0.6036476650036804,
      "grad_norm": 4.396045684814453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015574976801872253,
      "logits/rejected": -0.13950936496257782,
      "logps/chosen": -242.6716766357422,
      "logps/rejected": -275.853271484375,
      "loss": 0.4485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9324846267700195,
      "rewards/margins": 0.8571470379829407,
      "rewards/rejected": -1.7896316051483154,
      "step": 4613
    },
    {
      "epoch": 0.6037785229410322,
      "grad_norm": 5.229666233062744,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016965731978416443,
      "logits/rejected": -0.1743757128715515,
      "logps/chosen": -313.9320983886719,
      "logps/rejected": -332.0022277832031,
      "loss": 0.4134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1684885025024414,
      "rewards/margins": 1.42231023311615,
      "rewards/rejected": -1.5907988548278809,
      "step": 4614
    },
    {
      "epoch": 0.6039093808783839,
      "grad_norm": 8.009262084960938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007626622915267944,
      "logits/rejected": -0.1538122296333313,
      "logps/chosen": -334.0740966796875,
      "logps/rejected": -317.37066650390625,
      "loss": 0.6941,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1740938425064087,
      "rewards/margins": 0.3182101249694824,
      "rewards/rejected": -1.4923039674758911,
      "step": 4615
    },
    {
      "epoch": 0.6040402388157357,
      "grad_norm": 11.384148597717285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03628719970583916,
      "logits/rejected": 0.051151521503925323,
      "logps/chosen": -447.5171813964844,
      "logps/rejected": -739.9949951171875,
      "loss": 0.7926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6192829608917236,
      "rewards/margins": 0.9613510370254517,
      "rewards/rejected": -2.580634117126465,
      "step": 4616
    },
    {
      "epoch": 0.6041710967530874,
      "grad_norm": 6.224289417266846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1996956467628479,
      "logits/rejected": 0.3601490259170532,
      "logps/chosen": -359.74737548828125,
      "logps/rejected": -411.7920227050781,
      "loss": 0.3561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42908555269241333,
      "rewards/margins": 1.309926152229309,
      "rewards/rejected": -1.7390117645263672,
      "step": 4617
    },
    {
      "epoch": 0.6043019546904392,
      "grad_norm": 9.029176712036133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28832319378852844,
      "logits/rejected": -0.044862300157547,
      "logps/chosen": -437.1025695800781,
      "logps/rejected": -380.9736328125,
      "loss": 0.6824,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8919805884361267,
      "rewards/margins": 0.45572778582572937,
      "rewards/rejected": -1.3477083444595337,
      "step": 4618
    },
    {
      "epoch": 0.6044328126277909,
      "grad_norm": 6.170831203460693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0003380328416824341,
      "logits/rejected": -0.045101966708898544,
      "logps/chosen": -407.6951599121094,
      "logps/rejected": -463.4576721191406,
      "loss": 0.3951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.618399977684021,
      "rewards/margins": 2.143732786178589,
      "rewards/rejected": -2.7621328830718994,
      "step": 4619
    },
    {
      "epoch": 0.6045636705651427,
      "grad_norm": 7.119198799133301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08954598754644394,
      "logits/rejected": -0.04692552983760834,
      "logps/chosen": -457.34234619140625,
      "logps/rejected": -450.78363037109375,
      "loss": 0.5105,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8114608526229858,
      "rewards/margins": 0.5905380249023438,
      "rewards/rejected": -1.4019988775253296,
      "step": 4620
    },
    {
      "epoch": 0.6046945285024945,
      "grad_norm": 5.538558483123779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10203772783279419,
      "logits/rejected": -0.30271652340888977,
      "logps/chosen": -413.9796142578125,
      "logps/rejected": -367.3101806640625,
      "loss": 0.4769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.591168999671936,
      "rewards/margins": 1.1105067729949951,
      "rewards/rejected": -1.7016757726669312,
      "step": 4621
    },
    {
      "epoch": 0.6048253864398463,
      "grad_norm": 7.533397674560547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1942019760608673,
      "logits/rejected": 0.08133174479007721,
      "logps/chosen": -551.1884765625,
      "logps/rejected": -460.6895751953125,
      "loss": 0.6206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0549999475479126,
      "rewards/margins": 0.43439751863479614,
      "rewards/rejected": -1.4893975257873535,
      "step": 4622
    },
    {
      "epoch": 0.604956244377198,
      "grad_norm": 3.8994803428649902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23793290555477142,
      "logits/rejected": 0.10579760372638702,
      "logps/chosen": -524.3928833007812,
      "logps/rejected": -423.2872619628906,
      "loss": 0.2251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2472015619277954,
      "rewards/margins": 1.7944999933242798,
      "rewards/rejected": -1.547298550605774,
      "step": 4623
    },
    {
      "epoch": 0.6050871023145498,
      "grad_norm": 6.449070930480957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042007122188806534,
      "logits/rejected": 0.01172507181763649,
      "logps/chosen": -407.385986328125,
      "logps/rejected": -474.025146484375,
      "loss": 0.4394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0538980960845947,
      "rewards/margins": 0.9950373768806458,
      "rewards/rejected": -2.0489351749420166,
      "step": 4624
    },
    {
      "epoch": 0.6052179602519016,
      "grad_norm": 6.0807719230651855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0669388473033905,
      "logits/rejected": -0.2703779339790344,
      "logps/chosen": -483.31011962890625,
      "logps/rejected": -377.3884582519531,
      "loss": 0.4018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9764680862426758,
      "rewards/margins": 1.7173535823822021,
      "rewards/rejected": -2.693821668624878,
      "step": 4625
    },
    {
      "epoch": 0.6053488181892533,
      "grad_norm": 9.497395515441895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2799049913883209,
      "logits/rejected": 0.055804554373025894,
      "logps/chosen": -354.6206359863281,
      "logps/rejected": -487.503173828125,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.949408769607544,
      "rewards/margins": 0.2757560610771179,
      "rewards/rejected": -1.225164771080017,
      "step": 4626
    },
    {
      "epoch": 0.605479676126605,
      "grad_norm": 3.545206308364868,
      "learning_rate": 5e-06,
      "logits/chosen": 0.257793128490448,
      "logits/rejected": 0.13104723393917084,
      "logps/chosen": -579.5138549804688,
      "logps/rejected": -717.8248291015625,
      "loss": 0.2113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1610884964466095,
      "rewards/margins": 2.783233642578125,
      "rewards/rejected": -2.944321870803833,
      "step": 4627
    },
    {
      "epoch": 0.6056105340639568,
      "grad_norm": 6.793230056762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.192042738199234,
      "logits/rejected": 0.06032952293753624,
      "logps/chosen": -515.9589233398438,
      "logps/rejected": -453.9039001464844,
      "loss": 0.5656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6697967052459717,
      "rewards/margins": 0.4430912435054779,
      "rewards/rejected": -2.1128880977630615,
      "step": 4628
    },
    {
      "epoch": 0.6057413920013086,
      "grad_norm": 4.675992965698242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12314674258232117,
      "logits/rejected": 0.011464647948741913,
      "logps/chosen": -473.2738037109375,
      "logps/rejected": -298.45574951171875,
      "loss": 0.3755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2011660933494568,
      "rewards/margins": 1.2535299062728882,
      "rewards/rejected": -1.0523639917373657,
      "step": 4629
    },
    {
      "epoch": 0.6058722499386603,
      "grad_norm": 3.9259731769561768,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3706713616847992,
      "logits/rejected": -0.5488232374191284,
      "logps/chosen": -410.7066345214844,
      "logps/rejected": -478.43450927734375,
      "loss": 0.2756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9083475470542908,
      "rewards/margins": 1.590570330619812,
      "rewards/rejected": -2.498917818069458,
      "step": 4630
    },
    {
      "epoch": 0.6060031078760121,
      "grad_norm": 8.941076278686523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09895482659339905,
      "logits/rejected": -0.0026336880400776863,
      "logps/chosen": -357.3461608886719,
      "logps/rejected": -448.2738037109375,
      "loss": 0.6421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6619987487792969,
      "rewards/margins": 0.6997488737106323,
      "rewards/rejected": -1.3617476224899292,
      "step": 4631
    },
    {
      "epoch": 0.6061339658133639,
      "grad_norm": 4.553560256958008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11488012969493866,
      "logits/rejected": 0.04642049968242645,
      "logps/chosen": -407.84375,
      "logps/rejected": -461.0155029296875,
      "loss": 0.3094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8035507202148438,
      "rewards/margins": 1.506070852279663,
      "rewards/rejected": -2.309621810913086,
      "step": 4632
    },
    {
      "epoch": 0.6062648237507157,
      "grad_norm": 7.395326137542725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010066159069538116,
      "logits/rejected": 0.0738249272108078,
      "logps/chosen": -630.6187133789062,
      "logps/rejected": -663.1748046875,
      "loss": 0.3385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9260033369064331,
      "rewards/margins": 1.8083380460739136,
      "rewards/rejected": -2.7343411445617676,
      "step": 4633
    },
    {
      "epoch": 0.6063956816880673,
      "grad_norm": 7.909873962402344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11485229432582855,
      "logits/rejected": 0.06672386825084686,
      "logps/chosen": -481.949462890625,
      "logps/rejected": -452.7018127441406,
      "loss": 0.5274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7843511700630188,
      "rewards/margins": 1.3659899234771729,
      "rewards/rejected": -2.150341033935547,
      "step": 4634
    },
    {
      "epoch": 0.6065265396254191,
      "grad_norm": 9.274319648742676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.372308611869812,
      "logits/rejected": 0.062368281185626984,
      "logps/chosen": -504.7994384765625,
      "logps/rejected": -491.68975830078125,
      "loss": 0.5203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5986772775650024,
      "rewards/margins": 1.3265743255615234,
      "rewards/rejected": -1.9252514839172363,
      "step": 4635
    },
    {
      "epoch": 0.6066573975627709,
      "grad_norm": 6.655292987823486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2212315797805786,
      "logits/rejected": -0.135943204164505,
      "logps/chosen": -417.8554382324219,
      "logps/rejected": -462.2085266113281,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7228188514709473,
      "rewards/margins": 0.8060968518257141,
      "rewards/rejected": -1.528915524482727,
      "step": 4636
    },
    {
      "epoch": 0.6067882555001227,
      "grad_norm": 14.704380989074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1379399448633194,
      "logits/rejected": 0.11641151458024979,
      "logps/chosen": -441.0264587402344,
      "logps/rejected": -348.46612548828125,
      "loss": 0.8435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.101942300796509,
      "rewards/margins": 0.8200764656066895,
      "rewards/rejected": -2.9220187664031982,
      "step": 4637
    },
    {
      "epoch": 0.6069191134374744,
      "grad_norm": 9.189512252807617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3439967930316925,
      "logits/rejected": 0.31362271308898926,
      "logps/chosen": -457.95147705078125,
      "logps/rejected": -363.12420654296875,
      "loss": 0.6619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7258775234222412,
      "rewards/margins": 1.051978349685669,
      "rewards/rejected": -1.7778558731079102,
      "step": 4638
    },
    {
      "epoch": 0.6070499713748262,
      "grad_norm": 8.943689346313477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10691229999065399,
      "logits/rejected": -0.15900391340255737,
      "logps/chosen": -596.5236206054688,
      "logps/rejected": -558.6522216796875,
      "loss": 0.6061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5196592211723328,
      "rewards/margins": 0.8338133692741394,
      "rewards/rejected": -1.353472352027893,
      "step": 4639
    },
    {
      "epoch": 0.607180829312178,
      "grad_norm": 3.844299793243408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08594905585050583,
      "logits/rejected": -0.03527555987238884,
      "logps/chosen": -414.24871826171875,
      "logps/rejected": -470.4912109375,
      "loss": 0.2357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4088485836982727,
      "rewards/margins": 2.0729384422302246,
      "rewards/rejected": -2.4817869663238525,
      "step": 4640
    },
    {
      "epoch": 0.6073116872495298,
      "grad_norm": 5.719594478607178,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020316176116466522,
      "logits/rejected": 0.012457144446671009,
      "logps/chosen": -520.1259765625,
      "logps/rejected": -516.2257690429688,
      "loss": 0.298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1411354541778564,
      "rewards/margins": 1.7507343292236328,
      "rewards/rejected": -2.8918700218200684,
      "step": 4641
    },
    {
      "epoch": 0.6074425451868815,
      "grad_norm": 5.344538688659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1653691679239273,
      "logits/rejected": 0.02056644670665264,
      "logps/chosen": -353.40191650390625,
      "logps/rejected": -460.3119812011719,
      "loss": 0.2599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9136199355125427,
      "rewards/margins": 1.6801716089248657,
      "rewards/rejected": -2.5937917232513428,
      "step": 4642
    },
    {
      "epoch": 0.6075734031242332,
      "grad_norm": 7.023257732391357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06702287495136261,
      "logits/rejected": 0.012114256620407104,
      "logps/chosen": -550.0907592773438,
      "logps/rejected": -505.9294128417969,
      "loss": 0.5559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12925679981708527,
      "rewards/margins": 1.3817362785339355,
      "rewards/rejected": -1.2524794340133667,
      "step": 4643
    },
    {
      "epoch": 0.607704261061585,
      "grad_norm": 10.9970064163208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2885099947452545,
      "logits/rejected": -0.0726800337433815,
      "logps/chosen": -473.2361145019531,
      "logps/rejected": -452.481689453125,
      "loss": 0.9212,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.618187665939331,
      "rewards/margins": 0.058766186237335205,
      "rewards/rejected": -1.676953911781311,
      "step": 4644
    },
    {
      "epoch": 0.6078351189989368,
      "grad_norm": 5.738739967346191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1151689887046814,
      "logits/rejected": 0.1120985597372055,
      "logps/chosen": -343.53912353515625,
      "logps/rejected": -311.1080322265625,
      "loss": 0.4425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2071518898010254,
      "rewards/margins": 0.7667400240898132,
      "rewards/rejected": -1.9738922119140625,
      "step": 4645
    },
    {
      "epoch": 0.6079659769362885,
      "grad_norm": 10.894740104675293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014376677572727203,
      "logits/rejected": -0.07242950797080994,
      "logps/chosen": -323.37115478515625,
      "logps/rejected": -529.1659545898438,
      "loss": 0.9156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9756839275360107,
      "rewards/margins": 0.4127389192581177,
      "rewards/rejected": -2.388422966003418,
      "step": 4646
    },
    {
      "epoch": 0.6080968348736403,
      "grad_norm": 4.8168230056762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0961952954530716,
      "logits/rejected": -0.09120407700538635,
      "logps/chosen": -473.08404541015625,
      "logps/rejected": -475.41387939453125,
      "loss": 0.3479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5001688003540039,
      "rewards/margins": 1.458164095878601,
      "rewards/rejected": -1.958332896232605,
      "step": 4647
    },
    {
      "epoch": 0.6082276928109921,
      "grad_norm": 9.24829387664795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009336261078715324,
      "logits/rejected": 0.0026392843574285507,
      "logps/chosen": -349.03594970703125,
      "logps/rejected": -538.0439453125,
      "loss": 0.6411,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3253341913223267,
      "rewards/margins": 1.5245773792266846,
      "rewards/rejected": -2.8499114513397217,
      "step": 4648
    },
    {
      "epoch": 0.6083585507483438,
      "grad_norm": 8.256254196166992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0769713893532753,
      "logits/rejected": 0.16237075626850128,
      "logps/chosen": -393.5922546386719,
      "logps/rejected": -493.66033935546875,
      "loss": 0.4168,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8633466362953186,
      "rewards/margins": 2.0812458992004395,
      "rewards/rejected": -2.9445927143096924,
      "step": 4649
    },
    {
      "epoch": 0.6084894086856956,
      "grad_norm": 10.224973678588867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02509627491235733,
      "logits/rejected": 0.21995076537132263,
      "logps/chosen": -396.4552917480469,
      "logps/rejected": -624.812255859375,
      "loss": 0.6487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6505849957466125,
      "rewards/margins": 0.36115384101867676,
      "rewards/rejected": -1.011738657951355,
      "step": 4650
    },
    {
      "epoch": 0.6086202666230474,
      "grad_norm": 3.6646015644073486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09552742540836334,
      "logits/rejected": 0.08406905084848404,
      "logps/chosen": -530.6063232421875,
      "logps/rejected": -577.0245971679688,
      "loss": 0.2028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30290108919143677,
      "rewards/margins": 2.478078603744507,
      "rewards/rejected": -2.175177574157715,
      "step": 4651
    },
    {
      "epoch": 0.6087511245603991,
      "grad_norm": 7.903960704803467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08165329694747925,
      "logits/rejected": 0.11634843051433563,
      "logps/chosen": -425.01165771484375,
      "logps/rejected": -449.1859436035156,
      "loss": 0.4884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.23610258102417,
      "rewards/margins": 1.1837340593338013,
      "rewards/rejected": -2.4198365211486816,
      "step": 4652
    },
    {
      "epoch": 0.6088819824977508,
      "grad_norm": 11.413888931274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12742793560028076,
      "logits/rejected": -0.21956738829612732,
      "logps/chosen": -462.5339660644531,
      "logps/rejected": -403.2415771484375,
      "loss": 0.9228,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0694313049316406,
      "rewards/margins": 0.2356880009174347,
      "rewards/rejected": -1.305119276046753,
      "step": 4653
    },
    {
      "epoch": 0.6090128404351026,
      "grad_norm": 7.385771751403809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13132518529891968,
      "logits/rejected": -0.505179226398468,
      "logps/chosen": -603.437744140625,
      "logps/rejected": -495.4056396484375,
      "loss": 0.4735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26051998138427734,
      "rewards/margins": 0.8040632605552673,
      "rewards/rejected": -1.0645833015441895,
      "step": 4654
    },
    {
      "epoch": 0.6091436983724544,
      "grad_norm": 14.385481834411621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07581570744514465,
      "logits/rejected": -0.13924704492092133,
      "logps/chosen": -651.6767578125,
      "logps/rejected": -440.0938720703125,
      "loss": 0.8979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.512235403060913,
      "rewards/margins": 0.3599369525909424,
      "rewards/rejected": -1.872172236442566,
      "step": 4655
    },
    {
      "epoch": 0.6092745563098062,
      "grad_norm": 8.270490646362305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1352417916059494,
      "logits/rejected": -0.2718079090118408,
      "logps/chosen": -297.0854797363281,
      "logps/rejected": -284.57513427734375,
      "loss": 0.5767,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5807581543922424,
      "rewards/margins": 0.7936882972717285,
      "rewards/rejected": -1.3744465112686157,
      "step": 4656
    },
    {
      "epoch": 0.6094054142471579,
      "grad_norm": 6.638631820678711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2002103179693222,
      "logits/rejected": -0.28317582607269287,
      "logps/chosen": -374.4804382324219,
      "logps/rejected": -275.97674560546875,
      "loss": 0.4639,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2776157855987549,
      "rewards/margins": 0.9570757150650024,
      "rewards/rejected": -1.2346913814544678,
      "step": 4657
    },
    {
      "epoch": 0.6095362721845097,
      "grad_norm": 5.664541721343994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11346040666103363,
      "logits/rejected": -0.20805011689662933,
      "logps/chosen": -736.445068359375,
      "logps/rejected": -604.8656005859375,
      "loss": 0.3655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3672325015068054,
      "rewards/margins": 1.1660915613174438,
      "rewards/rejected": -1.5333242416381836,
      "step": 4658
    },
    {
      "epoch": 0.6096671301218615,
      "grad_norm": 10.586955070495605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1680191308259964,
      "logits/rejected": 0.07704534381628036,
      "logps/chosen": -491.6639709472656,
      "logps/rejected": -568.2443237304688,
      "loss": 0.7654,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7973664999008179,
      "rewards/margins": 0.46773016452789307,
      "rewards/rejected": -1.2650967836380005,
      "step": 4659
    },
    {
      "epoch": 0.6097979880592133,
      "grad_norm": 12.345656394958496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13165709376335144,
      "logits/rejected": -0.12814363837242126,
      "logps/chosen": -475.416259765625,
      "logps/rejected": -366.316650390625,
      "loss": 1.2764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2835464477539062,
      "rewards/margins": -0.23332750797271729,
      "rewards/rejected": -1.050218939781189,
      "step": 4660
    },
    {
      "epoch": 0.6099288459965649,
      "grad_norm": 6.242722988128662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08450306206941605,
      "logits/rejected": 0.07366529107093811,
      "logps/chosen": -436.6014404296875,
      "logps/rejected": -425.60498046875,
      "loss": 0.5143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1949901580810547,
      "rewards/margins": 0.7762114405632019,
      "rewards/rejected": -0.9712016582489014,
      "step": 4661
    },
    {
      "epoch": 0.6100597039339167,
      "grad_norm": 7.777980327606201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26742151379585266,
      "logits/rejected": 0.007953770458698273,
      "logps/chosen": -370.7498474121094,
      "logps/rejected": -387.15704345703125,
      "loss": 0.6551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.732118844985962,
      "rewards/margins": 0.4193313717842102,
      "rewards/rejected": -2.1514499187469482,
      "step": 4662
    },
    {
      "epoch": 0.6101905618712685,
      "grad_norm": 4.8743205070495605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07224977016448975,
      "logits/rejected": -0.21669428050518036,
      "logps/chosen": -322.213134765625,
      "logps/rejected": -295.8486633300781,
      "loss": 0.4012,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3395814895629883,
      "rewards/margins": 1.0990225076675415,
      "rewards/rejected": -1.4386039972305298,
      "step": 4663
    },
    {
      "epoch": 0.6103214198086203,
      "grad_norm": 6.6826605796813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19556047022342682,
      "logits/rejected": -0.15716828405857086,
      "logps/chosen": -404.66583251953125,
      "logps/rejected": -324.4736633300781,
      "loss": 0.6088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2125566601753235,
      "rewards/margins": 0.4669509530067444,
      "rewards/rejected": -0.6795076727867126,
      "step": 4664
    },
    {
      "epoch": 0.610452277745972,
      "grad_norm": 5.6922607421875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1997256726026535,
      "logits/rejected": 0.1974727362394333,
      "logps/chosen": -916.4981079101562,
      "logps/rejected": -616.4910278320312,
      "loss": 0.2901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.011407643556594849,
      "rewards/margins": 1.5907437801361084,
      "rewards/rejected": -1.579336166381836,
      "step": 4665
    },
    {
      "epoch": 0.6105831356833238,
      "grad_norm": 9.699000358581543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015268728137016296,
      "logits/rejected": 0.1731186807155609,
      "logps/chosen": -406.3182373046875,
      "logps/rejected": -480.6716613769531,
      "loss": 0.7273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4410528540611267,
      "rewards/margins": 0.2543870806694031,
      "rewards/rejected": -0.6954399347305298,
      "step": 4666
    },
    {
      "epoch": 0.6107139936206756,
      "grad_norm": 5.96547794342041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06556327641010284,
      "logits/rejected": 0.3083175718784332,
      "logps/chosen": -384.5361633300781,
      "logps/rejected": -361.44683837890625,
      "loss": 0.3377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20906727015972137,
      "rewards/margins": 1.5784056186676025,
      "rewards/rejected": -1.3693383932113647,
      "step": 4667
    },
    {
      "epoch": 0.6108448515580274,
      "grad_norm": 6.721506118774414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16619031131267548,
      "logits/rejected": 0.19585147500038147,
      "logps/chosen": -379.16015625,
      "logps/rejected": -390.4765625,
      "loss": 0.3952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4988609254360199,
      "rewards/margins": 1.3544502258300781,
      "rewards/rejected": -1.8533114194869995,
      "step": 4668
    },
    {
      "epoch": 0.610975709495379,
      "grad_norm": 5.640659332275391,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11564730107784271,
      "logits/rejected": 0.1046399176120758,
      "logps/chosen": -383.4933166503906,
      "logps/rejected": -557.4506225585938,
      "loss": 0.4198,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6649538278579712,
      "rewards/margins": 1.3097132444381714,
      "rewards/rejected": -1.9746670722961426,
      "step": 4669
    },
    {
      "epoch": 0.6111065674327308,
      "grad_norm": 9.660663604736328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3981538414955139,
      "logits/rejected": 0.30488717555999756,
      "logps/chosen": -741.8087768554688,
      "logps/rejected": -667.2167358398438,
      "loss": 0.8055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3179042339324951,
      "rewards/margins": 0.7061344385147095,
      "rewards/rejected": -1.0240386724472046,
      "step": 4670
    },
    {
      "epoch": 0.6112374253700826,
      "grad_norm": 9.921996116638184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09096589684486389,
      "logits/rejected": 0.14741250872612,
      "logps/chosen": -654.5177612304688,
      "logps/rejected": -394.0900573730469,
      "loss": 0.7669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3103904724121094,
      "rewards/margins": 0.8159961700439453,
      "rewards/rejected": -2.1263866424560547,
      "step": 4671
    },
    {
      "epoch": 0.6113682833074343,
      "grad_norm": 4.795433044433594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12925952672958374,
      "logits/rejected": 0.24709582328796387,
      "logps/chosen": -601.1138916015625,
      "logps/rejected": -454.4649353027344,
      "loss": 0.2876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6748167276382446,
      "rewards/margins": 1.457897424697876,
      "rewards/rejected": -2.132714033126831,
      "step": 4672
    },
    {
      "epoch": 0.6114991412447861,
      "grad_norm": 6.79123592376709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11240477859973907,
      "logits/rejected": -0.14423330128192902,
      "logps/chosen": -535.9493408203125,
      "logps/rejected": -565.920654296875,
      "loss": 0.4041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5181782245635986,
      "rewards/margins": 1.2588378190994263,
      "rewards/rejected": -1.7770161628723145,
      "step": 4673
    },
    {
      "epoch": 0.6116299991821379,
      "grad_norm": 7.015087604522705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03048844262957573,
      "logits/rejected": -0.09991823136806488,
      "logps/chosen": -627.181884765625,
      "logps/rejected": -656.00830078125,
      "loss": 0.3959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.120904341340065,
      "rewards/margins": 0.9232855439186096,
      "rewards/rejected": -1.0441899299621582,
      "step": 4674
    },
    {
      "epoch": 0.6117608571194897,
      "grad_norm": 7.579002857208252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2518618404865265,
      "logits/rejected": 0.21304062008857727,
      "logps/chosen": -389.26483154296875,
      "logps/rejected": -312.8209533691406,
      "loss": 0.5148,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5096719264984131,
      "rewards/margins": 0.917416512966156,
      "rewards/rejected": -1.4270884990692139,
      "step": 4675
    },
    {
      "epoch": 0.6118917150568414,
      "grad_norm": 6.774931907653809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06103098392486572,
      "logits/rejected": 0.15502694249153137,
      "logps/chosen": -347.3166198730469,
      "logps/rejected": -569.2374877929688,
      "loss": 0.4261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5490961074829102,
      "rewards/margins": 1.0077810287475586,
      "rewards/rejected": -1.5568772554397583,
      "step": 4676
    },
    {
      "epoch": 0.6120225729941932,
      "grad_norm": 9.632083892822266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44592493772506714,
      "logits/rejected": 0.5149376392364502,
      "logps/chosen": -676.53515625,
      "logps/rejected": -692.1112060546875,
      "loss": 0.6688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2038017362356186,
      "rewards/margins": 0.6279750466346741,
      "rewards/rejected": -0.8317767977714539,
      "step": 4677
    },
    {
      "epoch": 0.612153430931545,
      "grad_norm": 6.916402339935303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16254450380802155,
      "logits/rejected": 0.065424844622612,
      "logps/chosen": -553.8847045898438,
      "logps/rejected": -523.55810546875,
      "loss": 0.3663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4082881808280945,
      "rewards/margins": 1.364017128944397,
      "rewards/rejected": -1.7723052501678467,
      "step": 4678
    },
    {
      "epoch": 0.6122842888688967,
      "grad_norm": 5.730956554412842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009233228862285614,
      "logits/rejected": 0.21344348788261414,
      "logps/chosen": -742.9819946289062,
      "logps/rejected": -536.0925903320312,
      "loss": 0.2784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12425272166728973,
      "rewards/margins": 1.3378609418869019,
      "rewards/rejected": -1.462113618850708,
      "step": 4679
    },
    {
      "epoch": 0.6124151468062484,
      "grad_norm": 4.709086894989014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007634952664375305,
      "logits/rejected": -0.12352138012647629,
      "logps/chosen": -453.22174072265625,
      "logps/rejected": -402.90728759765625,
      "loss": 0.3505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4149666428565979,
      "rewards/margins": 1.1038520336151123,
      "rewards/rejected": -1.518818736076355,
      "step": 4680
    },
    {
      "epoch": 0.6125460047436002,
      "grad_norm": 13.398526191711426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06849813461303711,
      "logits/rejected": 7.940828800201416e-05,
      "logps/chosen": -475.511962890625,
      "logps/rejected": -500.84967041015625,
      "loss": 0.6655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3633651733398438,
      "rewards/margins": 0.19830265641212463,
      "rewards/rejected": -1.561667799949646,
      "step": 4681
    },
    {
      "epoch": 0.612676862680952,
      "grad_norm": 4.519617557525635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0420561321079731,
      "logits/rejected": 0.11600516736507416,
      "logps/chosen": -630.3970947265625,
      "logps/rejected": -549.7625732421875,
      "loss": 0.3083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1407650113105774,
      "rewards/margins": 1.478436827659607,
      "rewards/rejected": -1.619201898574829,
      "step": 4682
    },
    {
      "epoch": 0.6128077206183038,
      "grad_norm": 8.680000305175781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12111729383468628,
      "logits/rejected": -0.08346777409315109,
      "logps/chosen": -444.7604675292969,
      "logps/rejected": -382.6816711425781,
      "loss": 0.7603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.746110200881958,
      "rewards/margins": 0.26547038555145264,
      "rewards/rejected": -1.0115805864334106,
      "step": 4683
    },
    {
      "epoch": 0.6129385785556555,
      "grad_norm": 10.747125625610352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11626115441322327,
      "logits/rejected": -0.10558710992336273,
      "logps/chosen": -633.8104248046875,
      "logps/rejected": -386.763916015625,
      "loss": 0.8959,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1091006994247437,
      "rewards/margins": 0.09258578717708588,
      "rewards/rejected": -1.2016866207122803,
      "step": 4684
    },
    {
      "epoch": 0.6130694364930073,
      "grad_norm": 12.262307167053223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14036118984222412,
      "logits/rejected": 0.10146047174930573,
      "logps/chosen": -525.5971069335938,
      "logps/rejected": -567.709228515625,
      "loss": 1.3275,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.8196114301681519,
      "rewards/margins": -0.747728705406189,
      "rewards/rejected": -1.0718826055526733,
      "step": 4685
    },
    {
      "epoch": 0.6132002944303591,
      "grad_norm": 8.040122032165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06872968375682831,
      "logits/rejected": -0.030078500509262085,
      "logps/chosen": -369.556884765625,
      "logps/rejected": -375.4155578613281,
      "loss": 0.6775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3550424873828888,
      "rewards/margins": 0.5277482867240906,
      "rewards/rejected": -0.882790744304657,
      "step": 4686
    },
    {
      "epoch": 0.6133311523677109,
      "grad_norm": 11.407538414001465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2881027162075043,
      "logits/rejected": 0.13086532056331635,
      "logps/chosen": -539.3261108398438,
      "logps/rejected": -470.5506896972656,
      "loss": 0.9363,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7178274989128113,
      "rewards/margins": 0.1727951169013977,
      "rewards/rejected": -0.8906225562095642,
      "step": 4687
    },
    {
      "epoch": 0.6134620103050625,
      "grad_norm": 8.227001190185547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01371278241276741,
      "logits/rejected": -0.05785273760557175,
      "logps/chosen": -355.509521484375,
      "logps/rejected": -405.33660888671875,
      "loss": 0.7709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6706516146659851,
      "rewards/margins": 0.3552398383617401,
      "rewards/rejected": -1.0258915424346924,
      "step": 4688
    },
    {
      "epoch": 0.6135928682424143,
      "grad_norm": 4.610461711883545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03442943096160889,
      "logits/rejected": 0.13303327560424805,
      "logps/chosen": -301.1976318359375,
      "logps/rejected": -257.2062072753906,
      "loss": 0.3785,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.036855876445770264,
      "rewards/margins": 1.4100927114486694,
      "rewards/rejected": -1.446948528289795,
      "step": 4689
    },
    {
      "epoch": 0.6137237261797661,
      "grad_norm": 5.1341328620910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3461694121360779,
      "logits/rejected": -0.21493059396743774,
      "logps/chosen": -449.1805725097656,
      "logps/rejected": -367.0286865234375,
      "loss": 0.406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6536217331886292,
      "rewards/margins": 1.0163365602493286,
      "rewards/rejected": -1.6699583530426025,
      "step": 4690
    },
    {
      "epoch": 0.6138545841171179,
      "grad_norm": 5.509288787841797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13194206357002258,
      "logits/rejected": -0.09128191322088242,
      "logps/chosen": -422.4557800292969,
      "logps/rejected": -305.0088195800781,
      "loss": 0.3689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6793584823608398,
      "rewards/margins": 1.4108765125274658,
      "rewards/rejected": -2.0902349948883057,
      "step": 4691
    },
    {
      "epoch": 0.6139854420544696,
      "grad_norm": 12.130614280700684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029296867549419403,
      "logits/rejected": 0.1680280566215515,
      "logps/chosen": -427.1210021972656,
      "logps/rejected": -416.5248107910156,
      "loss": 0.7948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7984235882759094,
      "rewards/margins": 0.8467094302177429,
      "rewards/rejected": -1.645133137702942,
      "step": 4692
    },
    {
      "epoch": 0.6141162999918214,
      "grad_norm": 10.303662300109863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14406001567840576,
      "logits/rejected": -0.009179389104247093,
      "logps/chosen": -440.8164367675781,
      "logps/rejected": -293.0111389160156,
      "loss": 0.793,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.907410740852356,
      "rewards/margins": 0.019497528672218323,
      "rewards/rejected": -1.9269084930419922,
      "step": 4693
    },
    {
      "epoch": 0.6142471579291732,
      "grad_norm": 4.68497371673584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22536082565784454,
      "logits/rejected": 0.11459983885288239,
      "logps/chosen": -463.420166015625,
      "logps/rejected": -384.30096435546875,
      "loss": 0.4321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1935056746006012,
      "rewards/margins": 1.6880379915237427,
      "rewards/rejected": -1.881543755531311,
      "step": 4694
    },
    {
      "epoch": 0.6143780158665249,
      "grad_norm": 8.936493873596191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09041070938110352,
      "logits/rejected": 2.5715678930282593e-05,
      "logps/chosen": -354.48583984375,
      "logps/rejected": -349.0423889160156,
      "loss": 0.7588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.035448431968689,
      "rewards/margins": 0.050877392292022705,
      "rewards/rejected": -1.086325764656067,
      "step": 4695
    },
    {
      "epoch": 0.6145088738038766,
      "grad_norm": 6.851578235626221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03541877865791321,
      "logits/rejected": -0.15432016551494598,
      "logps/chosen": -621.2783813476562,
      "logps/rejected": -650.2462768554688,
      "loss": 0.5492,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.484347939491272,
      "rewards/margins": 1.1666494607925415,
      "rewards/rejected": -1.6509974002838135,
      "step": 4696
    },
    {
      "epoch": 0.6146397317412284,
      "grad_norm": 11.774043083190918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13609227538108826,
      "logits/rejected": -0.04838884621858597,
      "logps/chosen": -410.03759765625,
      "logps/rejected": -383.4625549316406,
      "loss": 0.8705,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2892273664474487,
      "rewards/margins": -0.07604865729808807,
      "rewards/rejected": -1.2131787538528442,
      "step": 4697
    },
    {
      "epoch": 0.6147705896785802,
      "grad_norm": 4.902614116668701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11427301913499832,
      "logits/rejected": -0.10441349446773529,
      "logps/chosen": -550.4088134765625,
      "logps/rejected": -470.4931640625,
      "loss": 0.3201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.046792030334472656,
      "rewards/margins": 1.6950348615646362,
      "rewards/rejected": -1.7418268918991089,
      "step": 4698
    },
    {
      "epoch": 0.6149014476159319,
      "grad_norm": 7.767417907714844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.370225727558136,
      "logits/rejected": 0.2719815969467163,
      "logps/chosen": -505.58941650390625,
      "logps/rejected": -478.5167236328125,
      "loss": 0.617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7501749396324158,
      "rewards/margins": 0.4057818055152893,
      "rewards/rejected": -1.155956745147705,
      "step": 4699
    },
    {
      "epoch": 0.6150323055532837,
      "grad_norm": 12.581440925598145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21175721287727356,
      "logits/rejected": -0.07255864143371582,
      "logps/chosen": -447.0425109863281,
      "logps/rejected": -410.5690612792969,
      "loss": 0.7385,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8558538556098938,
      "rewards/margins": 0.21011850237846375,
      "rewards/rejected": -1.0659723281860352,
      "step": 4700
    },
    {
      "epoch": 0.6151631634906355,
      "grad_norm": 6.277819633483887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01628248021006584,
      "logits/rejected": -0.02631686069071293,
      "logps/chosen": -395.3115234375,
      "logps/rejected": -398.7530517578125,
      "loss": 0.4921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.479723185300827,
      "rewards/margins": 0.9748401045799255,
      "rewards/rejected": -1.4545631408691406,
      "step": 4701
    },
    {
      "epoch": 0.6152940214279873,
      "grad_norm": 8.015738487243652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14245450496673584,
      "logits/rejected": -0.2883550524711609,
      "logps/chosen": -481.55108642578125,
      "logps/rejected": -486.3849182128906,
      "loss": 0.5683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5281559228897095,
      "rewards/margins": 1.1934382915496826,
      "rewards/rejected": -1.721594214439392,
      "step": 4702
    },
    {
      "epoch": 0.615424879365339,
      "grad_norm": 5.215970039367676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03739205375313759,
      "logits/rejected": -0.32542935013771057,
      "logps/chosen": -456.5338439941406,
      "logps/rejected": -435.6313781738281,
      "loss": 0.3221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25783634185791016,
      "rewards/margins": 4.050249099731445,
      "rewards/rejected": -4.308085918426514,
      "step": 4703
    },
    {
      "epoch": 0.6155557373026908,
      "grad_norm": 8.610798835754395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08038149774074554,
      "logits/rejected": -0.20359759032726288,
      "logps/chosen": -295.7020263671875,
      "logps/rejected": -431.4430236816406,
      "loss": 0.651,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47814369201660156,
      "rewards/margins": 0.7181503176689148,
      "rewards/rejected": -1.1962940692901611,
      "step": 4704
    },
    {
      "epoch": 0.6156865952400425,
      "grad_norm": 10.606505393981934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1146814376115799,
      "logits/rejected": -0.12889941036701202,
      "logps/chosen": -504.2857666015625,
      "logps/rejected": -429.9137268066406,
      "loss": 0.6789,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8705033659934998,
      "rewards/margins": 0.5097607374191284,
      "rewards/rejected": -1.380264163017273,
      "step": 4705
    },
    {
      "epoch": 0.6158174531773943,
      "grad_norm": 9.711203575134277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06336869299411774,
      "logits/rejected": 0.016171611845493317,
      "logps/chosen": -350.778564453125,
      "logps/rejected": -459.078125,
      "loss": 0.7768,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6908098459243774,
      "rewards/margins": 0.0076585859060287476,
      "rewards/rejected": -0.6984683871269226,
      "step": 4706
    },
    {
      "epoch": 0.615948311114746,
      "grad_norm": 5.3967437744140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12038247287273407,
      "logits/rejected": 0.025980839505791664,
      "logps/chosen": -436.2904052734375,
      "logps/rejected": -357.5180969238281,
      "loss": 0.3625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15310250222682953,
      "rewards/margins": 1.0855474472045898,
      "rewards/rejected": -0.9324449896812439,
      "step": 4707
    },
    {
      "epoch": 0.6160791690520978,
      "grad_norm": 4.6979804039001465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30027079582214355,
      "logits/rejected": -0.21141968667507172,
      "logps/chosen": -414.8363952636719,
      "logps/rejected": -375.55267333984375,
      "loss": 0.2314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3049997091293335,
      "rewards/margins": 1.6126132011413574,
      "rewards/rejected": -1.9176130294799805,
      "step": 4708
    },
    {
      "epoch": 0.6162100269894496,
      "grad_norm": 7.190274238586426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010263096541166306,
      "logits/rejected": 0.1283586621284485,
      "logps/chosen": -620.0162963867188,
      "logps/rejected": -515.9744873046875,
      "loss": 0.4767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.403913676738739,
      "rewards/margins": 0.757025420665741,
      "rewards/rejected": -1.16093909740448,
      "step": 4709
    },
    {
      "epoch": 0.6163408849268014,
      "grad_norm": 7.007535934448242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07520260661840439,
      "logits/rejected": 0.012971462681889534,
      "logps/chosen": -403.3134460449219,
      "logps/rejected": -421.4552001953125,
      "loss": 0.4462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11861088126897812,
      "rewards/margins": 1.4954016208648682,
      "rewards/rejected": -1.614012360572815,
      "step": 4710
    },
    {
      "epoch": 0.6164717428641531,
      "grad_norm": 10.630741119384766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16786666214466095,
      "logits/rejected": -0.2455834448337555,
      "logps/chosen": -389.6685485839844,
      "logps/rejected": -398.5084228515625,
      "loss": 0.7151,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6766317486763,
      "rewards/margins": 0.5058348178863525,
      "rewards/rejected": -1.1824666261672974,
      "step": 4711
    },
    {
      "epoch": 0.6166026008015049,
      "grad_norm": 12.348156929016113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1300184428691864,
      "logits/rejected": -0.18222975730895996,
      "logps/chosen": -510.0784606933594,
      "logps/rejected": -788.7498779296875,
      "loss": 0.8696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1402788162231445,
      "rewards/margins": 0.352717787027359,
      "rewards/rejected": -1.4929966926574707,
      "step": 4712
    },
    {
      "epoch": 0.6167334587388567,
      "grad_norm": 7.659974575042725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0805431678891182,
      "logits/rejected": 0.26989853382110596,
      "logps/chosen": -367.11083984375,
      "logps/rejected": -442.78125,
      "loss": 0.5832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4510665833950043,
      "rewards/margins": 0.5429643988609314,
      "rewards/rejected": -0.9940309524536133,
      "step": 4713
    },
    {
      "epoch": 0.6168643166762084,
      "grad_norm": 10.666611671447754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055256687104701996,
      "logits/rejected": 0.040680862963199615,
      "logps/chosen": -472.9507751464844,
      "logps/rejected": -487.6177978515625,
      "loss": 0.5289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.895012617111206,
      "rewards/margins": 0.7053794860839844,
      "rewards/rejected": -1.6003921031951904,
      "step": 4714
    },
    {
      "epoch": 0.6169951746135601,
      "grad_norm": 7.165259838104248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07662799954414368,
      "logits/rejected": 0.11807546764612198,
      "logps/chosen": -573.7841186523438,
      "logps/rejected": -437.35137939453125,
      "loss": 0.4666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.037782490253448486,
      "rewards/margins": 1.2790169715881348,
      "rewards/rejected": -1.2412344217300415,
      "step": 4715
    },
    {
      "epoch": 0.6171260325509119,
      "grad_norm": 3.435481071472168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09050557017326355,
      "logits/rejected": 0.031713906675577164,
      "logps/chosen": -418.3607177734375,
      "logps/rejected": -404.140625,
      "loss": 0.2363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4256545901298523,
      "rewards/margins": 1.9787784814834595,
      "rewards/rejected": -2.404433012008667,
      "step": 4716
    },
    {
      "epoch": 0.6172568904882637,
      "grad_norm": 8.43850040435791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05592350661754608,
      "logits/rejected": -0.05304727330803871,
      "logps/chosen": -624.7793579101562,
      "logps/rejected": -599.4103393554688,
      "loss": 0.6,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5727983117103577,
      "rewards/margins": 0.5063335299491882,
      "rewards/rejected": -1.0791319608688354,
      "step": 4717
    },
    {
      "epoch": 0.6173877484256154,
      "grad_norm": 7.040582656860352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01594962179660797,
      "logits/rejected": 0.28001487255096436,
      "logps/chosen": -399.8033447265625,
      "logps/rejected": -562.6506958007812,
      "loss": 0.392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12263565510511398,
      "rewards/margins": 0.9002365469932556,
      "rewards/rejected": -1.0228722095489502,
      "step": 4718
    },
    {
      "epoch": 0.6175186063629672,
      "grad_norm": 7.134341239929199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10717585682868958,
      "logits/rejected": -0.33041664958000183,
      "logps/chosen": -368.4893798828125,
      "logps/rejected": -401.27850341796875,
      "loss": 0.5183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.195085346698761,
      "rewards/margins": 0.7944374084472656,
      "rewards/rejected": -0.9895226955413818,
      "step": 4719
    },
    {
      "epoch": 0.617649464300319,
      "grad_norm": 9.885876655578613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09725663810968399,
      "logits/rejected": -0.3006415367126465,
      "logps/chosen": -394.3551330566406,
      "logps/rejected": -396.7516174316406,
      "loss": 0.6502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1315521001815796,
      "rewards/margins": 0.5081303119659424,
      "rewards/rejected": -1.639682412147522,
      "step": 4720
    },
    {
      "epoch": 0.6177803222376708,
      "grad_norm": 6.602859020233154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010292816907167435,
      "logits/rejected": -0.06586329638957977,
      "logps/chosen": -562.4797973632812,
      "logps/rejected": -457.9515686035156,
      "loss": 0.406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4695800542831421,
      "rewards/margins": 0.8475291728973389,
      "rewards/rejected": -1.3171093463897705,
      "step": 4721
    },
    {
      "epoch": 0.6179111801750224,
      "grad_norm": 11.866934776306152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22564826905727386,
      "logits/rejected": 0.17906101047992706,
      "logps/chosen": -535.6500244140625,
      "logps/rejected": -397.3934326171875,
      "loss": 1.0143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2021633386611938,
      "rewards/margins": -0.04604493826627731,
      "rewards/rejected": -1.1561182737350464,
      "step": 4722
    },
    {
      "epoch": 0.6180420381123742,
      "grad_norm": 5.035234451293945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19072967767715454,
      "logits/rejected": -0.071710005402565,
      "logps/chosen": -387.89556884765625,
      "logps/rejected": -703.858642578125,
      "loss": 0.2939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5097822546958923,
      "rewards/margins": 1.5333220958709717,
      "rewards/rejected": -2.0431041717529297,
      "step": 4723
    },
    {
      "epoch": 0.618172896049726,
      "grad_norm": 6.031998634338379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12723994255065918,
      "logits/rejected": -0.22034922242164612,
      "logps/chosen": -262.60760498046875,
      "logps/rejected": -338.88848876953125,
      "loss": 0.3738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.723828911781311,
      "rewards/margins": 1.014053463935852,
      "rewards/rejected": -1.737882375717163,
      "step": 4724
    },
    {
      "epoch": 0.6183037539870778,
      "grad_norm": 6.162853717803955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08599220216274261,
      "logits/rejected": -0.19040712714195251,
      "logps/chosen": -396.20245361328125,
      "logps/rejected": -444.90020751953125,
      "loss": 0.3381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40535739064216614,
      "rewards/margins": 1.607771396636963,
      "rewards/rejected": -2.0131287574768066,
      "step": 4725
    },
    {
      "epoch": 0.6184346119244295,
      "grad_norm": 3.8697309494018555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15448099374771118,
      "logits/rejected": -0.09629320353269577,
      "logps/chosen": -464.20574951171875,
      "logps/rejected": -388.60260009765625,
      "loss": 0.2719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33129042387008667,
      "rewards/margins": 1.5981382131576538,
      "rewards/rejected": -1.9294285774230957,
      "step": 4726
    },
    {
      "epoch": 0.6185654698617813,
      "grad_norm": 7.316739559173584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03670773655176163,
      "logits/rejected": 0.07006891071796417,
      "logps/chosen": -532.5256958007812,
      "logps/rejected": -381.6253662109375,
      "loss": 0.5174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07084540277719498,
      "rewards/margins": 1.0422035455703735,
      "rewards/rejected": -1.113049030303955,
      "step": 4727
    },
    {
      "epoch": 0.6186963277991331,
      "grad_norm": 5.044277667999268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07156780362129211,
      "logits/rejected": 0.14525626599788666,
      "logps/chosen": -467.38836669921875,
      "logps/rejected": -460.1068420410156,
      "loss": 0.3229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0782129243016243,
      "rewards/margins": 1.1614820957183838,
      "rewards/rejected": -1.2396949529647827,
      "step": 4728
    },
    {
      "epoch": 0.6188271857364849,
      "grad_norm": 2.550225019454956,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19893446564674377,
      "logits/rejected": 0.08859853446483612,
      "logps/chosen": -430.4469909667969,
      "logps/rejected": -417.3701171875,
      "loss": 0.1957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5916951894760132,
      "rewards/margins": 2.1382129192352295,
      "rewards/rejected": -2.7299082279205322,
      "step": 4729
    },
    {
      "epoch": 0.6189580436738366,
      "grad_norm": 4.855984687805176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19687139987945557,
      "logits/rejected": 0.2373466193675995,
      "logps/chosen": -394.7586975097656,
      "logps/rejected": -371.596435546875,
      "loss": 0.3504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40338099002838135,
      "rewards/margins": 1.2548010349273682,
      "rewards/rejected": -1.65818190574646,
      "step": 4730
    },
    {
      "epoch": 0.6190889016111883,
      "grad_norm": 7.730186939239502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1173952966928482,
      "logits/rejected": 0.049261197447776794,
      "logps/chosen": -406.7318115234375,
      "logps/rejected": -389.16845703125,
      "loss": 0.6163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25707948207855225,
      "rewards/margins": 0.7524642944335938,
      "rewards/rejected": -1.009543776512146,
      "step": 4731
    },
    {
      "epoch": 0.6192197595485401,
      "grad_norm": 5.264451503753662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2948993444442749,
      "logits/rejected": 0.0352582186460495,
      "logps/chosen": -440.2301940917969,
      "logps/rejected": -590.2749633789062,
      "loss": 0.3587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21981754899024963,
      "rewards/margins": 1.6692233085632324,
      "rewards/rejected": -1.8890410661697388,
      "step": 4732
    },
    {
      "epoch": 0.6193506174858919,
      "grad_norm": 5.3501410484313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04914054274559021,
      "logits/rejected": -0.05628131330013275,
      "logps/chosen": -588.0155639648438,
      "logps/rejected": -351.80816650390625,
      "loss": 0.2877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12779387831687927,
      "rewards/margins": 1.530801773071289,
      "rewards/rejected": -1.4030078649520874,
      "step": 4733
    },
    {
      "epoch": 0.6194814754232436,
      "grad_norm": 3.7545812129974365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21664586663246155,
      "logits/rejected": -0.30855852365493774,
      "logps/chosen": -469.5728759765625,
      "logps/rejected": -373.4888000488281,
      "loss": 0.1913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0426416397094727,
      "rewards/margins": 1.8686670064926147,
      "rewards/rejected": -2.911308765411377,
      "step": 4734
    },
    {
      "epoch": 0.6196123333605954,
      "grad_norm": 6.955250263214111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01102931797504425,
      "logits/rejected": 0.04196477308869362,
      "logps/chosen": -602.3931274414062,
      "logps/rejected": -520.42041015625,
      "loss": 0.4685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6063641309738159,
      "rewards/margins": 1.5001837015151978,
      "rewards/rejected": -2.1065478324890137,
      "step": 4735
    },
    {
      "epoch": 0.6197431912979472,
      "grad_norm": 7.008327007293701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35861289501190186,
      "logits/rejected": 0.07378898561000824,
      "logps/chosen": -349.7712707519531,
      "logps/rejected": -318.4499206542969,
      "loss": 0.6062,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8200031518936157,
      "rewards/margins": 0.49437206983566284,
      "rewards/rejected": -1.3143752813339233,
      "step": 4736
    },
    {
      "epoch": 0.6198740492352989,
      "grad_norm": 4.736485481262207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3652093708515167,
      "logits/rejected": 0.43361276388168335,
      "logps/chosen": -532.2525634765625,
      "logps/rejected": -512.369873046875,
      "loss": 0.2946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2736484408378601,
      "rewards/margins": 1.8521370887756348,
      "rewards/rejected": -2.1257853507995605,
      "step": 4737
    },
    {
      "epoch": 0.6200049071726507,
      "grad_norm": 8.425642967224121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1047922745347023,
      "logits/rejected": 0.08168946206569672,
      "logps/chosen": -450.05914306640625,
      "logps/rejected": -374.434814453125,
      "loss": 0.61,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.547463059425354,
      "rewards/margins": 1.008928894996643,
      "rewards/rejected": -1.556391954421997,
      "step": 4738
    },
    {
      "epoch": 0.6201357651100025,
      "grad_norm": 6.480363845825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009551584720611572,
      "logits/rejected": -0.22179315984249115,
      "logps/chosen": -739.7186279296875,
      "logps/rejected": -616.06689453125,
      "loss": 0.4539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03555753827095032,
      "rewards/margins": 0.6527200937271118,
      "rewards/rejected": -0.6171625256538391,
      "step": 4739
    },
    {
      "epoch": 0.6202666230473542,
      "grad_norm": 8.372903823852539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05720282346010208,
      "logits/rejected": -0.08536537736654282,
      "logps/chosen": -340.86822509765625,
      "logps/rejected": -300.02581787109375,
      "loss": 0.5557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1040946245193481,
      "rewards/margins": 0.5036083459854126,
      "rewards/rejected": -1.6077029705047607,
      "step": 4740
    },
    {
      "epoch": 0.6203974809847059,
      "grad_norm": 7.560397148132324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06427495926618576,
      "logits/rejected": -0.22665460407733917,
      "logps/chosen": -597.302490234375,
      "logps/rejected": -420.02056884765625,
      "loss": 0.5793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5208653211593628,
      "rewards/margins": 0.8633347153663635,
      "rewards/rejected": -1.3841999769210815,
      "step": 4741
    },
    {
      "epoch": 0.6205283389220577,
      "grad_norm": 4.941709518432617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.064811110496521,
      "logits/rejected": 0.017126042395830154,
      "logps/chosen": -448.05224609375,
      "logps/rejected": -494.79058837890625,
      "loss": 0.3636,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35361766815185547,
      "rewards/margins": 1.2897030115127563,
      "rewards/rejected": -1.6433207988739014,
      "step": 4742
    },
    {
      "epoch": 0.6206591968594095,
      "grad_norm": 6.939019203186035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16057471930980682,
      "logits/rejected": 0.01504179835319519,
      "logps/chosen": -548.6775512695312,
      "logps/rejected": -427.5421447753906,
      "loss": 0.5861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7669076919555664,
      "rewards/margins": 0.6605401635169983,
      "rewards/rejected": -1.4274479150772095,
      "step": 4743
    },
    {
      "epoch": 0.6207900547967613,
      "grad_norm": 6.812914848327637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22312620282173157,
      "logits/rejected": 0.20514404773712158,
      "logps/chosen": -367.2479553222656,
      "logps/rejected": -302.6647644042969,
      "loss": 0.6392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6416860222816467,
      "rewards/margins": 0.29466745257377625,
      "rewards/rejected": -0.9363535642623901,
      "step": 4744
    },
    {
      "epoch": 0.620920912734113,
      "grad_norm": 9.245728492736816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37345027923583984,
      "logits/rejected": 0.20491060614585876,
      "logps/chosen": -490.1856689453125,
      "logps/rejected": -598.7594604492188,
      "loss": 0.7015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8010936975479126,
      "rewards/margins": 0.3473171591758728,
      "rewards/rejected": -1.1484107971191406,
      "step": 4745
    },
    {
      "epoch": 0.6210517706714648,
      "grad_norm": 5.0579423904418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21849000453948975,
      "logits/rejected": 0.041742123663425446,
      "logps/chosen": -592.2782592773438,
      "logps/rejected": -450.0968017578125,
      "loss": 0.268,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28139516711235046,
      "rewards/margins": 2.151226043701172,
      "rewards/rejected": -2.4326210021972656,
      "step": 4746
    },
    {
      "epoch": 0.6211826286088166,
      "grad_norm": 7.081427574157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12927493453025818,
      "logits/rejected": -0.26473790407180786,
      "logps/chosen": -544.6171264648438,
      "logps/rejected": -484.29693603515625,
      "loss": 0.5119,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21723249554634094,
      "rewards/margins": 1.1746867895126343,
      "rewards/rejected": -1.3919193744659424,
      "step": 4747
    },
    {
      "epoch": 0.6213134865461684,
      "grad_norm": 7.755185127258301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.355307400226593,
      "logits/rejected": 0.20639313757419586,
      "logps/chosen": -467.73236083984375,
      "logps/rejected": -484.7965087890625,
      "loss": 0.6952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0160242319107056,
      "rewards/margins": 1.5218867063522339,
      "rewards/rejected": -2.5379109382629395,
      "step": 4748
    },
    {
      "epoch": 0.62144434448352,
      "grad_norm": 6.749966621398926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0643279105424881,
      "logits/rejected": 0.07269908487796783,
      "logps/chosen": -636.8797607421875,
      "logps/rejected": -668.390625,
      "loss": 0.4901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6466661691665649,
      "rewards/margins": 0.9890329241752625,
      "rewards/rejected": -1.6356991529464722,
      "step": 4749
    },
    {
      "epoch": 0.6215752024208718,
      "grad_norm": 8.840007781982422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008431617170572281,
      "logits/rejected": 0.1596420854330063,
      "logps/chosen": -479.5162048339844,
      "logps/rejected": -458.36749267578125,
      "loss": 0.6214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6630274057388306,
      "rewards/margins": 0.6128665804862976,
      "rewards/rejected": -1.2758939266204834,
      "step": 4750
    },
    {
      "epoch": 0.6217060603582236,
      "grad_norm": 5.928753852844238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14624693989753723,
      "logits/rejected": 0.002710409229621291,
      "logps/chosen": -411.5892639160156,
      "logps/rejected": -427.10833740234375,
      "loss": 0.4781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43088969588279724,
      "rewards/margins": 0.7424149513244629,
      "rewards/rejected": -1.1733046770095825,
      "step": 4751
    },
    {
      "epoch": 0.6218369182955754,
      "grad_norm": 6.730956554412842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08622557669878006,
      "logits/rejected": -0.07938361912965775,
      "logps/chosen": -440.34014892578125,
      "logps/rejected": -451.1766052246094,
      "loss": 0.5433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3819066882133484,
      "rewards/margins": 0.8643977642059326,
      "rewards/rejected": -1.2463045120239258,
      "step": 4752
    },
    {
      "epoch": 0.6219677762329271,
      "grad_norm": 8.557751655578613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0008768253028392792,
      "logits/rejected": -0.005162011831998825,
      "logps/chosen": -341.3774719238281,
      "logps/rejected": -360.3943176269531,
      "loss": 0.9593,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8583057522773743,
      "rewards/margins": -0.14293235540390015,
      "rewards/rejected": -0.7153734564781189,
      "step": 4753
    },
    {
      "epoch": 0.6220986341702789,
      "grad_norm": 6.2018303871154785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12371628731489182,
      "logits/rejected": -0.08456532657146454,
      "logps/chosen": -351.75579833984375,
      "logps/rejected": -301.7154235839844,
      "loss": 0.5142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.38754522800445557,
      "rewards/margins": 1.1510722637176514,
      "rewards/rejected": -0.7635271549224854,
      "step": 4754
    },
    {
      "epoch": 0.6222294921076307,
      "grad_norm": 9.184993743896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4667043089866638,
      "logits/rejected": -0.2557082772254944,
      "logps/chosen": -497.84197998046875,
      "logps/rejected": -377.4283752441406,
      "loss": 0.5409,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3656597137451172,
      "rewards/margins": 0.8764693140983582,
      "rewards/rejected": -1.2421289682388306,
      "step": 4755
    },
    {
      "epoch": 0.6223603500449825,
      "grad_norm": 4.107783317565918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2887856960296631,
      "logits/rejected": 0.2373472899198532,
      "logps/chosen": -349.84185791015625,
      "logps/rejected": -423.6682434082031,
      "loss": 0.4174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.027798641473054886,
      "rewards/margins": 1.7619510889053345,
      "rewards/rejected": -1.7341521978378296,
      "step": 4756
    },
    {
      "epoch": 0.6224912079823341,
      "grad_norm": 6.6537652015686035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11742173135280609,
      "logits/rejected": 0.2000027447938919,
      "logps/chosen": -551.5221557617188,
      "logps/rejected": -475.46319580078125,
      "loss": 0.4691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5841034054756165,
      "rewards/margins": 0.8494004011154175,
      "rewards/rejected": -0.26529693603515625,
      "step": 4757
    },
    {
      "epoch": 0.6226220659196859,
      "grad_norm": 9.524077415466309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20297029614448547,
      "logits/rejected": -0.29174333810806274,
      "logps/chosen": -471.905029296875,
      "logps/rejected": -382.8957214355469,
      "loss": 0.7081,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7346338033676147,
      "rewards/margins": 0.22717857360839844,
      "rewards/rejected": -0.961812436580658,
      "step": 4758
    },
    {
      "epoch": 0.6227529238570377,
      "grad_norm": 6.695625305175781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016754265874624252,
      "logits/rejected": -0.09632710367441177,
      "logps/chosen": -407.16290283203125,
      "logps/rejected": -457.19659423828125,
      "loss": 0.5145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48729443550109863,
      "rewards/margins": 1.0929794311523438,
      "rewards/rejected": -1.5802738666534424,
      "step": 4759
    },
    {
      "epoch": 0.6228837817943894,
      "grad_norm": 6.051212310791016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012830201536417007,
      "logits/rejected": -0.01612129621207714,
      "logps/chosen": -404.8427429199219,
      "logps/rejected": -337.537841796875,
      "loss": 0.4826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3287695050239563,
      "rewards/margins": 0.7884852886199951,
      "rewards/rejected": -1.1172548532485962,
      "step": 4760
    },
    {
      "epoch": 0.6230146397317412,
      "grad_norm": 5.53188943862915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1924322545528412,
      "logits/rejected": 0.15653829276561737,
      "logps/chosen": -656.0073852539062,
      "logps/rejected": -533.3287353515625,
      "loss": 0.3646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12504006922245026,
      "rewards/margins": 0.9701993465423584,
      "rewards/rejected": -0.8451594114303589,
      "step": 4761
    },
    {
      "epoch": 0.623145497669093,
      "grad_norm": 8.571969985961914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18136760592460632,
      "logits/rejected": -0.037811603397130966,
      "logps/chosen": -423.6053161621094,
      "logps/rejected": -354.6783142089844,
      "loss": 0.4818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.014728724956512451,
      "rewards/margins": 1.6331802606582642,
      "rewards/rejected": -1.6479089260101318,
      "step": 4762
    },
    {
      "epoch": 0.6232763556064448,
      "grad_norm": 7.116455078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04689183086156845,
      "logits/rejected": 0.07538226246833801,
      "logps/chosen": -291.479736328125,
      "logps/rejected": -344.796630859375,
      "loss": 0.4459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.753223180770874,
      "rewards/margins": 1.2046103477478027,
      "rewards/rejected": -1.9578335285186768,
      "step": 4763
    },
    {
      "epoch": 0.6234072135437965,
      "grad_norm": 7.260962009429932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07596427947282791,
      "logits/rejected": -0.0511045902967453,
      "logps/chosen": -449.15875244140625,
      "logps/rejected": -474.69366455078125,
      "loss": 0.5104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3832803964614868,
      "rewards/margins": 1.0222806930541992,
      "rewards/rejected": -1.4055612087249756,
      "step": 4764
    },
    {
      "epoch": 0.6235380714811483,
      "grad_norm": 9.798130989074707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3016788065433502,
      "logits/rejected": 0.4923792779445648,
      "logps/chosen": -344.69830322265625,
      "logps/rejected": -419.8302307128906,
      "loss": 0.9018,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.6357802152633667,
      "rewards/margins": -0.3045421838760376,
      "rewards/rejected": -1.3312379121780396,
      "step": 4765
    },
    {
      "epoch": 0.6236689294185,
      "grad_norm": 10.979662895202637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012310434132814407,
      "logits/rejected": -0.02986060455441475,
      "logps/chosen": -429.4971923828125,
      "logps/rejected": -450.9598388671875,
      "loss": 0.7929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15017986297607422,
      "rewards/margins": 0.5455362796783447,
      "rewards/rejected": -0.6957160830497742,
      "step": 4766
    },
    {
      "epoch": 0.6237997873558518,
      "grad_norm": 4.966725826263428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13803642988204956,
      "logits/rejected": 0.1485704630613327,
      "logps/chosen": -377.7193603515625,
      "logps/rejected": -388.6884460449219,
      "loss": 0.3752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47033578157424927,
      "rewards/margins": 1.218902826309204,
      "rewards/rejected": -0.7485670447349548,
      "step": 4767
    },
    {
      "epoch": 0.6239306452932035,
      "grad_norm": 8.014063835144043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1775514781475067,
      "logits/rejected": 0.2870499789714813,
      "logps/chosen": -413.349365234375,
      "logps/rejected": -435.90582275390625,
      "loss": 0.7304,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11446592956781387,
      "rewards/margins": 0.32439231872558594,
      "rewards/rejected": -0.4388582408428192,
      "step": 4768
    },
    {
      "epoch": 0.6240615032305553,
      "grad_norm": 6.9641313552856445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020636625587940216,
      "logits/rejected": -0.09376796334981918,
      "logps/chosen": -393.0367736816406,
      "logps/rejected": -415.27752685546875,
      "loss": 0.505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2699471712112427,
      "rewards/margins": 0.7176544666290283,
      "rewards/rejected": -1.987601637840271,
      "step": 4769
    },
    {
      "epoch": 0.6241923611679071,
      "grad_norm": 7.412303447723389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11464472115039825,
      "logits/rejected": -0.06582780182361603,
      "logps/chosen": -319.2964172363281,
      "logps/rejected": -621.7900390625,
      "loss": 0.4594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8271224498748779,
      "rewards/margins": 1.0971870422363281,
      "rewards/rejected": -1.9243097305297852,
      "step": 4770
    },
    {
      "epoch": 0.6243232191052589,
      "grad_norm": 6.864755630493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13525418937206268,
      "logits/rejected": 0.07124636322259903,
      "logps/chosen": -322.6278991699219,
      "logps/rejected": -336.91583251953125,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7006538510322571,
      "rewards/margins": 0.2774704098701477,
      "rewards/rejected": -0.9781242609024048,
      "step": 4771
    },
    {
      "epoch": 0.6244540770426106,
      "grad_norm": 10.189728736877441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4283454418182373,
      "logits/rejected": -0.27333828806877136,
      "logps/chosen": -600.812744140625,
      "logps/rejected": -601.4192504882812,
      "loss": 0.5116,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04456157982349396,
      "rewards/margins": 0.8189363479614258,
      "rewards/rejected": -0.8634979128837585,
      "step": 4772
    },
    {
      "epoch": 0.6245849349799624,
      "grad_norm": 7.508368968963623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03833357244729996,
      "logits/rejected": -0.04072261229157448,
      "logps/chosen": -411.6989440917969,
      "logps/rejected": -429.1108093261719,
      "loss": 0.5359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.006943941116333,
      "rewards/margins": 0.6252084970474243,
      "rewards/rejected": -1.6321524381637573,
      "step": 4773
    },
    {
      "epoch": 0.6247157929173142,
      "grad_norm": 10.319830894470215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047126561403274536,
      "logits/rejected": 0.11823558807373047,
      "logps/chosen": -664.1673583984375,
      "logps/rejected": -389.74993896484375,
      "loss": 0.7213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5699408054351807,
      "rewards/margins": 0.7047470211982727,
      "rewards/rejected": -2.2746880054473877,
      "step": 4774
    },
    {
      "epoch": 0.624846650854666,
      "grad_norm": 5.387734889984131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18526189029216766,
      "logits/rejected": 0.2246030867099762,
      "logps/chosen": -375.58160400390625,
      "logps/rejected": -334.1199645996094,
      "loss": 0.4763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0518276691436768,
      "rewards/margins": 0.7135043144226074,
      "rewards/rejected": -1.7653318643569946,
      "step": 4775
    },
    {
      "epoch": 0.6249775087920176,
      "grad_norm": 14.279332160949707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23816299438476562,
      "logits/rejected": 0.06452105939388275,
      "logps/chosen": -502.6705627441406,
      "logps/rejected": -401.3678283691406,
      "loss": 1.2889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3505442142486572,
      "rewards/margins": -0.5604573488235474,
      "rewards/rejected": -0.7900868058204651,
      "step": 4776
    },
    {
      "epoch": 0.6251083667293694,
      "grad_norm": 8.045340538024902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12477270513772964,
      "logits/rejected": -0.22302718460559845,
      "logps/chosen": -356.8468933105469,
      "logps/rejected": -590.5982666015625,
      "loss": 0.5138,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9092108011245728,
      "rewards/margins": 0.8203754425048828,
      "rewards/rejected": -1.7295862436294556,
      "step": 4777
    },
    {
      "epoch": 0.6252392246667212,
      "grad_norm": 5.872868061065674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1114812046289444,
      "logits/rejected": 0.043502554297447205,
      "logps/chosen": -717.8692626953125,
      "logps/rejected": -698.49609375,
      "loss": 0.4009,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11542091518640518,
      "rewards/margins": 0.9151429533958435,
      "rewards/rejected": -1.0305638313293457,
      "step": 4778
    },
    {
      "epoch": 0.625370082604073,
      "grad_norm": 4.306975841522217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11504042148590088,
      "logits/rejected": -0.2804704010486603,
      "logps/chosen": -455.6640930175781,
      "logps/rejected": -319.3800964355469,
      "loss": 0.2452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15132257342338562,
      "rewards/margins": 1.8811402320861816,
      "rewards/rejected": -1.7298176288604736,
      "step": 4779
    },
    {
      "epoch": 0.6255009405414247,
      "grad_norm": 5.697368621826172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21312758326530457,
      "logits/rejected": -0.19550181925296783,
      "logps/chosen": -416.4119873046875,
      "logps/rejected": -385.220703125,
      "loss": 0.3515,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.686461865901947,
      "rewards/margins": 0.9481728076934814,
      "rewards/rejected": -1.6346346139907837,
      "step": 4780
    },
    {
      "epoch": 0.6256317984787765,
      "grad_norm": 13.203169822692871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3570321798324585,
      "logits/rejected": -0.3574941158294678,
      "logps/chosen": -367.18804931640625,
      "logps/rejected": -549.8426513671875,
      "loss": 0.7758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.308674931526184,
      "rewards/margins": 0.5739670991897583,
      "rewards/rejected": -1.8826417922973633,
      "step": 4781
    },
    {
      "epoch": 0.6257626564161283,
      "grad_norm": 11.790593147277832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08811427652835846,
      "logits/rejected": -0.03764200210571289,
      "logps/chosen": -499.14398193359375,
      "logps/rejected": -533.2659912109375,
      "loss": 0.6678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.500937283039093,
      "rewards/margins": 1.3944041728973389,
      "rewards/rejected": -1.8953415155410767,
      "step": 4782
    },
    {
      "epoch": 0.62589351435348,
      "grad_norm": 6.812265872955322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0752941220998764,
      "logits/rejected": 0.053499966859817505,
      "logps/chosen": -392.2716979980469,
      "logps/rejected": -361.885009765625,
      "loss": 0.355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21371498703956604,
      "rewards/margins": 1.116310477256775,
      "rewards/rejected": -1.330025553703308,
      "step": 4783
    },
    {
      "epoch": 0.6260243722908317,
      "grad_norm": 11.410316467285156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04542097449302673,
      "logits/rejected": -0.3144502341747284,
      "logps/chosen": -538.9522705078125,
      "logps/rejected": -439.1015625,
      "loss": 0.6611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9684526920318604,
      "rewards/margins": 0.49917179346084595,
      "rewards/rejected": -1.467624545097351,
      "step": 4784
    },
    {
      "epoch": 0.6261552302281835,
      "grad_norm": 8.62070083618164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031684037297964096,
      "logits/rejected": 0.1705503910779953,
      "logps/chosen": -556.802001953125,
      "logps/rejected": -573.9803466796875,
      "loss": 0.7686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24723322689533234,
      "rewards/margins": 0.5394211411476135,
      "rewards/rejected": -0.2921878695487976,
      "step": 4785
    },
    {
      "epoch": 0.6262860881655353,
      "grad_norm": 7.774996280670166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1802348494529724,
      "logits/rejected": -0.07195253670215607,
      "logps/chosen": -340.812255859375,
      "logps/rejected": -332.7418518066406,
      "loss": 0.5201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2162954807281494,
      "rewards/margins": 0.7634332776069641,
      "rewards/rejected": -1.9797288179397583,
      "step": 4786
    },
    {
      "epoch": 0.626416946102887,
      "grad_norm": 4.6950883865356445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040746673941612244,
      "logits/rejected": -0.028428491204977036,
      "logps/chosen": -475.56329345703125,
      "logps/rejected": -389.9692077636719,
      "loss": 0.318,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8108224868774414,
      "rewards/margins": 1.6115267276763916,
      "rewards/rejected": -2.422348976135254,
      "step": 4787
    },
    {
      "epoch": 0.6265478040402388,
      "grad_norm": 11.112854957580566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1663619875907898,
      "logits/rejected": -0.08586210012435913,
      "logps/chosen": -480.5400695800781,
      "logps/rejected": -716.9855346679688,
      "loss": 0.724,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7403055429458618,
      "rewards/margins": 0.49051225185394287,
      "rewards/rejected": -1.2308177947998047,
      "step": 4788
    },
    {
      "epoch": 0.6266786619775906,
      "grad_norm": 6.184844493865967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09848476946353912,
      "logits/rejected": 0.028672270476818085,
      "logps/chosen": -432.2580261230469,
      "logps/rejected": -443.27593994140625,
      "loss": 0.4788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8780155181884766,
      "rewards/margins": 0.5982751846313477,
      "rewards/rejected": -1.4762907028198242,
      "step": 4789
    },
    {
      "epoch": 0.6268095199149424,
      "grad_norm": 7.529881477355957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13033096492290497,
      "logits/rejected": 0.09763224422931671,
      "logps/chosen": -406.5978698730469,
      "logps/rejected": -395.19830322265625,
      "loss": 0.7373,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9963226318359375,
      "rewards/margins": 0.5908693671226501,
      "rewards/rejected": -1.5871919393539429,
      "step": 4790
    },
    {
      "epoch": 0.6269403778522941,
      "grad_norm": 6.324160099029541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09777282178401947,
      "logits/rejected": -0.11230921745300293,
      "logps/chosen": -379.50054931640625,
      "logps/rejected": -363.9241943359375,
      "loss": 0.375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04863357916474342,
      "rewards/margins": 1.1248849630355835,
      "rewards/rejected": -1.0762513875961304,
      "step": 4791
    },
    {
      "epoch": 0.6270712357896459,
      "grad_norm": 7.744998931884766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1050078272819519,
      "logits/rejected": -0.09212085604667664,
      "logps/chosen": -441.59783935546875,
      "logps/rejected": -413.4037780761719,
      "loss": 0.5622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3080696165561676,
      "rewards/margins": 0.552280604839325,
      "rewards/rejected": -0.8603502511978149,
      "step": 4792
    },
    {
      "epoch": 0.6272020937269976,
      "grad_norm": 10.29993724822998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007666692137718201,
      "logits/rejected": -0.1924792230129242,
      "logps/chosen": -504.5889587402344,
      "logps/rejected": -410.6455993652344,
      "loss": 0.6865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9030582904815674,
      "rewards/margins": 0.49341219663619995,
      "rewards/rejected": -1.3964704275131226,
      "step": 4793
    },
    {
      "epoch": 0.6273329516643494,
      "grad_norm": 8.728706359863281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07313410192728043,
      "logits/rejected": -0.06918854266405106,
      "logps/chosen": -362.3835754394531,
      "logps/rejected": -424.7021484375,
      "loss": 0.6386,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27634209394454956,
      "rewards/margins": 1.1711435317993164,
      "rewards/rejected": -1.4474858045578003,
      "step": 4794
    },
    {
      "epoch": 0.6274638096017011,
      "grad_norm": 6.178041458129883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18905498087406158,
      "logits/rejected": 0.2427048683166504,
      "logps/chosen": -545.6390991210938,
      "logps/rejected": -541.0473022460938,
      "loss": 0.4904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02342034876346588,
      "rewards/margins": 0.709289014339447,
      "rewards/rejected": -0.6858686208724976,
      "step": 4795
    },
    {
      "epoch": 0.6275946675390529,
      "grad_norm": 9.115797996520996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25095120072364807,
      "logits/rejected": -0.2213541865348816,
      "logps/chosen": -584.6994018554688,
      "logps/rejected": -462.75885009765625,
      "loss": 0.5604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5298411846160889,
      "rewards/margins": 0.8342044949531555,
      "rewards/rejected": -2.3640456199645996,
      "step": 4796
    },
    {
      "epoch": 0.6277255254764047,
      "grad_norm": 3.3065099716186523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10416791588068008,
      "logits/rejected": -0.012850799597799778,
      "logps/chosen": -328.6690673828125,
      "logps/rejected": -347.8978576660156,
      "loss": 0.2211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5230382084846497,
      "rewards/margins": 1.6674137115478516,
      "rewards/rejected": -1.1443754434585571,
      "step": 4797
    },
    {
      "epoch": 0.6278563834137565,
      "grad_norm": 12.783915519714355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3872491717338562,
      "logits/rejected": -0.06885091960430145,
      "logps/chosen": -824.8336791992188,
      "logps/rejected": -483.2491149902344,
      "loss": 0.4323,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9711039662361145,
      "rewards/margins": 0.9251552224159241,
      "rewards/rejected": -1.8962591886520386,
      "step": 4798
    },
    {
      "epoch": 0.6279872413511082,
      "grad_norm": 6.578712463378906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22354161739349365,
      "logits/rejected": -0.031005550175905228,
      "logps/chosen": -408.70123291015625,
      "logps/rejected": -336.21331787109375,
      "loss": 0.5061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3422068357467651,
      "rewards/margins": 0.9192403554916382,
      "rewards/rejected": -2.2614474296569824,
      "step": 4799
    },
    {
      "epoch": 0.62811809928846,
      "grad_norm": 6.864332675933838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10841740667819977,
      "logits/rejected": 0.29655638337135315,
      "logps/chosen": -627.14453125,
      "logps/rejected": -471.87109375,
      "loss": 0.5609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07430574297904968,
      "rewards/margins": 1.0248746871948242,
      "rewards/rejected": -1.0991805791854858,
      "step": 4800
    },
    {
      "epoch": 0.6282489572258118,
      "grad_norm": 9.635241508483887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17639042437076569,
      "logits/rejected": 0.06674925982952118,
      "logps/chosen": -510.2326965332031,
      "logps/rejected": -456.6126403808594,
      "loss": 0.7501,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9224356412887573,
      "rewards/margins": 0.192561537027359,
      "rewards/rejected": -1.114997148513794,
      "step": 4801
    },
    {
      "epoch": 0.6283798151631634,
      "grad_norm": 6.043928623199463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08085360378026962,
      "logits/rejected": -0.010947741568088531,
      "logps/chosen": -241.09066772460938,
      "logps/rejected": -232.9669189453125,
      "loss": 0.5673,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6764537692070007,
      "rewards/margins": 0.7244812250137329,
      "rewards/rejected": -1.4009350538253784,
      "step": 4802
    },
    {
      "epoch": 0.6285106731005152,
      "grad_norm": 5.920629024505615,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024941131472587585,
      "logits/rejected": -0.18244005739688873,
      "logps/chosen": -265.95623779296875,
      "logps/rejected": -314.6831359863281,
      "loss": 0.5567,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2662264406681061,
      "rewards/margins": 0.7331529259681702,
      "rewards/rejected": -0.9993793964385986,
      "step": 4803
    },
    {
      "epoch": 0.628641531037867,
      "grad_norm": 5.878748893737793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08429671078920364,
      "logits/rejected": 0.022024422883987427,
      "logps/chosen": -514.9683227539062,
      "logps/rejected": -562.1159057617188,
      "loss": 0.4056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5617386102676392,
      "rewards/margins": 1.8027037382125854,
      "rewards/rejected": -2.3644423484802246,
      "step": 4804
    },
    {
      "epoch": 0.6287723889752188,
      "grad_norm": 8.668302536010742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060485709458589554,
      "logits/rejected": -0.2175956666469574,
      "logps/chosen": -492.9566955566406,
      "logps/rejected": -455.7601318359375,
      "loss": 0.4586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4668736457824707,
      "rewards/margins": 1.65477454662323,
      "rewards/rejected": -2.121648073196411,
      "step": 4805
    },
    {
      "epoch": 0.6289032469125705,
      "grad_norm": 6.903409957885742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09597171097993851,
      "logits/rejected": 0.20381613075733185,
      "logps/chosen": -493.3736572265625,
      "logps/rejected": -444.9425048828125,
      "loss": 0.5675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.041078295558691025,
      "rewards/margins": 0.5320292115211487,
      "rewards/rejected": -0.5731075406074524,
      "step": 4806
    },
    {
      "epoch": 0.6290341048499223,
      "grad_norm": 6.20186185836792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2809605300426483,
      "logits/rejected": 0.39710479974746704,
      "logps/chosen": -377.7308349609375,
      "logps/rejected": -492.4341735839844,
      "loss": 0.4391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3722724914550781,
      "rewards/margins": 1.3171532154083252,
      "rewards/rejected": -1.6894257068634033,
      "step": 4807
    },
    {
      "epoch": 0.6291649627872741,
      "grad_norm": 7.613002300262451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024760112166404724,
      "logits/rejected": -0.05094463378190994,
      "logps/chosen": -581.5632934570312,
      "logps/rejected": -558.2131958007812,
      "loss": 0.5076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3616394102573395,
      "rewards/margins": 0.9420864582061768,
      "rewards/rejected": -1.3037258386611938,
      "step": 4808
    },
    {
      "epoch": 0.6292958207246259,
      "grad_norm": 8.437989234924316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1811731457710266,
      "logits/rejected": 0.1612679362297058,
      "logps/chosen": -479.08966064453125,
      "logps/rejected": -547.8511962890625,
      "loss": 0.7329,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1391360759735107,
      "rewards/margins": -0.0003800913691520691,
      "rewards/rejected": -1.1387560367584229,
      "step": 4809
    },
    {
      "epoch": 0.6294266786619775,
      "grad_norm": 7.26354455947876,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10724461823701859,
      "logits/rejected": -0.043561093509197235,
      "logps/chosen": -455.594482421875,
      "logps/rejected": -452.42547607421875,
      "loss": 0.4574,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19594934582710266,
      "rewards/margins": 1.2048900127410889,
      "rewards/rejected": -1.4008394479751587,
      "step": 4810
    },
    {
      "epoch": 0.6295575365993293,
      "grad_norm": 8.704809188842773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1981414556503296,
      "logits/rejected": -0.11244082450866699,
      "logps/chosen": -574.4242553710938,
      "logps/rejected": -632.0607299804688,
      "loss": 0.6101,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2436830699443817,
      "rewards/margins": 1.0130178928375244,
      "rewards/rejected": -1.2567009925842285,
      "step": 4811
    },
    {
      "epoch": 0.6296883945366811,
      "grad_norm": 9.320935249328613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00619104690849781,
      "logits/rejected": 0.0998515784740448,
      "logps/chosen": -726.9347534179688,
      "logps/rejected": -459.1648864746094,
      "loss": 0.6435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15340231359004974,
      "rewards/margins": 0.7210373878479004,
      "rewards/rejected": -0.8744397163391113,
      "step": 4812
    },
    {
      "epoch": 0.6298192524740329,
      "grad_norm": 6.384265422821045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36297792196273804,
      "logits/rejected": 0.33142000436782837,
      "logps/chosen": -429.97149658203125,
      "logps/rejected": -443.6891174316406,
      "loss": 0.4981,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2357839345932007,
      "rewards/margins": 1.1292502880096436,
      "rewards/rejected": -2.3650341033935547,
      "step": 4813
    },
    {
      "epoch": 0.6299501104113846,
      "grad_norm": 9.012585639953613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03265344351530075,
      "logits/rejected": 0.1470085084438324,
      "logps/chosen": -406.8015441894531,
      "logps/rejected": -442.7358703613281,
      "loss": 0.6225,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7379096746444702,
      "rewards/margins": 0.6679964065551758,
      "rewards/rejected": -1.405906081199646,
      "step": 4814
    },
    {
      "epoch": 0.6300809683487364,
      "grad_norm": 9.083913803100586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3005172908306122,
      "logits/rejected": 0.0884283185005188,
      "logps/chosen": -378.4520263671875,
      "logps/rejected": -481.81060791015625,
      "loss": 0.9671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5708732604980469,
      "rewards/margins": 0.19713440537452698,
      "rewards/rejected": -0.7680076360702515,
      "step": 4815
    },
    {
      "epoch": 0.6302118262860882,
      "grad_norm": 8.560083389282227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39939695596694946,
      "logits/rejected": -0.29683977365493774,
      "logps/chosen": -435.64483642578125,
      "logps/rejected": -444.07281494140625,
      "loss": 0.4786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.038265619426965714,
      "rewards/margins": 0.8380602598190308,
      "rewards/rejected": -0.799794614315033,
      "step": 4816
    },
    {
      "epoch": 0.63034268422344,
      "grad_norm": 8.792593955993652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006843153387308121,
      "logits/rejected": -0.0015880726277828217,
      "logps/chosen": -403.06878662109375,
      "logps/rejected": -463.3277282714844,
      "loss": 0.7169,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.762266993522644,
      "rewards/margins": 0.3839111030101776,
      "rewards/rejected": -1.146178126335144,
      "step": 4817
    },
    {
      "epoch": 0.6304735421607917,
      "grad_norm": 10.309081077575684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12680211663246155,
      "logits/rejected": 7.337704300880432e-05,
      "logps/chosen": -881.6315307617188,
      "logps/rejected": -818.285400390625,
      "loss": 0.5132,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11519049853086472,
      "rewards/margins": 0.6386066675186157,
      "rewards/rejected": -0.7537971138954163,
      "step": 4818
    },
    {
      "epoch": 0.6306044000981434,
      "grad_norm": 6.5317158699035645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06613122671842575,
      "logits/rejected": -0.10706744343042374,
      "logps/chosen": -354.703125,
      "logps/rejected": -338.52825927734375,
      "loss": 0.6068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32883092761039734,
      "rewards/margins": 0.37051886320114136,
      "rewards/rejected": -0.6993497610092163,
      "step": 4819
    },
    {
      "epoch": 0.6307352580354952,
      "grad_norm": 4.815422534942627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17437933385372162,
      "logits/rejected": 0.021129511296749115,
      "logps/chosen": -648.1831665039062,
      "logps/rejected": -425.5237121582031,
      "loss": 0.3469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3657684922218323,
      "rewards/margins": 1.2071354389190674,
      "rewards/rejected": -1.5729038715362549,
      "step": 4820
    },
    {
      "epoch": 0.630866115972847,
      "grad_norm": 8.310443878173828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0024918708950281143,
      "logits/rejected": 0.08486463129520416,
      "logps/chosen": -556.7481689453125,
      "logps/rejected": -618.2000732421875,
      "loss": 0.4623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1642833650112152,
      "rewards/margins": 0.8271564245223999,
      "rewards/rejected": -0.9914398193359375,
      "step": 4821
    },
    {
      "epoch": 0.6309969739101987,
      "grad_norm": 5.294393539428711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16977223753929138,
      "logits/rejected": 0.29270312190055847,
      "logps/chosen": -328.9492492675781,
      "logps/rejected": -381.92041015625,
      "loss": 0.4648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16520364582538605,
      "rewards/margins": 0.6768545508384705,
      "rewards/rejected": -0.8420581817626953,
      "step": 4822
    },
    {
      "epoch": 0.6311278318475505,
      "grad_norm": 6.857133388519287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011792320758104324,
      "logits/rejected": 0.0757383406162262,
      "logps/chosen": -420.3373107910156,
      "logps/rejected": -336.8558349609375,
      "loss": 0.3937,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3411460816860199,
      "rewards/margins": 1.0207010507583618,
      "rewards/rejected": -1.3618470430374146,
      "step": 4823
    },
    {
      "epoch": 0.6312586897849023,
      "grad_norm": 6.1223344802856445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006025034934282303,
      "logits/rejected": 0.2834402918815613,
      "logps/chosen": -436.63116455078125,
      "logps/rejected": -409.1827087402344,
      "loss": 0.3892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.67264723777771,
      "rewards/margins": 0.9376109838485718,
      "rewards/rejected": -1.6102583408355713,
      "step": 4824
    },
    {
      "epoch": 0.631389547722254,
      "grad_norm": 8.909560203552246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040344059467315674,
      "logits/rejected": -0.15093255043029785,
      "logps/chosen": -524.5721435546875,
      "logps/rejected": -444.1636657714844,
      "loss": 0.6865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7818506360054016,
      "rewards/margins": 0.22023656964302063,
      "rewards/rejected": -1.0020872354507446,
      "step": 4825
    },
    {
      "epoch": 0.6315204056596058,
      "grad_norm": 6.41315221786499,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29213619232177734,
      "logits/rejected": 0.21459737420082092,
      "logps/chosen": -376.7785949707031,
      "logps/rejected": -419.9173278808594,
      "loss": 0.4055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7864266037940979,
      "rewards/margins": 1.1004040241241455,
      "rewards/rejected": -1.8868305683135986,
      "step": 4826
    },
    {
      "epoch": 0.6316512635969576,
      "grad_norm": 7.62299919128418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19259974360466003,
      "logits/rejected": -0.010433094576001167,
      "logps/chosen": -467.27618408203125,
      "logps/rejected": -449.94354248046875,
      "loss": 0.471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2892261743545532,
      "rewards/margins": 0.9097853302955627,
      "rewards/rejected": -2.1990113258361816,
      "step": 4827
    },
    {
      "epoch": 0.6317821215343093,
      "grad_norm": 5.952317714691162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12995821237564087,
      "logits/rejected": 0.09936346113681793,
      "logps/chosen": -526.7400512695312,
      "logps/rejected": -417.9934387207031,
      "loss": 0.5129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4511200189590454,
      "rewards/margins": 0.9785345792770386,
      "rewards/rejected": -0.5274145603179932,
      "step": 4828
    },
    {
      "epoch": 0.631912979471661,
      "grad_norm": 7.427004337310791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07375659048557281,
      "logits/rejected": 0.14413480460643768,
      "logps/chosen": -306.7626953125,
      "logps/rejected": -345.8718566894531,
      "loss": 0.4484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41513121128082275,
      "rewards/margins": 1.0777791738510132,
      "rewards/rejected": -1.492910385131836,
      "step": 4829
    },
    {
      "epoch": 0.6320438374090128,
      "grad_norm": 7.218934535980225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11792866885662079,
      "logits/rejected": -0.11373091489076614,
      "logps/chosen": -368.1488037109375,
      "logps/rejected": -320.89129638671875,
      "loss": 0.4999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20442833006381989,
      "rewards/margins": 0.7714915871620178,
      "rewards/rejected": -0.9759199619293213,
      "step": 4830
    },
    {
      "epoch": 0.6321746953463646,
      "grad_norm": 7.323819160461426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06294436752796173,
      "logits/rejected": 0.032515749335289,
      "logps/chosen": -435.0303955078125,
      "logps/rejected": -326.8638610839844,
      "loss": 0.4907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10647965222597122,
      "rewards/margins": 0.9316447973251343,
      "rewards/rejected": -1.038124442100525,
      "step": 4831
    },
    {
      "epoch": 0.6323055532837164,
      "grad_norm": 9.069559097290039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1437690407037735,
      "logits/rejected": 0.14617809653282166,
      "logps/chosen": -559.3397216796875,
      "logps/rejected": -570.4639282226562,
      "loss": 0.459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7583250999450684,
      "rewards/margins": 1.29606294631958,
      "rewards/rejected": -2.0543878078460693,
      "step": 4832
    },
    {
      "epoch": 0.6324364112210681,
      "grad_norm": 7.519340991973877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2605280578136444,
      "logits/rejected": 0.29869404435157776,
      "logps/chosen": -315.4009094238281,
      "logps/rejected": -487.7467041015625,
      "loss": 0.6122,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.31769752502441406,
      "rewards/margins": 0.6883978247642517,
      "rewards/rejected": -1.006095290184021,
      "step": 4833
    },
    {
      "epoch": 0.6325672691584199,
      "grad_norm": 4.72331428527832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05363892391324043,
      "logits/rejected": -0.2020876258611679,
      "logps/chosen": -635.7628173828125,
      "logps/rejected": -434.74639892578125,
      "loss": 0.4208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12109509110450745,
      "rewards/margins": 1.6793584823608398,
      "rewards/rejected": -1.5582633018493652,
      "step": 4834
    },
    {
      "epoch": 0.6326981270957717,
      "grad_norm": 5.540227890014648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01351245492696762,
      "logits/rejected": -0.05041228234767914,
      "logps/chosen": -344.03326416015625,
      "logps/rejected": -480.2618103027344,
      "loss": 0.3584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9664139747619629,
      "rewards/margins": 1.1638764142990112,
      "rewards/rejected": -2.1302902698516846,
      "step": 4835
    },
    {
      "epoch": 0.6328289850331235,
      "grad_norm": 5.199012756347656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029770303517580032,
      "logits/rejected": -0.20975753664970398,
      "logps/chosen": -465.9062805175781,
      "logps/rejected": -464.95074462890625,
      "loss": 0.3492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07970942556858063,
      "rewards/margins": 1.4856935739517212,
      "rewards/rejected": -1.4059841632843018,
      "step": 4836
    },
    {
      "epoch": 0.6329598429704751,
      "grad_norm": 5.246423721313477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29915645718574524,
      "logits/rejected": 0.3270326852798462,
      "logps/chosen": -491.8480224609375,
      "logps/rejected": -520.3975219726562,
      "loss": 0.3515,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3769811689853668,
      "rewards/margins": 1.0987749099731445,
      "rewards/rejected": -1.4757561683654785,
      "step": 4837
    },
    {
      "epoch": 0.6330907009078269,
      "grad_norm": 7.15829610824585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2562013268470764,
      "logits/rejected": 0.21726518869400024,
      "logps/chosen": -414.076416015625,
      "logps/rejected": -427.51300048828125,
      "loss": 0.5996,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.99382084608078,
      "rewards/margins": 0.4146593511104584,
      "rewards/rejected": -1.4084800481796265,
      "step": 4838
    },
    {
      "epoch": 0.6332215588451787,
      "grad_norm": 7.944554805755615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09579375386238098,
      "logits/rejected": 0.14703026413917542,
      "logps/chosen": -573.440673828125,
      "logps/rejected": -716.2764892578125,
      "loss": 0.3736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9333646297454834,
      "rewards/margins": 1.492891788482666,
      "rewards/rejected": -2.4262564182281494,
      "step": 4839
    },
    {
      "epoch": 0.6333524167825305,
      "grad_norm": 5.780395984649658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052398040890693665,
      "logits/rejected": -0.104324571788311,
      "logps/chosen": -329.81365966796875,
      "logps/rejected": -440.22479248046875,
      "loss": 0.5115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5976219177246094,
      "rewards/margins": 0.8877472877502441,
      "rewards/rejected": -1.485369324684143,
      "step": 4840
    },
    {
      "epoch": 0.6334832747198822,
      "grad_norm": 6.503421306610107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13272055983543396,
      "logits/rejected": 0.06125656142830849,
      "logps/chosen": -425.96575927734375,
      "logps/rejected": -484.35089111328125,
      "loss": 0.4681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5164541602134705,
      "rewards/margins": 0.6501498222351074,
      "rewards/rejected": -1.166603922843933,
      "step": 4841
    },
    {
      "epoch": 0.633614132657234,
      "grad_norm": 7.901930809020996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017874781042337418,
      "logits/rejected": 0.07874812185764313,
      "logps/chosen": -458.42279052734375,
      "logps/rejected": -454.2322692871094,
      "loss": 0.5562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8277454376220703,
      "rewards/margins": 0.5180798768997192,
      "rewards/rejected": -1.3458251953125,
      "step": 4842
    },
    {
      "epoch": 0.6337449905945858,
      "grad_norm": 6.747294902801514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0946575254201889,
      "logits/rejected": -0.0005542375147342682,
      "logps/chosen": -594.3426513671875,
      "logps/rejected": -525.9529418945312,
      "loss": 0.4915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03535464406013489,
      "rewards/margins": 0.7399508953094482,
      "rewards/rejected": -0.7753055691719055,
      "step": 4843
    },
    {
      "epoch": 0.6338758485319376,
      "grad_norm": 8.492046356201172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007814452052116394,
      "logits/rejected": 0.20533248782157898,
      "logps/chosen": -703.342041015625,
      "logps/rejected": -504.35137939453125,
      "loss": 0.4803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5793742537498474,
      "rewards/margins": 0.8915372490882874,
      "rewards/rejected": -1.4709116220474243,
      "step": 4844
    },
    {
      "epoch": 0.6340067064692892,
      "grad_norm": 5.636306285858154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2517297863960266,
      "logits/rejected": -0.6776862144470215,
      "logps/chosen": -482.54931640625,
      "logps/rejected": -1185.0831298828125,
      "loss": 0.4974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5963647961616516,
      "rewards/margins": 1.6344314813613892,
      "rewards/rejected": -2.2307960987091064,
      "step": 4845
    },
    {
      "epoch": 0.634137564406641,
      "grad_norm": 3.593684434890747,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1896490752696991,
      "logits/rejected": -0.3158062994480133,
      "logps/chosen": -304.5944519042969,
      "logps/rejected": -338.8011474609375,
      "loss": 0.2033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2883341312408447,
      "rewards/margins": 1.7675021886825562,
      "rewards/rejected": -2.0558362007141113,
      "step": 4846
    },
    {
      "epoch": 0.6342684223439928,
      "grad_norm": 6.216789245605469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07322407513856888,
      "logits/rejected": 0.08446882665157318,
      "logps/chosen": -311.8804931640625,
      "logps/rejected": -493.027587890625,
      "loss": 0.3712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.72353196144104,
      "rewards/margins": 1.4052419662475586,
      "rewards/rejected": -2.1287739276885986,
      "step": 4847
    },
    {
      "epoch": 0.6343992802813445,
      "grad_norm": 4.74751091003418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2144901305437088,
      "logits/rejected": -0.03392297402024269,
      "logps/chosen": -346.049072265625,
      "logps/rejected": -448.1253662109375,
      "loss": 0.3183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7437238693237305,
      "rewards/margins": 1.0864143371582031,
      "rewards/rejected": -1.8301382064819336,
      "step": 4848
    },
    {
      "epoch": 0.6345301382186963,
      "grad_norm": 6.742302417755127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10166831314563751,
      "logits/rejected": 0.03354642540216446,
      "logps/chosen": -639.6376953125,
      "logps/rejected": -527.1036987304688,
      "loss": 0.4597,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5582090616226196,
      "rewards/margins": 0.976585865020752,
      "rewards/rejected": -1.5347950458526611,
      "step": 4849
    },
    {
      "epoch": 0.6346609961560481,
      "grad_norm": 6.569746971130371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08208061754703522,
      "logits/rejected": -0.22574974596500397,
      "logps/chosen": -440.0191650390625,
      "logps/rejected": -392.61968994140625,
      "loss": 0.5449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8901728987693787,
      "rewards/margins": 0.5689817667007446,
      "rewards/rejected": -1.4591546058654785,
      "step": 4850
    },
    {
      "epoch": 0.6347918540933999,
      "grad_norm": 8.189311027526855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2673477530479431,
      "logits/rejected": 0.21782207489013672,
      "logps/chosen": -442.114990234375,
      "logps/rejected": -504.5828857421875,
      "loss": 0.6157,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46602267026901245,
      "rewards/margins": 0.7226988077163696,
      "rewards/rejected": -1.1887215375900269,
      "step": 4851
    },
    {
      "epoch": 0.6349227120307516,
      "grad_norm": 5.661494731903076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31780683994293213,
      "logits/rejected": 0.1683903932571411,
      "logps/chosen": -447.3597412109375,
      "logps/rejected": -496.2569580078125,
      "loss": 0.3414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29403743147850037,
      "rewards/margins": 1.429093360900879,
      "rewards/rejected": -1.1350558996200562,
      "step": 4852
    },
    {
      "epoch": 0.6350535699681034,
      "grad_norm": 5.104898452758789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011420376598834991,
      "logits/rejected": 0.1978072226047516,
      "logps/chosen": -479.1835632324219,
      "logps/rejected": -670.4153442382812,
      "loss": 0.2927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.019024133682251,
      "rewards/margins": 2.2910921573638916,
      "rewards/rejected": -3.3101162910461426,
      "step": 4853
    },
    {
      "epoch": 0.6351844279054552,
      "grad_norm": 5.383807182312012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027215346693992615,
      "logits/rejected": -0.05839257687330246,
      "logps/chosen": -410.6429443359375,
      "logps/rejected": -477.10919189453125,
      "loss": 0.4075,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4000944197177887,
      "rewards/margins": 0.9765052795410156,
      "rewards/rejected": -1.3765997886657715,
      "step": 4854
    },
    {
      "epoch": 0.6353152858428069,
      "grad_norm": 5.631038665771484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006977178156375885,
      "logits/rejected": -0.026857122778892517,
      "logps/chosen": -394.7254638671875,
      "logps/rejected": -386.19171142578125,
      "loss": 0.4347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9200336933135986,
      "rewards/margins": 0.7224354147911072,
      "rewards/rejected": -1.6424689292907715,
      "step": 4855
    },
    {
      "epoch": 0.6354461437801586,
      "grad_norm": 7.408034324645996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13134866952896118,
      "logits/rejected": -0.01830749772489071,
      "logps/chosen": -432.1872253417969,
      "logps/rejected": -422.0546569824219,
      "loss": 0.3394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18742580711841583,
      "rewards/margins": 1.8263599872589111,
      "rewards/rejected": -2.0137858390808105,
      "step": 4856
    },
    {
      "epoch": 0.6355770017175104,
      "grad_norm": 4.639094352722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07936413586139679,
      "logits/rejected": 0.016047731041908264,
      "logps/chosen": -427.9778137207031,
      "logps/rejected": -380.0366516113281,
      "loss": 0.3218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5294122695922852,
      "rewards/margins": 1.1616051197052002,
      "rewards/rejected": -1.691017508506775,
      "step": 4857
    },
    {
      "epoch": 0.6357078596548622,
      "grad_norm": 9.540898323059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15313348174095154,
      "logits/rejected": -0.1379115730524063,
      "logps/chosen": -415.4769287109375,
      "logps/rejected": -392.7122802734375,
      "loss": 0.7157,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2298933267593384,
      "rewards/margins": 0.41595572233200073,
      "rewards/rejected": -1.6458491086959839,
      "step": 4858
    },
    {
      "epoch": 0.635838717592214,
      "grad_norm": 6.448486804962158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2100888341665268,
      "logits/rejected": -0.15128430724143982,
      "logps/chosen": -285.928955078125,
      "logps/rejected": -409.1813659667969,
      "loss": 0.3995,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.877179741859436,
      "rewards/margins": 1.2269127368927002,
      "rewards/rejected": -2.1040923595428467,
      "step": 4859
    },
    {
      "epoch": 0.6359695755295657,
      "grad_norm": 8.753425598144531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044407919049263,
      "logits/rejected": 0.08809379488229752,
      "logps/chosen": -432.19744873046875,
      "logps/rejected": -721.185302734375,
      "loss": 0.503,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1470255851745605,
      "rewards/margins": 0.45411208271980286,
      "rewards/rejected": -1.601137638092041,
      "step": 4860
    },
    {
      "epoch": 0.6361004334669175,
      "grad_norm": 6.108490943908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3098439574241638,
      "logits/rejected": -0.2384168803691864,
      "logps/chosen": -405.7752380371094,
      "logps/rejected": -470.3715515136719,
      "loss": 0.3096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8015168905258179,
      "rewards/margins": 1.3306022882461548,
      "rewards/rejected": -2.1321191787719727,
      "step": 4861
    },
    {
      "epoch": 0.6362312914042693,
      "grad_norm": 4.6673359870910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07975082844495773,
      "logits/rejected": -0.22294560074806213,
      "logps/chosen": -469.85223388671875,
      "logps/rejected": -502.6391906738281,
      "loss": 0.3282,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08333472162485123,
      "rewards/margins": 2.0247914791107178,
      "rewards/rejected": -2.108126401901245,
      "step": 4862
    },
    {
      "epoch": 0.636362149341621,
      "grad_norm": 7.219948768615723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2369125336408615,
      "logits/rejected": 0.2599174976348877,
      "logps/chosen": -616.935791015625,
      "logps/rejected": -523.5291137695312,
      "loss": 0.4635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6882286667823792,
      "rewards/margins": 1.1639127731323242,
      "rewards/rejected": -1.852141261100769,
      "step": 4863
    },
    {
      "epoch": 0.6364930072789727,
      "grad_norm": 8.59803295135498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1192527711391449,
      "logits/rejected": 0.11947882175445557,
      "logps/chosen": -461.5545654296875,
      "logps/rejected": -348.9900207519531,
      "loss": 0.5022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30977919697761536,
      "rewards/margins": 0.8183883428573608,
      "rewards/rejected": -1.1281675100326538,
      "step": 4864
    },
    {
      "epoch": 0.6366238652163245,
      "grad_norm": 6.391237735748291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03523285686969757,
      "logits/rejected": 0.011326435953378677,
      "logps/chosen": -341.7595520019531,
      "logps/rejected": -438.8254699707031,
      "loss": 0.5077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2999666929244995,
      "rewards/margins": 0.7834444642066956,
      "rewards/rejected": -2.08341121673584,
      "step": 4865
    },
    {
      "epoch": 0.6367547231536763,
      "grad_norm": 10.067594528198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.379064679145813,
      "logits/rejected": -0.2954684793949127,
      "logps/chosen": -343.416748046875,
      "logps/rejected": -390.8380126953125,
      "loss": 0.9515,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4448078870773315,
      "rewards/margins": -0.08728636801242828,
      "rewards/rejected": -1.3575215339660645,
      "step": 4866
    },
    {
      "epoch": 0.6368855810910281,
      "grad_norm": 5.027072906494141,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08080074936151505,
      "logits/rejected": 0.04095388576388359,
      "logps/chosen": -566.3508911132812,
      "logps/rejected": -457.643798828125,
      "loss": 0.3033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2677028477191925,
      "rewards/margins": 1.9734410047531128,
      "rewards/rejected": -2.2411439418792725,
      "step": 4867
    },
    {
      "epoch": 0.6370164390283798,
      "grad_norm": 11.847046852111816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12479505687952042,
      "logits/rejected": -0.14441300928592682,
      "logps/chosen": -460.951904296875,
      "logps/rejected": -426.23291015625,
      "loss": 0.7525,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1014026403427124,
      "rewards/margins": 0.5139976739883423,
      "rewards/rejected": -1.6154003143310547,
      "step": 4868
    },
    {
      "epoch": 0.6371472969657316,
      "grad_norm": 5.444902420043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09276704490184784,
      "logits/rejected": -0.11342491209506989,
      "logps/chosen": -330.9547119140625,
      "logps/rejected": -367.3377380371094,
      "loss": 0.2677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9474664926528931,
      "rewards/margins": 1.7704662084579468,
      "rewards/rejected": -2.71793270111084,
      "step": 4869
    },
    {
      "epoch": 0.6372781549030834,
      "grad_norm": 8.882987976074219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05926303565502167,
      "logits/rejected": -0.019488781690597534,
      "logps/chosen": -429.0050964355469,
      "logps/rejected": -371.8312683105469,
      "loss": 0.7177,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0557653903961182,
      "rewards/margins": 0.26068514585494995,
      "rewards/rejected": -1.316450595855713,
      "step": 4870
    },
    {
      "epoch": 0.637409012840435,
      "grad_norm": 6.969089984893799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07897543907165527,
      "logits/rejected": -0.05384926497936249,
      "logps/chosen": -467.1081237792969,
      "logps/rejected": -466.35736083984375,
      "loss": 0.5291,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.197324514389038,
      "rewards/margins": 0.8022512197494507,
      "rewards/rejected": -1.9995757341384888,
      "step": 4871
    },
    {
      "epoch": 0.6375398707777868,
      "grad_norm": 12.816763877868652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023042920976877213,
      "logits/rejected": -0.022484447807073593,
      "logps/chosen": -496.2271728515625,
      "logps/rejected": -510.347412109375,
      "loss": 0.934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3887443542480469,
      "rewards/margins": -0.15142817795276642,
      "rewards/rejected": -1.2373161315917969,
      "step": 4872
    },
    {
      "epoch": 0.6376707287151386,
      "grad_norm": 7.171318054199219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13882510364055634,
      "logits/rejected": -0.19365495443344116,
      "logps/chosen": -390.6228332519531,
      "logps/rejected": -305.64154052734375,
      "loss": 0.5228,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6852480173110962,
      "rewards/margins": 0.7954553961753845,
      "rewards/rejected": -1.4807034730911255,
      "step": 4873
    },
    {
      "epoch": 0.6378015866524904,
      "grad_norm": 7.211106777191162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.155316561460495,
      "logits/rejected": -0.10036933422088623,
      "logps/chosen": -501.3670654296875,
      "logps/rejected": -479.9554138183594,
      "loss": 0.4346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0568596124649048,
      "rewards/margins": 0.942126452922821,
      "rewards/rejected": -1.998986005783081,
      "step": 4874
    },
    {
      "epoch": 0.6379324445898421,
      "grad_norm": 9.569374084472656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1591292917728424,
      "logits/rejected": 0.16404509544372559,
      "logps/chosen": -614.1922607421875,
      "logps/rejected": -618.5556640625,
      "loss": 0.6499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4091955423355103,
      "rewards/margins": 0.37917423248291016,
      "rewards/rejected": -1.7883696556091309,
      "step": 4875
    },
    {
      "epoch": 0.6380633025271939,
      "grad_norm": 6.700501918792725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11605240404605865,
      "logits/rejected": 0.12958525121212006,
      "logps/chosen": -331.33172607421875,
      "logps/rejected": -313.47607421875,
      "loss": 0.627,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6079901456832886,
      "rewards/margins": 0.6383602619171143,
      "rewards/rejected": -1.2463505268096924,
      "step": 4876
    },
    {
      "epoch": 0.6381941604645457,
      "grad_norm": 6.066207408905029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12479442358016968,
      "logits/rejected": -0.017952490597963333,
      "logps/chosen": -499.23199462890625,
      "logps/rejected": -766.5443115234375,
      "loss": 0.3088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7439937591552734,
      "rewards/margins": 1.3109039068222046,
      "rewards/rejected": -2.0548977851867676,
      "step": 4877
    },
    {
      "epoch": 0.6383250184018975,
      "grad_norm": 5.998417854309082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1631164401769638,
      "logits/rejected": -0.03143121302127838,
      "logps/chosen": -672.1063232421875,
      "logps/rejected": -469.24755859375,
      "loss": 0.2804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44765034317970276,
      "rewards/margins": 1.6672394275665283,
      "rewards/rejected": -2.1148898601531982,
      "step": 4878
    },
    {
      "epoch": 0.6384558763392492,
      "grad_norm": 5.51646089553833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4151705205440521,
      "logits/rejected": -0.359121173620224,
      "logps/chosen": -351.1617126464844,
      "logps/rejected": -422.5985107421875,
      "loss": 0.2351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6138114929199219,
      "rewards/margins": 1.8082209825515747,
      "rewards/rejected": -2.422032356262207,
      "step": 4879
    },
    {
      "epoch": 0.638586734276601,
      "grad_norm": 7.57819938659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16452227532863617,
      "logits/rejected": -0.11183489114046097,
      "logps/chosen": -524.6069946289062,
      "logps/rejected": -483.6021423339844,
      "loss": 0.4655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2440818548202515,
      "rewards/margins": 0.946725606918335,
      "rewards/rejected": -2.190807580947876,
      "step": 4880
    },
    {
      "epoch": 0.6387175922139527,
      "grad_norm": 13.603168487548828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13681671023368835,
      "logits/rejected": -0.003132641315460205,
      "logps/chosen": -445.6482238769531,
      "logps/rejected": -444.4562072753906,
      "loss": 1.1419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8690239191055298,
      "rewards/margins": -0.33334505558013916,
      "rewards/rejected": -1.5356789827346802,
      "step": 4881
    },
    {
      "epoch": 0.6388484501513045,
      "grad_norm": 7.6311774253845215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04459013044834137,
      "logits/rejected": 0.09150667488574982,
      "logps/chosen": -520.6174926757812,
      "logps/rejected": -498.72412109375,
      "loss": 0.5546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48406752943992615,
      "rewards/margins": 0.8355310559272766,
      "rewards/rejected": -1.31959867477417,
      "step": 4882
    },
    {
      "epoch": 0.6389793080886562,
      "grad_norm": 11.543521881103516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21151769161224365,
      "logits/rejected": -0.008403102867305279,
      "logps/chosen": -577.2779541015625,
      "logps/rejected": -450.525146484375,
      "loss": 0.6726,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1297235488891602,
      "rewards/margins": 0.5099197030067444,
      "rewards/rejected": -1.6396431922912598,
      "step": 4883
    },
    {
      "epoch": 0.639110166026008,
      "grad_norm": 6.218032360076904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3180485963821411,
      "logits/rejected": -0.34285804629325867,
      "logps/chosen": -470.6781311035156,
      "logps/rejected": -528.6825561523438,
      "loss": 0.3286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6318275928497314,
      "rewards/margins": 1.2884094715118408,
      "rewards/rejected": -1.9202369451522827,
      "step": 4884
    },
    {
      "epoch": 0.6392410239633598,
      "grad_norm": 3.764497995376587,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27993834018707275,
      "logits/rejected": 0.31600457429885864,
      "logps/chosen": -356.0246887207031,
      "logps/rejected": -424.0727233886719,
      "loss": 0.2619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7513256072998047,
      "rewards/margins": 1.688706398010254,
      "rewards/rejected": -2.4400322437286377,
      "step": 4885
    },
    {
      "epoch": 0.6393718819007116,
      "grad_norm": 10.270286560058594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13611124455928802,
      "logits/rejected": -0.12591716647148132,
      "logps/chosen": -492.3313293457031,
      "logps/rejected": -311.7361145019531,
      "loss": 0.6717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5832169055938721,
      "rewards/margins": 0.7091021537780762,
      "rewards/rejected": -1.2923190593719482,
      "step": 4886
    },
    {
      "epoch": 0.6395027398380633,
      "grad_norm": 12.511096954345703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03508296236395836,
      "logits/rejected": -0.2645435333251953,
      "logps/chosen": -773.4910888671875,
      "logps/rejected": -529.6473388671875,
      "loss": 0.7447,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.135982871055603,
      "rewards/margins": 0.540115237236023,
      "rewards/rejected": -1.6760982275009155,
      "step": 4887
    },
    {
      "epoch": 0.6396335977754151,
      "grad_norm": 4.177989959716797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09772703796625137,
      "logits/rejected": 0.09389200806617737,
      "logps/chosen": -487.7865295410156,
      "logps/rejected": -409.5303955078125,
      "loss": 0.3929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10857601463794708,
      "rewards/margins": 1.1742171049118042,
      "rewards/rejected": -1.2827930450439453,
      "step": 4888
    },
    {
      "epoch": 0.6397644557127669,
      "grad_norm": 9.159683227539062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13046641647815704,
      "logits/rejected": -0.1012381911277771,
      "logps/chosen": -568.19384765625,
      "logps/rejected": -339.6308288574219,
      "loss": 0.5758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30344313383102417,
      "rewards/margins": 0.883513867855072,
      "rewards/rejected": -1.1869571208953857,
      "step": 4889
    },
    {
      "epoch": 0.6398953136501185,
      "grad_norm": 7.890264511108398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18112996220588684,
      "logits/rejected": 0.0662718117237091,
      "logps/chosen": -672.6127319335938,
      "logps/rejected": -510.9516906738281,
      "loss": 0.5227,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5451563596725464,
      "rewards/margins": 0.8050745129585266,
      "rewards/rejected": -1.3502308130264282,
      "step": 4890
    },
    {
      "epoch": 0.6400261715874703,
      "grad_norm": 5.553528308868408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.000647030770778656,
      "logits/rejected": -0.06576263159513474,
      "logps/chosen": -435.29541015625,
      "logps/rejected": -424.8873291015625,
      "loss": 0.3144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4714353084564209,
      "rewards/margins": 1.5692843198776245,
      "rewards/rejected": -2.040719509124756,
      "step": 4891
    },
    {
      "epoch": 0.6401570295248221,
      "grad_norm": 7.6893110275268555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10851938277482986,
      "logits/rejected": -0.1087062731385231,
      "logps/chosen": -596.1282958984375,
      "logps/rejected": -446.68084716796875,
      "loss": 0.5168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8208532333374023,
      "rewards/margins": 1.125069499015808,
      "rewards/rejected": -1.9459228515625,
      "step": 4892
    },
    {
      "epoch": 0.6402878874621739,
      "grad_norm": 5.910944938659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04218438267707825,
      "logits/rejected": 0.12938790023326874,
      "logps/chosen": -490.8974304199219,
      "logps/rejected": -445.8097839355469,
      "loss": 0.3345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22644537687301636,
      "rewards/margins": 1.157283902168274,
      "rewards/rejected": -1.3837292194366455,
      "step": 4893
    },
    {
      "epoch": 0.6404187453995256,
      "grad_norm": 4.846904754638672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22686734795570374,
      "logits/rejected": -0.35655510425567627,
      "logps/chosen": -621.4552001953125,
      "logps/rejected": -482.60968017578125,
      "loss": 0.2453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.511309027671814,
      "rewards/margins": 1.6506010293960571,
      "rewards/rejected": -2.161910057067871,
      "step": 4894
    },
    {
      "epoch": 0.6405496033368774,
      "grad_norm": 6.620492935180664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05004056543111801,
      "logits/rejected": -0.07424704730510712,
      "logps/chosen": -444.6082763671875,
      "logps/rejected": -368.23089599609375,
      "loss": 0.4595,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.427471399307251,
      "rewards/margins": 0.7918838858604431,
      "rewards/rejected": -2.219355344772339,
      "step": 4895
    },
    {
      "epoch": 0.6406804612742292,
      "grad_norm": 9.449313163757324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19991138577461243,
      "logits/rejected": -0.11865097284317017,
      "logps/chosen": -514.935302734375,
      "logps/rejected": -483.2733459472656,
      "loss": 0.515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9069024324417114,
      "rewards/margins": 1.299201488494873,
      "rewards/rejected": -2.206104040145874,
      "step": 4896
    },
    {
      "epoch": 0.640811319211581,
      "grad_norm": 9.689620018005371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1675690859556198,
      "logits/rejected": -0.20806191861629486,
      "logps/chosen": -589.46630859375,
      "logps/rejected": -360.69268798828125,
      "loss": 0.7401,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9005208015441895,
      "rewards/margins": 0.5761867165565491,
      "rewards/rejected": -1.4767074584960938,
      "step": 4897
    },
    {
      "epoch": 0.6409421771489326,
      "grad_norm": 9.421876907348633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01927095651626587,
      "logits/rejected": -0.20405283570289612,
      "logps/chosen": -357.1950988769531,
      "logps/rejected": -311.3983154296875,
      "loss": 0.807,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1590557098388672,
      "rewards/margins": 0.25567570328712463,
      "rewards/rejected": -1.414731502532959,
      "step": 4898
    },
    {
      "epoch": 0.6410730350862844,
      "grad_norm": 9.64151668548584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16297122836112976,
      "logits/rejected": -0.18755270540714264,
      "logps/chosen": -490.2358703613281,
      "logps/rejected": -584.0399780273438,
      "loss": 0.6597,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17473891377449036,
      "rewards/margins": 0.5959682464599609,
      "rewards/rejected": -0.7707072496414185,
      "step": 4899
    },
    {
      "epoch": 0.6412038930236362,
      "grad_norm": 4.286140441894531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0038475170731544495,
      "logits/rejected": 0.0658903419971466,
      "logps/chosen": -490.4453125,
      "logps/rejected": -500.01239013671875,
      "loss": 0.3486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4193475842475891,
      "rewards/margins": 1.7417612075805664,
      "rewards/rejected": -2.16110897064209,
      "step": 4900
    },
    {
      "epoch": 0.641334750960988,
      "grad_norm": 5.09479284286499,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013455159962177277,
      "logits/rejected": -0.05427497625350952,
      "logps/chosen": -383.26434326171875,
      "logps/rejected": -376.9930114746094,
      "loss": 0.3488,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4923194944858551,
      "rewards/margins": 1.3344451189041138,
      "rewards/rejected": -1.8267645835876465,
      "step": 4901
    },
    {
      "epoch": 0.6414656088983397,
      "grad_norm": 7.241476535797119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043265487998723984,
      "logits/rejected": 0.005829727277159691,
      "logps/chosen": -397.7980651855469,
      "logps/rejected": -474.9749755859375,
      "loss": 0.4456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22741356492042542,
      "rewards/margins": 1.2242931127548218,
      "rewards/rejected": -1.4517065286636353,
      "step": 4902
    },
    {
      "epoch": 0.6415964668356915,
      "grad_norm": 7.572305679321289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07600889354944229,
      "logits/rejected": 0.0445237010717392,
      "logps/chosen": -399.2221984863281,
      "logps/rejected": -483.92913818359375,
      "loss": 0.4259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38248157501220703,
      "rewards/margins": 1.0401475429534912,
      "rewards/rejected": -1.4226291179656982,
      "step": 4903
    },
    {
      "epoch": 0.6417273247730433,
      "grad_norm": 5.728835105895996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1357555240392685,
      "logits/rejected": -0.17313537001609802,
      "logps/chosen": -244.91275024414062,
      "logps/rejected": -254.43276977539062,
      "loss": 0.6437,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5401780605316162,
      "rewards/margins": 0.8990938663482666,
      "rewards/rejected": -1.4392720460891724,
      "step": 4904
    },
    {
      "epoch": 0.6418581827103951,
      "grad_norm": 5.845974445343018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1447169929742813,
      "logits/rejected": -0.17625199258327484,
      "logps/chosen": -469.10943603515625,
      "logps/rejected": -383.8923034667969,
      "loss": 0.3831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2268049418926239,
      "rewards/margins": 1.214470624923706,
      "rewards/rejected": -1.4412754774093628,
      "step": 4905
    },
    {
      "epoch": 0.6419890406477468,
      "grad_norm": 8.93375301361084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1045457124710083,
      "logits/rejected": -0.10830041021108627,
      "logps/chosen": -522.9063110351562,
      "logps/rejected": -432.96685791015625,
      "loss": 0.7714,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.39748552441596985,
      "rewards/margins": 0.27968519926071167,
      "rewards/rejected": -0.6771707534790039,
      "step": 4906
    },
    {
      "epoch": 0.6421198985850985,
      "grad_norm": 11.692405700683594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17112022638320923,
      "logits/rejected": 0.36943328380584717,
      "logps/chosen": -663.2412719726562,
      "logps/rejected": -615.5790405273438,
      "loss": 0.8855,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7920223474502563,
      "rewards/margins": 0.1603672206401825,
      "rewards/rejected": -0.9523895978927612,
      "step": 4907
    },
    {
      "epoch": 0.6422507565224503,
      "grad_norm": 9.682229995727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03931056708097458,
      "logits/rejected": -0.23235997557640076,
      "logps/chosen": -622.4869995117188,
      "logps/rejected": -483.1039123535156,
      "loss": 0.6223,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0153534412384033,
      "rewards/margins": 0.699387788772583,
      "rewards/rejected": -1.7147411108016968,
      "step": 4908
    },
    {
      "epoch": 0.6423816144598021,
      "grad_norm": 10.518232345581055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025548595935106277,
      "logits/rejected": -0.09282129257917404,
      "logps/chosen": -494.0904846191406,
      "logps/rejected": -360.4998474121094,
      "loss": 0.7613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5555681586265564,
      "rewards/margins": -0.06740894913673401,
      "rewards/rejected": -0.4881591796875,
      "step": 4909
    },
    {
      "epoch": 0.6425124723971538,
      "grad_norm": 5.2699971199035645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11201779544353485,
      "logits/rejected": 0.045688651502132416,
      "logps/chosen": -711.7982177734375,
      "logps/rejected": -496.76629638671875,
      "loss": 0.343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08609753847122192,
      "rewards/margins": 1.87669837474823,
      "rewards/rejected": -1.9627959728240967,
      "step": 4910
    },
    {
      "epoch": 0.6426433303345056,
      "grad_norm": 6.681458950042725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5045360922813416,
      "logits/rejected": 0.3414389193058014,
      "logps/chosen": -629.9227294921875,
      "logps/rejected": -504.259033203125,
      "loss": 0.4139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6251171231269836,
      "rewards/margins": 0.8636935949325562,
      "rewards/rejected": -1.488810658454895,
      "step": 4911
    },
    {
      "epoch": 0.6427741882718574,
      "grad_norm": 10.370772361755371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2192572057247162,
      "logits/rejected": -0.40608394145965576,
      "logps/chosen": -500.4346923828125,
      "logps/rejected": -350.89312744140625,
      "loss": 0.6616,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3088455200195312,
      "rewards/margins": 0.7786989212036133,
      "rewards/rejected": -2.0875442028045654,
      "step": 4912
    },
    {
      "epoch": 0.6429050462092091,
      "grad_norm": 7.250514030456543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10673187673091888,
      "logits/rejected": 0.051301755011081696,
      "logps/chosen": -404.56201171875,
      "logps/rejected": -510.50128173828125,
      "loss": 0.4101,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8306457996368408,
      "rewards/margins": 1.0802123546600342,
      "rewards/rejected": -1.9108580350875854,
      "step": 4913
    },
    {
      "epoch": 0.6430359041465609,
      "grad_norm": 6.9459991455078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1157664805650711,
      "logits/rejected": -0.3157936632633209,
      "logps/chosen": -412.11346435546875,
      "logps/rejected": -369.09344482421875,
      "loss": 0.5741,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.816606879234314,
      "rewards/margins": 0.4451878070831299,
      "rewards/rejected": -1.2617946863174438,
      "step": 4914
    },
    {
      "epoch": 0.6431667620839127,
      "grad_norm": 8.122779846191406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2884548008441925,
      "logits/rejected": 0.2278463989496231,
      "logps/chosen": -319.1300354003906,
      "logps/rejected": -398.4200744628906,
      "loss": 0.6407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9716910123825073,
      "rewards/margins": 1.227347493171692,
      "rewards/rejected": -2.19903826713562,
      "step": 4915
    },
    {
      "epoch": 0.6432976200212644,
      "grad_norm": 5.067844867706299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10820689052343369,
      "logits/rejected": 0.0709538608789444,
      "logps/chosen": -519.91064453125,
      "logps/rejected": -507.53424072265625,
      "loss": 0.2729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35840606689453125,
      "rewards/margins": 1.4663126468658447,
      "rewards/rejected": -1.824718713760376,
      "step": 4916
    },
    {
      "epoch": 0.6434284779586161,
      "grad_norm": 9.338945388793945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16583064198493958,
      "logits/rejected": -0.36324524879455566,
      "logps/chosen": -358.2715759277344,
      "logps/rejected": -410.2822265625,
      "loss": 0.6083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9058792591094971,
      "rewards/margins": 1.0142850875854492,
      "rewards/rejected": -1.9201643466949463,
      "step": 4917
    },
    {
      "epoch": 0.6435593358959679,
      "grad_norm": 10.292008399963379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09926571696996689,
      "logits/rejected": -0.024895476177334785,
      "logps/chosen": -554.8553466796875,
      "logps/rejected": -287.10467529296875,
      "loss": 0.6447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9682255387306213,
      "rewards/margins": 0.5038412809371948,
      "rewards/rejected": -1.4720666408538818,
      "step": 4918
    },
    {
      "epoch": 0.6436901938333197,
      "grad_norm": 10.920121192932129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00967961736023426,
      "logits/rejected": 0.024937178939580917,
      "logps/chosen": -575.445068359375,
      "logps/rejected": -543.7740478515625,
      "loss": 0.8669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4606773853302002,
      "rewards/margins": 0.30636292695999146,
      "rewards/rejected": -1.7670403718948364,
      "step": 4919
    },
    {
      "epoch": 0.6438210517706715,
      "grad_norm": 6.2073163986206055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1310342699289322,
      "logits/rejected": -0.1659260392189026,
      "logps/chosen": -336.65093994140625,
      "logps/rejected": -314.2591552734375,
      "loss": 0.3527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1956394463777542,
      "rewards/margins": 1.7138279676437378,
      "rewards/rejected": -1.909467339515686,
      "step": 4920
    },
    {
      "epoch": 0.6439519097080232,
      "grad_norm": 2.567359209060669,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08766844868659973,
      "logits/rejected": -0.35929369926452637,
      "logps/chosen": -430.64569091796875,
      "logps/rejected": -544.5299072265625,
      "loss": 0.1937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06969118118286133,
      "rewards/margins": 1.8039013147354126,
      "rewards/rejected": -1.7342101335525513,
      "step": 4921
    },
    {
      "epoch": 0.644082767645375,
      "grad_norm": 5.512282848358154,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03567208722233772,
      "logits/rejected": -0.09771007299423218,
      "logps/chosen": -414.7953796386719,
      "logps/rejected": -438.1368713378906,
      "loss": 0.3806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8499937057495117,
      "rewards/margins": 1.1531906127929688,
      "rewards/rejected": -2.0031840801239014,
      "step": 4922
    },
    {
      "epoch": 0.6442136255827268,
      "grad_norm": 5.924254894256592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015986885875463486,
      "logits/rejected": 0.02279473841190338,
      "logps/chosen": -616.19775390625,
      "logps/rejected": -420.23797607421875,
      "loss": 0.2576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4590989947319031,
      "rewards/margins": 1.5401684045791626,
      "rewards/rejected": -1.0810695886611938,
      "step": 4923
    },
    {
      "epoch": 0.6443444835200786,
      "grad_norm": 8.904881477355957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11506930738687515,
      "logits/rejected": 0.037796225398778915,
      "logps/chosen": -484.0346374511719,
      "logps/rejected": -565.2754516601562,
      "loss": 0.4796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006326884031295776,
      "rewards/margins": 0.8846191763877869,
      "rewards/rejected": -0.8782923221588135,
      "step": 4924
    },
    {
      "epoch": 0.6444753414574302,
      "grad_norm": 5.82301139831543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.318869411945343,
      "logits/rejected": -0.3521208167076111,
      "logps/chosen": -396.9930419921875,
      "logps/rejected": -458.69915771484375,
      "loss": 0.4398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9085822701454163,
      "rewards/margins": 0.6636667251586914,
      "rewards/rejected": -1.5722490549087524,
      "step": 4925
    },
    {
      "epoch": 0.644606199394782,
      "grad_norm": 9.148452758789062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08958525210618973,
      "logits/rejected": -0.08740495145320892,
      "logps/chosen": -388.53985595703125,
      "logps/rejected": -426.52410888671875,
      "loss": 0.6273,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2021055370569229,
      "rewards/margins": 1.0477126836776733,
      "rewards/rejected": -1.2498183250427246,
      "step": 4926
    },
    {
      "epoch": 0.6447370573321338,
      "grad_norm": 6.822624206542969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3149009943008423,
      "logits/rejected": -0.16990381479263306,
      "logps/chosen": -577.157958984375,
      "logps/rejected": -404.6108703613281,
      "loss": 0.3473,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3292512893676758,
      "rewards/margins": 1.5651346445083618,
      "rewards/rejected": -1.8943859338760376,
      "step": 4927
    },
    {
      "epoch": 0.6448679152694856,
      "grad_norm": 12.847652435302734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07682874798774719,
      "logits/rejected": -0.3535400331020355,
      "logps/chosen": -491.9835510253906,
      "logps/rejected": -295.7986755371094,
      "loss": 0.9158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7468464374542236,
      "rewards/margins": 0.44820648431777954,
      "rewards/rejected": -2.1950528621673584,
      "step": 4928
    },
    {
      "epoch": 0.6449987732068373,
      "grad_norm": 7.178208827972412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.367405503988266,
      "logits/rejected": 0.33633625507354736,
      "logps/chosen": -592.0179443359375,
      "logps/rejected": -631.360595703125,
      "loss": 0.5657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.052808210253715515,
      "rewards/margins": 0.8514318466186523,
      "rewards/rejected": -0.7986236810684204,
      "step": 4929
    },
    {
      "epoch": 0.6451296311441891,
      "grad_norm": 9.652752876281738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1690226048231125,
      "logits/rejected": -0.1324853003025055,
      "logps/chosen": -488.7850341796875,
      "logps/rejected": -322.14654541015625,
      "loss": 0.5439,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5756193399429321,
      "rewards/margins": 0.9708084464073181,
      "rewards/rejected": -1.5464279651641846,
      "step": 4930
    },
    {
      "epoch": 0.6452604890815409,
      "grad_norm": 11.732839584350586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11299842596054077,
      "logits/rejected": -0.16190503537654877,
      "logps/chosen": -457.2015380859375,
      "logps/rejected": -613.9199829101562,
      "loss": 0.6253,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.808837890625,
      "rewards/margins": 1.5134263038635254,
      "rewards/rejected": -2.3222639560699463,
      "step": 4931
    },
    {
      "epoch": 0.6453913470188927,
      "grad_norm": 7.640094757080078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29797592759132385,
      "logits/rejected": 0.21665242314338684,
      "logps/chosen": -659.7929077148438,
      "logps/rejected": -599.1314697265625,
      "loss": 0.6055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5700092315673828,
      "rewards/margins": 0.48948824405670166,
      "rewards/rejected": -1.0594974756240845,
      "step": 4932
    },
    {
      "epoch": 0.6455222049562443,
      "grad_norm": 8.724406242370605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06448494642972946,
      "logits/rejected": 0.23928691446781158,
      "logps/chosen": -415.0558776855469,
      "logps/rejected": -420.4871826171875,
      "loss": 0.6583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7544626593589783,
      "rewards/margins": 0.27871227264404297,
      "rewards/rejected": -1.0331748723983765,
      "step": 4933
    },
    {
      "epoch": 0.6456530628935961,
      "grad_norm": 18.15542984008789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21123096346855164,
      "logits/rejected": -0.09831728041172028,
      "logps/chosen": -672.39697265625,
      "logps/rejected": -601.1051635742188,
      "loss": 0.8166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.204918384552002,
      "rewards/margins": 0.2138337939977646,
      "rewards/rejected": -1.4187523126602173,
      "step": 4934
    },
    {
      "epoch": 0.6457839208309479,
      "grad_norm": 3.891057252883911,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3226543068885803,
      "logits/rejected": 0.1371956765651703,
      "logps/chosen": -428.77850341796875,
      "logps/rejected": -369.66070556640625,
      "loss": 0.3167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2597908079624176,
      "rewards/margins": 1.5293853282928467,
      "rewards/rejected": -1.789176344871521,
      "step": 4935
    },
    {
      "epoch": 0.6459147787682996,
      "grad_norm": 4.351723670959473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3611747622489929,
      "logits/rejected": 0.38611912727355957,
      "logps/chosen": -363.98681640625,
      "logps/rejected": -359.72174072265625,
      "loss": 0.2522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12426149100065231,
      "rewards/margins": 1.5649820566177368,
      "rewards/rejected": -1.440720796585083,
      "step": 4936
    },
    {
      "epoch": 0.6460456367056514,
      "grad_norm": 7.612734317779541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36539706587791443,
      "logits/rejected": 0.2717244327068329,
      "logps/chosen": -531.7955932617188,
      "logps/rejected": -425.7449035644531,
      "loss": 0.481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33357468247413635,
      "rewards/margins": 1.0616508722305298,
      "rewards/rejected": -1.3952256441116333,
      "step": 4937
    },
    {
      "epoch": 0.6461764946430032,
      "grad_norm": 11.068315505981445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13181598484516144,
      "logits/rejected": -0.3234456181526184,
      "logps/chosen": -287.5348205566406,
      "logps/rejected": -319.1042785644531,
      "loss": 0.6753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9925516843795776,
      "rewards/margins": 0.25461751222610474,
      "rewards/rejected": -1.247169017791748,
      "step": 4938
    },
    {
      "epoch": 0.646307352580355,
      "grad_norm": 6.888608455657959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12926682829856873,
      "logits/rejected": 0.02449849620461464,
      "logps/chosen": -326.5675048828125,
      "logps/rejected": -314.3443298339844,
      "loss": 0.6787,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8067541718482971,
      "rewards/margins": 0.3971635699272156,
      "rewards/rejected": -1.2039177417755127,
      "step": 4939
    },
    {
      "epoch": 0.6464382105177067,
      "grad_norm": 4.963829040527344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2356783151626587,
      "logits/rejected": 0.17718368768692017,
      "logps/chosen": -557.5928344726562,
      "logps/rejected": -413.1649169921875,
      "loss": 0.3202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06338156759738922,
      "rewards/margins": 1.1701741218566895,
      "rewards/rejected": -1.233555793762207,
      "step": 4940
    },
    {
      "epoch": 0.6465690684550585,
      "grad_norm": 6.264833927154541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04646572470664978,
      "logits/rejected": -0.15180863440036774,
      "logps/chosen": -580.7398681640625,
      "logps/rejected": -437.37420654296875,
      "loss": 0.4034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13637122511863708,
      "rewards/margins": 1.0241321325302124,
      "rewards/rejected": -1.1605033874511719,
      "step": 4941
    },
    {
      "epoch": 0.6466999263924103,
      "grad_norm": 11.829997062683105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08734340965747833,
      "logits/rejected": -0.19310688972473145,
      "logps/chosen": -579.5980834960938,
      "logps/rejected": -402.183837890625,
      "loss": 0.6895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.60507732629776,
      "rewards/margins": 0.4261764585971832,
      "rewards/rejected": -1.0312538146972656,
      "step": 4942
    },
    {
      "epoch": 0.646830784329762,
      "grad_norm": 6.993394374847412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12487374991178513,
      "logits/rejected": -0.04087379574775696,
      "logps/chosen": -412.1265869140625,
      "logps/rejected": -343.1532897949219,
      "loss": 0.5315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4156153202056885,
      "rewards/margins": 0.9572859406471252,
      "rewards/rejected": -1.372901201248169,
      "step": 4943
    },
    {
      "epoch": 0.6469616422671137,
      "grad_norm": 3.579441547393799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31519615650177,
      "logits/rejected": 0.037454165518283844,
      "logps/chosen": -467.1439208984375,
      "logps/rejected": -379.7493896484375,
      "loss": 0.2081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2854541838169098,
      "rewards/margins": 2.257269859313965,
      "rewards/rejected": -1.971815586090088,
      "step": 4944
    },
    {
      "epoch": 0.6470925002044655,
      "grad_norm": 6.460052490234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15519464015960693,
      "logits/rejected": -0.16450348496437073,
      "logps/chosen": -662.3841552734375,
      "logps/rejected": -587.0689086914062,
      "loss": 0.4224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.555403470993042,
      "rewards/margins": 1.418077826499939,
      "rewards/rejected": -1.973481297492981,
      "step": 4945
    },
    {
      "epoch": 0.6472233581418173,
      "grad_norm": 5.503535270690918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00789795070886612,
      "logits/rejected": -0.15743806958198547,
      "logps/chosen": -277.4876708984375,
      "logps/rejected": -266.4935607910156,
      "loss": 0.4865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2993053197860718,
      "rewards/margins": 0.5994104146957397,
      "rewards/rejected": -0.8987157940864563,
      "step": 4946
    },
    {
      "epoch": 0.6473542160791691,
      "grad_norm": 3.7581467628479004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07138551026582718,
      "logits/rejected": -0.08549611270427704,
      "logps/chosen": -433.5835266113281,
      "logps/rejected": -314.204833984375,
      "loss": 0.2379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.021650418639183044,
      "rewards/margins": 2.1203112602233887,
      "rewards/rejected": -2.098660945892334,
      "step": 4947
    },
    {
      "epoch": 0.6474850740165208,
      "grad_norm": 8.035051345825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03576914593577385,
      "logits/rejected": -0.04767156392335892,
      "logps/chosen": -427.5349426269531,
      "logps/rejected": -519.4046630859375,
      "loss": 0.6306,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0681540966033936,
      "rewards/margins": 0.32475244998931885,
      "rewards/rejected": -1.3929065465927124,
      "step": 4948
    },
    {
      "epoch": 0.6476159319538726,
      "grad_norm": 8.99272346496582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08323973417282104,
      "logits/rejected": -0.10787525773048401,
      "logps/chosen": -442.38531494140625,
      "logps/rejected": -563.7006225585938,
      "loss": 0.6288,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.118470311164856,
      "rewards/margins": 0.6965541839599609,
      "rewards/rejected": -1.8150246143341064,
      "step": 4949
    },
    {
      "epoch": 0.6477467898912244,
      "grad_norm": 6.116940498352051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15383325517177582,
      "logits/rejected": 0.31785085797309875,
      "logps/chosen": -450.3749084472656,
      "logps/rejected": -404.76025390625,
      "loss": 0.3923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3681434690952301,
      "rewards/margins": 0.82413250207901,
      "rewards/rejected": -1.1922760009765625,
      "step": 4950
    },
    {
      "epoch": 0.6478776478285762,
      "grad_norm": 6.075835227966309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16428503394126892,
      "logits/rejected": -0.31823259592056274,
      "logps/chosen": -564.2423706054688,
      "logps/rejected": -410.302734375,
      "loss": 0.3937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004850391298532486,
      "rewards/margins": 1.3418711423873901,
      "rewards/rejected": -1.3370206356048584,
      "step": 4951
    },
    {
      "epoch": 0.6480085057659278,
      "grad_norm": 8.768804550170898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10334359109401703,
      "logits/rejected": 0.22480304539203644,
      "logps/chosen": -487.6602478027344,
      "logps/rejected": -468.7493896484375,
      "loss": 0.5664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24213391542434692,
      "rewards/margins": 0.7178499102592468,
      "rewards/rejected": -0.9599838256835938,
      "step": 4952
    },
    {
      "epoch": 0.6481393637032796,
      "grad_norm": 7.940460681915283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07225195318460464,
      "logits/rejected": 0.09157487750053406,
      "logps/chosen": -468.51409912109375,
      "logps/rejected": -501.52301025390625,
      "loss": 0.5097,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16354523599147797,
      "rewards/margins": 0.9849655032157898,
      "rewards/rejected": -0.8214203119277954,
      "step": 4953
    },
    {
      "epoch": 0.6482702216406314,
      "grad_norm": 9.32398796081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049224913120269775,
      "logits/rejected": 0.10516496002674103,
      "logps/chosen": -436.13568115234375,
      "logps/rejected": -592.2020874023438,
      "loss": 0.8856,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5696325302124023,
      "rewards/margins": 0.16653099656105042,
      "rewards/rejected": -0.7361635565757751,
      "step": 4954
    },
    {
      "epoch": 0.6484010795779831,
      "grad_norm": 7.796981334686279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19337724149227142,
      "logits/rejected": -0.17788273096084595,
      "logps/chosen": -438.4322814941406,
      "logps/rejected": -400.55755615234375,
      "loss": 0.7304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39247626066207886,
      "rewards/margins": 0.7345060706138611,
      "rewards/rejected": -1.12698233127594,
      "step": 4955
    },
    {
      "epoch": 0.6485319375153349,
      "grad_norm": 13.10626220703125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11955864727497101,
      "logits/rejected": -0.270293265581131,
      "logps/chosen": -442.7412109375,
      "logps/rejected": -533.6976928710938,
      "loss": 1.1886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6885128617286682,
      "rewards/margins": 0.2625853419303894,
      "rewards/rejected": -0.9510981440544128,
      "step": 4956
    },
    {
      "epoch": 0.6486627954526867,
      "grad_norm": 8.833305358886719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2779790163040161,
      "logits/rejected": 0.32764697074890137,
      "logps/chosen": -401.1005554199219,
      "logps/rejected": -368.5848083496094,
      "loss": 0.6709,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.33604949712753296,
      "rewards/margins": 0.8395727872848511,
      "rewards/rejected": -1.1756222248077393,
      "step": 4957
    },
    {
      "epoch": 0.6487936533900385,
      "grad_norm": 8.493285179138184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010736260563135147,
      "logits/rejected": 0.21254749596118927,
      "logps/chosen": -338.8453369140625,
      "logps/rejected": -433.78619384765625,
      "loss": 0.8079,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.041902344673871994,
      "rewards/margins": 0.4400390088558197,
      "rewards/rejected": -0.3981367349624634,
      "step": 4958
    },
    {
      "epoch": 0.6489245113273902,
      "grad_norm": 5.473093509674072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16045767068862915,
      "logits/rejected": -0.10930894315242767,
      "logps/chosen": -417.820556640625,
      "logps/rejected": -368.7027893066406,
      "loss": 0.4727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1679434776306152,
      "rewards/margins": 0.6299129724502563,
      "rewards/rejected": -1.7978564500808716,
      "step": 4959
    },
    {
      "epoch": 0.6490553692647419,
      "grad_norm": 8.680113792419434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23496773838996887,
      "logits/rejected": 0.2644748091697693,
      "logps/chosen": -604.035888671875,
      "logps/rejected": -433.5779113769531,
      "loss": 0.3459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2060108184814453,
      "rewards/margins": 1.3781673908233643,
      "rewards/rejected": -1.172156572341919,
      "step": 4960
    },
    {
      "epoch": 0.6491862272020937,
      "grad_norm": 8.021109580993652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10603252798318863,
      "logits/rejected": -0.2355666160583496,
      "logps/chosen": -550.3016357421875,
      "logps/rejected": -508.57916259765625,
      "loss": 0.595,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7889724969863892,
      "rewards/margins": 0.4560823440551758,
      "rewards/rejected": -1.2450547218322754,
      "step": 4961
    },
    {
      "epoch": 0.6493170851394455,
      "grad_norm": 6.2663798332214355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13670101761817932,
      "logits/rejected": -0.37553608417510986,
      "logps/chosen": -494.0199279785156,
      "logps/rejected": -455.41748046875,
      "loss": 0.414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3309415578842163,
      "rewards/margins": 0.9858318567276001,
      "rewards/rejected": -1.3167734146118164,
      "step": 4962
    },
    {
      "epoch": 0.6494479430767972,
      "grad_norm": 5.842372417449951,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3659011423587799,
      "logits/rejected": 0.2509883642196655,
      "logps/chosen": -611.0753784179688,
      "logps/rejected": -655.87353515625,
      "loss": 0.3393,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2291250228881836,
      "rewards/margins": 2.6580002307891846,
      "rewards/rejected": -2.428875207901001,
      "step": 4963
    },
    {
      "epoch": 0.649578801014149,
      "grad_norm": 6.678496360778809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04336598142981529,
      "logits/rejected": 0.28712743520736694,
      "logps/chosen": -359.05059814453125,
      "logps/rejected": -366.201416015625,
      "loss": 0.586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5163255929946899,
      "rewards/margins": 0.5199827551841736,
      "rewards/rejected": -1.0363082885742188,
      "step": 4964
    },
    {
      "epoch": 0.6497096589515008,
      "grad_norm": 4.908217430114746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13422006368637085,
      "logits/rejected": -0.2976459562778473,
      "logps/chosen": -408.6279296875,
      "logps/rejected": -360.181884765625,
      "loss": 0.2385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5707054138183594,
      "rewards/margins": 1.9760241508483887,
      "rewards/rejected": -2.546729564666748,
      "step": 4965
    },
    {
      "epoch": 0.6498405168888526,
      "grad_norm": 7.6127214431762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10921242833137512,
      "logits/rejected": -0.0981508418917656,
      "logps/chosen": -504.3830871582031,
      "logps/rejected": -427.2215576171875,
      "loss": 0.5667,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29676035046577454,
      "rewards/margins": 1.0596812963485718,
      "rewards/rejected": -0.7629209756851196,
      "step": 4966
    },
    {
      "epoch": 0.6499713748262043,
      "grad_norm": 4.89320707321167,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14608658850193024,
      "logits/rejected": 0.20938719809055328,
      "logps/chosen": -361.7570495605469,
      "logps/rejected": -373.57220458984375,
      "loss": 0.4045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4393385052680969,
      "rewards/margins": 1.1318107843399048,
      "rewards/rejected": -1.571149230003357,
      "step": 4967
    },
    {
      "epoch": 0.650102232763556,
      "grad_norm": 8.353947639465332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31474730372428894,
      "logits/rejected": 0.08323728293180466,
      "logps/chosen": -612.9727783203125,
      "logps/rejected": -542.1490478515625,
      "loss": 0.426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10230293869972229,
      "rewards/margins": 0.9873794913291931,
      "rewards/rejected": -1.0896824598312378,
      "step": 4968
    },
    {
      "epoch": 0.6502330907009078,
      "grad_norm": 6.704935550689697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008236156776547432,
      "logits/rejected": 0.1736387312412262,
      "logps/chosen": -590.4363403320312,
      "logps/rejected": -457.5818786621094,
      "loss": 0.368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.40527915954589844,
      "rewards/margins": 1.1314834356307983,
      "rewards/rejected": -0.7262043356895447,
      "step": 4969
    },
    {
      "epoch": 0.6503639486382596,
      "grad_norm": 8.113992691040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19074687361717224,
      "logits/rejected": 0.031629156321287155,
      "logps/chosen": -786.372802734375,
      "logps/rejected": -522.6288452148438,
      "loss": 0.3866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6426807641983032,
      "rewards/margins": 1.5442787408828735,
      "rewards/rejected": -0.9015979766845703,
      "step": 4970
    },
    {
      "epoch": 0.6504948065756113,
      "grad_norm": 4.816510200500488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5131980180740356,
      "logits/rejected": 0.344339519739151,
      "logps/chosen": -378.36822509765625,
      "logps/rejected": -268.5033264160156,
      "loss": 0.2742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14000701904296875,
      "rewards/margins": 1.6392403841018677,
      "rewards/rejected": -1.779247522354126,
      "step": 4971
    },
    {
      "epoch": 0.6506256645129631,
      "grad_norm": 11.214872360229492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3875337243080139,
      "logits/rejected": -0.40486374497413635,
      "logps/chosen": -354.79998779296875,
      "logps/rejected": -347.4294128417969,
      "loss": 0.8201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9814859628677368,
      "rewards/margins": 0.37847432494163513,
      "rewards/rejected": -1.3599601984024048,
      "step": 4972
    },
    {
      "epoch": 0.6507565224503149,
      "grad_norm": 5.519558429718018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09985274076461792,
      "logits/rejected": -0.1311778426170349,
      "logps/chosen": -400.5791320800781,
      "logps/rejected": -262.83544921875,
      "loss": 0.3535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8631929755210876,
      "rewards/margins": 1.4573941230773926,
      "rewards/rejected": -2.320586919784546,
      "step": 4973
    },
    {
      "epoch": 0.6508873803876667,
      "grad_norm": 4.454111576080322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013917064294219017,
      "logits/rejected": -0.09148664772510529,
      "logps/chosen": -423.6899108886719,
      "logps/rejected": -315.2453308105469,
      "loss": 0.2929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2168048918247223,
      "rewards/margins": 1.453755259513855,
      "rewards/rejected": -1.6705601215362549,
      "step": 4974
    },
    {
      "epoch": 0.6510182383250184,
      "grad_norm": 6.756706237792969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23957601189613342,
      "logits/rejected": 0.1669168919324875,
      "logps/chosen": -534.9993286132812,
      "logps/rejected": -554.1360473632812,
      "loss": 0.5161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.48631787300109863,
      "rewards/margins": 1.127724528312683,
      "rewards/rejected": -0.6414066553115845,
      "step": 4975
    },
    {
      "epoch": 0.6511490962623702,
      "grad_norm": 3.9698832035064697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24736490845680237,
      "logits/rejected": 0.2400965392589569,
      "logps/chosen": -745.1722412109375,
      "logps/rejected": -383.4731750488281,
      "loss": 0.1656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45431748032569885,
      "rewards/margins": 2.318868637084961,
      "rewards/rejected": -1.864551305770874,
      "step": 4976
    },
    {
      "epoch": 0.651279954199722,
      "grad_norm": 7.810396194458008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11042055487632751,
      "logits/rejected": -0.3534824550151825,
      "logps/chosen": -508.6643981933594,
      "logps/rejected": -645.391845703125,
      "loss": 0.7017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8240315318107605,
      "rewards/margins": 0.5596320629119873,
      "rewards/rejected": -1.383663535118103,
      "step": 4977
    },
    {
      "epoch": 0.6514108121370736,
      "grad_norm": 7.037626266479492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37127941846847534,
      "logits/rejected": 0.34584924578666687,
      "logps/chosen": -413.3845520019531,
      "logps/rejected": -364.3277587890625,
      "loss": 0.3993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.44053757190704346,
      "rewards/margins": 1.2215956449508667,
      "rewards/rejected": -1.6621332168579102,
      "step": 4978
    },
    {
      "epoch": 0.6515416700744254,
      "grad_norm": 8.527681350708008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0009456649422645569,
      "logits/rejected": 0.375771164894104,
      "logps/chosen": -434.8135681152344,
      "logps/rejected": -460.92877197265625,
      "loss": 0.72,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4953860640525818,
      "rewards/margins": 0.5646942853927612,
      "rewards/rejected": -1.0600804090499878,
      "step": 4979
    },
    {
      "epoch": 0.6516725280117772,
      "grad_norm": 8.598226547241211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34576404094696045,
      "logits/rejected": -0.3615407943725586,
      "logps/chosen": -374.8606262207031,
      "logps/rejected": -651.794189453125,
      "loss": 0.4304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10450232028961182,
      "rewards/margins": 1.0316082239151,
      "rewards/rejected": -1.136110544204712,
      "step": 4980
    },
    {
      "epoch": 0.651803385949129,
      "grad_norm": 7.251986980438232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23632371425628662,
      "logits/rejected": -0.3405747711658478,
      "logps/chosen": -287.7615051269531,
      "logps/rejected": -322.11346435546875,
      "loss": 0.7046,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.832668125629425,
      "rewards/margins": 0.24242857098579407,
      "rewards/rejected": -1.075096845626831,
      "step": 4981
    },
    {
      "epoch": 0.6519342438864807,
      "grad_norm": 6.923449993133545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1699526458978653,
      "logits/rejected": -0.002927641849964857,
      "logps/chosen": -374.6693420410156,
      "logps/rejected": -390.5314636230469,
      "loss": 0.4666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13062839210033417,
      "rewards/margins": 1.009735345840454,
      "rewards/rejected": -0.879106879234314,
      "step": 4982
    },
    {
      "epoch": 0.6520651018238325,
      "grad_norm": 9.507596015930176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11440357565879822,
      "logits/rejected": 0.12349984049797058,
      "logps/chosen": -481.30328369140625,
      "logps/rejected": -533.6802978515625,
      "loss": 0.8358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08648835122585297,
      "rewards/margins": 0.702675998210907,
      "rewards/rejected": -0.7891643643379211,
      "step": 4983
    },
    {
      "epoch": 0.6521959597611843,
      "grad_norm": 8.302001953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26195505261421204,
      "logits/rejected": 0.34714388847351074,
      "logps/chosen": -487.59735107421875,
      "logps/rejected": -477.0498962402344,
      "loss": 0.7064,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.46610355377197266,
      "rewards/margins": 0.3682260811328888,
      "rewards/rejected": -0.8343296647071838,
      "step": 4984
    },
    {
      "epoch": 0.6523268176985361,
      "grad_norm": 7.650428771972656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14974279701709747,
      "logits/rejected": 0.1649174839258194,
      "logps/chosen": -475.85791015625,
      "logps/rejected": -619.1484375,
      "loss": 0.5576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13287794589996338,
      "rewards/margins": 0.48903483152389526,
      "rewards/rejected": -0.6219127178192139,
      "step": 4985
    },
    {
      "epoch": 0.6524576756358877,
      "grad_norm": 7.485235214233398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27216726541519165,
      "logits/rejected": 0.22414439916610718,
      "logps/chosen": -463.6277770996094,
      "logps/rejected": -353.02923583984375,
      "loss": 0.5012,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12185001373291016,
      "rewards/margins": 1.3729419708251953,
      "rewards/rejected": -1.4947919845581055,
      "step": 4986
    },
    {
      "epoch": 0.6525885335732395,
      "grad_norm": 5.460172653198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041134193539619446,
      "logits/rejected": -0.029404670000076294,
      "logps/chosen": -422.1152648925781,
      "logps/rejected": -337.6467590332031,
      "loss": 0.4664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5414884686470032,
      "rewards/margins": 0.7095870971679688,
      "rewards/rejected": -1.2510755062103271,
      "step": 4987
    },
    {
      "epoch": 0.6527193915105913,
      "grad_norm": 4.405059337615967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044842325150966644,
      "logits/rejected": -0.09142785519361496,
      "logps/chosen": -397.23760986328125,
      "logps/rejected": -309.92388916015625,
      "loss": 0.2948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.262808620929718,
      "rewards/margins": 1.3222042322158813,
      "rewards/rejected": -1.5850127935409546,
      "step": 4988
    },
    {
      "epoch": 0.6528502494479431,
      "grad_norm": 7.081340789794922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13442109525203705,
      "logits/rejected": -0.1438790112733841,
      "logps/chosen": -632.9273681640625,
      "logps/rejected": -526.4813232421875,
      "loss": 0.4588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.055724136531353,
      "rewards/margins": 1.42108154296875,
      "rewards/rejected": -1.3653573989868164,
      "step": 4989
    },
    {
      "epoch": 0.6529811073852948,
      "grad_norm": 10.199420928955078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13542720675468445,
      "logits/rejected": -0.18660429120063782,
      "logps/chosen": -550.87451171875,
      "logps/rejected": -634.6231689453125,
      "loss": 0.6365,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9362659454345703,
      "rewards/margins": 0.5407568216323853,
      "rewards/rejected": -1.4770227670669556,
      "step": 4990
    },
    {
      "epoch": 0.6531119653226466,
      "grad_norm": 8.626315116882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23876212537288666,
      "logits/rejected": -0.23262086510658264,
      "logps/chosen": -356.2588806152344,
      "logps/rejected": -686.3125610351562,
      "loss": 0.5694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7140564322471619,
      "rewards/margins": 0.6532342433929443,
      "rewards/rejected": -1.3672906160354614,
      "step": 4991
    },
    {
      "epoch": 0.6532428232599984,
      "grad_norm": 4.614048957824707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03963862359523773,
      "logits/rejected": -0.11657688021659851,
      "logps/chosen": -451.2161560058594,
      "logps/rejected": -500.84039306640625,
      "loss": 0.3461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2961982786655426,
      "rewards/margins": 1.933704137802124,
      "rewards/rejected": -2.229902744293213,
      "step": 4992
    },
    {
      "epoch": 0.6533736811973502,
      "grad_norm": 7.638701438903809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10248342156410217,
      "logits/rejected": -0.23718298971652985,
      "logps/chosen": -432.67169189453125,
      "logps/rejected": -389.8118896484375,
      "loss": 0.5674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.048585906624794006,
      "rewards/margins": 0.7057230472564697,
      "rewards/rejected": -0.7543088793754578,
      "step": 4993
    },
    {
      "epoch": 0.6535045391347019,
      "grad_norm": 6.191092491149902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16338181495666504,
      "logits/rejected": -0.12655258178710938,
      "logps/chosen": -552.3626098632812,
      "logps/rejected": -495.5003356933594,
      "loss": 0.4482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5437170267105103,
      "rewards/margins": 0.9668489694595337,
      "rewards/rejected": -1.510565996170044,
      "step": 4994
    },
    {
      "epoch": 0.6536353970720536,
      "grad_norm": 5.202304363250732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07964806258678436,
      "logits/rejected": 0.07619085162878036,
      "logps/chosen": -480.51708984375,
      "logps/rejected": -456.43597412109375,
      "loss": 0.2784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015400496311485767,
      "rewards/margins": 1.4135539531707764,
      "rewards/rejected": -1.4289543628692627,
      "step": 4995
    },
    {
      "epoch": 0.6537662550094054,
      "grad_norm": 4.423934459686279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20067757368087769,
      "logits/rejected": -0.25860241055488586,
      "logps/chosen": -324.5140380859375,
      "logps/rejected": -373.90875244140625,
      "loss": 0.3222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4074968099594116,
      "rewards/margins": 1.3306164741516113,
      "rewards/rejected": -1.738113284111023,
      "step": 4996
    },
    {
      "epoch": 0.6538971129467572,
      "grad_norm": 7.47042989730835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04358585923910141,
      "logits/rejected": 0.04484491050243378,
      "logps/chosen": -491.67779541015625,
      "logps/rejected": -495.8304443359375,
      "loss": 0.4165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38233986496925354,
      "rewards/margins": 1.2602789402008057,
      "rewards/rejected": -0.8779390454292297,
      "step": 4997
    },
    {
      "epoch": 0.6540279708841089,
      "grad_norm": 6.003218650817871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11992567777633667,
      "logits/rejected": 0.014589117839932442,
      "logps/chosen": -397.2207336425781,
      "logps/rejected": -493.6124267578125,
      "loss": 0.3619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33901023864746094,
      "rewards/margins": 2.120762348175049,
      "rewards/rejected": -2.4597725868225098,
      "step": 4998
    },
    {
      "epoch": 0.6541588288214607,
      "grad_norm": 10.590201377868652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09832771122455597,
      "logits/rejected": 0.2550082802772522,
      "logps/chosen": -408.68829345703125,
      "logps/rejected": -408.8728942871094,
      "loss": 0.7155,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1762937307357788,
      "rewards/margins": 0.5386118292808533,
      "rewards/rejected": -1.7149056196212769,
      "step": 4999
    },
    {
      "epoch": 0.6542896867588125,
      "grad_norm": 8.44347858428955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12803597748279572,
      "logits/rejected": 0.07928255200386047,
      "logps/chosen": -360.3609313964844,
      "logps/rejected": -379.94122314453125,
      "loss": 0.6351,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1690473556518555,
      "rewards/margins": 0.960092306137085,
      "rewards/rejected": -2.1291399002075195,
      "step": 5000
    },
    {
      "epoch": 0.6544205446961642,
      "grad_norm": 7.470067501068115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009154529310762882,
      "logits/rejected": 0.23473532497882843,
      "logps/chosen": -329.23455810546875,
      "logps/rejected": -384.6049499511719,
      "loss": 0.5518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5881273150444031,
      "rewards/margins": 0.6018335223197937,
      "rewards/rejected": -1.1899609565734863,
      "step": 5001
    },
    {
      "epoch": 0.654551402633516,
      "grad_norm": 7.885444641113281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03124932385981083,
      "logits/rejected": 0.0766211524605751,
      "logps/chosen": -574.598388671875,
      "logps/rejected": -468.8419189453125,
      "loss": 0.5382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8991461396217346,
      "rewards/margins": 0.9599124193191528,
      "rewards/rejected": -1.8590584993362427,
      "step": 5002
    },
    {
      "epoch": 0.6546822605708678,
      "grad_norm": 8.647966384887695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21539923548698425,
      "logits/rejected": 0.3641834557056427,
      "logps/chosen": -452.2823486328125,
      "logps/rejected": -505.785888671875,
      "loss": 0.556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2391367107629776,
      "rewards/margins": 1.3633800745010376,
      "rewards/rejected": -1.6025166511535645,
      "step": 5003
    },
    {
      "epoch": 0.6548131185082195,
      "grad_norm": 7.653145790100098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09201759099960327,
      "logits/rejected": 0.13870404660701752,
      "logps/chosen": -421.925048828125,
      "logps/rejected": -377.9512634277344,
      "loss": 0.4305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5854171514511108,
      "rewards/margins": 1.3817976713180542,
      "rewards/rejected": -1.967214822769165,
      "step": 5004
    },
    {
      "epoch": 0.6549439764455712,
      "grad_norm": 9.299163818359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12847349047660828,
      "logits/rejected": 0.2806047797203064,
      "logps/chosen": -620.5225830078125,
      "logps/rejected": -627.6314086914062,
      "loss": 0.5481,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3742753863334656,
      "rewards/margins": 0.5800039768218994,
      "rewards/rejected": -0.9542793035507202,
      "step": 5005
    },
    {
      "epoch": 0.655074834382923,
      "grad_norm": 12.031471252441406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4198640286922455,
      "logits/rejected": 0.13354678452014923,
      "logps/chosen": -481.1265869140625,
      "logps/rejected": -348.08905029296875,
      "loss": 0.8533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1255338191986084,
      "rewards/margins": 0.112967848777771,
      "rewards/rejected": -1.238501787185669,
      "step": 5006
    },
    {
      "epoch": 0.6552056923202748,
      "grad_norm": 4.57749605178833,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01800062507390976,
      "logits/rejected": 0.05956931412220001,
      "logps/chosen": -304.2225341796875,
      "logps/rejected": -312.14068603515625,
      "loss": 0.3631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.576011061668396,
      "rewards/margins": 1.3257696628570557,
      "rewards/rejected": -1.9017808437347412,
      "step": 5007
    },
    {
      "epoch": 0.6553365502576266,
      "grad_norm": 9.532305717468262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17472968995571136,
      "logits/rejected": -0.20846790075302124,
      "logps/chosen": -581.4095458984375,
      "logps/rejected": -439.81640625,
      "loss": 0.5645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3259931802749634,
      "rewards/margins": 1.0055880546569824,
      "rewards/rejected": -1.3315812349319458,
      "step": 5008
    },
    {
      "epoch": 0.6554674081949783,
      "grad_norm": 7.507796764373779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10816585272550583,
      "logits/rejected": 0.07105880230665207,
      "logps/chosen": -450.7745666503906,
      "logps/rejected": -520.1558837890625,
      "loss": 0.593,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.917201042175293,
      "rewards/margins": 0.4936767816543579,
      "rewards/rejected": -1.4108779430389404,
      "step": 5009
    },
    {
      "epoch": 0.6555982661323301,
      "grad_norm": 7.303613185882568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2678520679473877,
      "logits/rejected": 0.2532750964164734,
      "logps/chosen": -510.3255310058594,
      "logps/rejected": -403.50933837890625,
      "loss": 0.4285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19537514448165894,
      "rewards/margins": 1.4006404876708984,
      "rewards/rejected": -1.5960155725479126,
      "step": 5010
    },
    {
      "epoch": 0.6557291240696819,
      "grad_norm": 6.403043746948242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02488735318183899,
      "logits/rejected": -0.06397467851638794,
      "logps/chosen": -361.8653259277344,
      "logps/rejected": -324.43121337890625,
      "loss": 0.402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20091934502124786,
      "rewards/margins": 1.3989591598510742,
      "rewards/rejected": -1.5998784303665161,
      "step": 5011
    },
    {
      "epoch": 0.6558599820070337,
      "grad_norm": 6.5778374671936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18590877950191498,
      "logits/rejected": -0.18301083147525787,
      "logps/chosen": -381.3138427734375,
      "logps/rejected": -410.0979919433594,
      "loss": 0.6697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46141988039016724,
      "rewards/margins": 0.7910377383232117,
      "rewards/rejected": -1.2524577379226685,
      "step": 5012
    },
    {
      "epoch": 0.6559908399443853,
      "grad_norm": 10.317097663879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17671188712120056,
      "logits/rejected": 0.24253937602043152,
      "logps/chosen": -483.509765625,
      "logps/rejected": -499.4701843261719,
      "loss": 0.7727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.606889545917511,
      "rewards/margins": 0.41042619943618774,
      "rewards/rejected": -1.0173157453536987,
      "step": 5013
    },
    {
      "epoch": 0.6561216978817371,
      "grad_norm": 6.581751346588135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06841762363910675,
      "logits/rejected": -0.145472913980484,
      "logps/chosen": -462.0390625,
      "logps/rejected": -362.62115478515625,
      "loss": 0.6091,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.719726026058197,
      "rewards/margins": 0.9313698410987854,
      "rewards/rejected": -1.6510958671569824,
      "step": 5014
    },
    {
      "epoch": 0.6562525558190889,
      "grad_norm": 8.443877220153809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12480603903532028,
      "logits/rejected": -0.10251551866531372,
      "logps/chosen": -357.4201965332031,
      "logps/rejected": -453.661865234375,
      "loss": 0.7443,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6982333660125732,
      "rewards/margins": 0.07676677405834198,
      "rewards/rejected": -0.7750001549720764,
      "step": 5015
    },
    {
      "epoch": 0.6563834137564407,
      "grad_norm": 9.563657760620117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09854519367218018,
      "logits/rejected": 0.21758446097373962,
      "logps/chosen": -443.28265380859375,
      "logps/rejected": -344.47698974609375,
      "loss": 0.7683,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.910044252872467,
      "rewards/margins": 0.20255127549171448,
      "rewards/rejected": -1.1125954389572144,
      "step": 5016
    },
    {
      "epoch": 0.6565142716937924,
      "grad_norm": 6.547821998596191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14754921197891235,
      "logits/rejected": -0.21373823285102844,
      "logps/chosen": -397.6826477050781,
      "logps/rejected": -325.6393737792969,
      "loss": 0.4653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.015688419342041,
      "rewards/margins": 0.8604768514633179,
      "rewards/rejected": -1.8761653900146484,
      "step": 5017
    },
    {
      "epoch": 0.6566451296311442,
      "grad_norm": 8.419692039489746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17613160610198975,
      "logits/rejected": -0.06330643594264984,
      "logps/chosen": -414.6762390136719,
      "logps/rejected": -443.44683837890625,
      "loss": 0.6424,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48655301332473755,
      "rewards/margins": 0.8104596138000488,
      "rewards/rejected": -1.2970126867294312,
      "step": 5018
    },
    {
      "epoch": 0.656775987568496,
      "grad_norm": 6.268312454223633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017239093780517578,
      "logits/rejected": -0.012686364352703094,
      "logps/chosen": -528.14697265625,
      "logps/rejected": -604.5487060546875,
      "loss": 0.3899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28051623702049255,
      "rewards/margins": 1.161271095275879,
      "rewards/rejected": -1.4417873620986938,
      "step": 5019
    },
    {
      "epoch": 0.6569068455058478,
      "grad_norm": 8.271976470947266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09849043190479279,
      "logits/rejected": -0.08287634700536728,
      "logps/chosen": -497.5229187011719,
      "logps/rejected": -394.5395202636719,
      "loss": 0.5849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4703039824962616,
      "rewards/margins": 0.2635427415370941,
      "rewards/rejected": -0.7338466048240662,
      "step": 5020
    },
    {
      "epoch": 0.6570377034431995,
      "grad_norm": 9.409354209899902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19168037176132202,
      "logits/rejected": 0.20979882776737213,
      "logps/chosen": -315.02703857421875,
      "logps/rejected": -369.3271484375,
      "loss": 1.0411,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.060807228088379,
      "rewards/margins": -0.3887298107147217,
      "rewards/rejected": -0.6720774173736572,
      "step": 5021
    },
    {
      "epoch": 0.6571685613805512,
      "grad_norm": 5.048542499542236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019494421780109406,
      "logits/rejected": 0.23452383279800415,
      "logps/chosen": -327.95050048828125,
      "logps/rejected": -322.02252197265625,
      "loss": 0.3484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30668070912361145,
      "rewards/margins": 1.1991643905639648,
      "rewards/rejected": -1.5058451890945435,
      "step": 5022
    },
    {
      "epoch": 0.657299419317903,
      "grad_norm": 7.401302337646484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1042337641119957,
      "logits/rejected": -0.1138683557510376,
      "logps/chosen": -490.6870422363281,
      "logps/rejected": -504.1589660644531,
      "loss": 0.6781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9331039190292358,
      "rewards/margins": 0.3814605474472046,
      "rewards/rejected": -1.3145644664764404,
      "step": 5023
    },
    {
      "epoch": 0.6574302772552547,
      "grad_norm": 8.675261497497559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11892532557249069,
      "logits/rejected": -0.09257741272449493,
      "logps/chosen": -274.63397216796875,
      "logps/rejected": -325.7501525878906,
      "loss": 0.8449,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2547792196273804,
      "rewards/margins": 0.3796391785144806,
      "rewards/rejected": -1.6344183683395386,
      "step": 5024
    },
    {
      "epoch": 0.6575611351926065,
      "grad_norm": 5.355310916900635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1861295998096466,
      "logits/rejected": -0.2131250500679016,
      "logps/chosen": -430.75408935546875,
      "logps/rejected": -383.35589599609375,
      "loss": 0.4275,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8073093891143799,
      "rewards/margins": 1.1237930059432983,
      "rewards/rejected": -1.9311023950576782,
      "step": 5025
    },
    {
      "epoch": 0.6576919931299583,
      "grad_norm": 8.6324462890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03611943870782852,
      "logits/rejected": -0.05993713438510895,
      "logps/chosen": -338.434814453125,
      "logps/rejected": -516.4411010742188,
      "loss": 0.7359,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2173303365707397,
      "rewards/margins": 0.42766308784484863,
      "rewards/rejected": -1.644993543624878,
      "step": 5026
    },
    {
      "epoch": 0.6578228510673101,
      "grad_norm": 9.67179012298584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008081663399934769,
      "logits/rejected": 0.2510736882686615,
      "logps/chosen": -266.20703125,
      "logps/rejected": -382.8333740234375,
      "loss": 0.7911,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.9756830930709839,
      "rewards/margins": 0.10388702154159546,
      "rewards/rejected": -1.0795700550079346,
      "step": 5027
    },
    {
      "epoch": 0.6579537090046618,
      "grad_norm": 5.79713773727417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08969302475452423,
      "logits/rejected": 0.16593524813652039,
      "logps/chosen": -409.5109558105469,
      "logps/rejected": -377.07025146484375,
      "loss": 0.4256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27687129378318787,
      "rewards/margins": 0.9788988828659058,
      "rewards/rejected": -0.7020275592803955,
      "step": 5028
    },
    {
      "epoch": 0.6580845669420136,
      "grad_norm": 5.039716720581055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3733829855918884,
      "logits/rejected": 0.2950241267681122,
      "logps/chosen": -363.7823181152344,
      "logps/rejected": -333.05950927734375,
      "loss": 0.3475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40099677443504333,
      "rewards/margins": 1.1829636096954346,
      "rewards/rejected": -1.5839604139328003,
      "step": 5029
    },
    {
      "epoch": 0.6582154248793654,
      "grad_norm": 8.1222562789917,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0452936515212059,
      "logits/rejected": 0.1698281615972519,
      "logps/chosen": -552.182373046875,
      "logps/rejected": -497.6002197265625,
      "loss": 0.5628,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5283148288726807,
      "rewards/margins": 0.5957910418510437,
      "rewards/rejected": -1.1241058111190796,
      "step": 5030
    },
    {
      "epoch": 0.6583462828167171,
      "grad_norm": 6.591593265533447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22318050265312195,
      "logits/rejected": -0.006982855498790741,
      "logps/chosen": -414.23095703125,
      "logps/rejected": -453.9215087890625,
      "loss": 0.3405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9498769044876099,
      "rewards/margins": 1.5991114377975464,
      "rewards/rejected": -2.5489885807037354,
      "step": 5031
    },
    {
      "epoch": 0.6584771407540688,
      "grad_norm": 6.33543586730957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11903294175863266,
      "logits/rejected": 0.13100463151931763,
      "logps/chosen": -483.7342834472656,
      "logps/rejected": -475.30712890625,
      "loss": 0.445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3295181393623352,
      "rewards/margins": 1.2053117752075195,
      "rewards/rejected": -1.5348299741744995,
      "step": 5032
    },
    {
      "epoch": 0.6586079986914206,
      "grad_norm": 6.1751556396484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2657299339771271,
      "logits/rejected": 0.06796190142631531,
      "logps/chosen": -420.4084777832031,
      "logps/rejected": -526.5450439453125,
      "loss": 0.3658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7345991134643555,
      "rewards/margins": 1.9841777086257935,
      "rewards/rejected": -2.7187767028808594,
      "step": 5033
    },
    {
      "epoch": 0.6587388566287724,
      "grad_norm": 11.345915794372559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17990881204605103,
      "logits/rejected": -0.1563257873058319,
      "logps/chosen": -500.8890075683594,
      "logps/rejected": -493.2445983886719,
      "loss": 0.8562,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2639501094818115,
      "rewards/margins": 0.07916218787431717,
      "rewards/rejected": -1.3431124687194824,
      "step": 5034
    },
    {
      "epoch": 0.6588697145661242,
      "grad_norm": 7.403372764587402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14112532138824463,
      "logits/rejected": 0.14425352215766907,
      "logps/chosen": -415.60089111328125,
      "logps/rejected": -464.2763671875,
      "loss": 0.4162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7763140201568604,
      "rewards/margins": 0.902204692363739,
      "rewards/rejected": -1.6785187721252441,
      "step": 5035
    },
    {
      "epoch": 0.6590005725034759,
      "grad_norm": 9.619603157043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2769140601158142,
      "logits/rejected": -0.19701406359672546,
      "logps/chosen": -211.41329956054688,
      "logps/rejected": -194.9893341064453,
      "loss": 0.9074,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.277867317199707,
      "rewards/margins": 0.03437995910644531,
      "rewards/rejected": -1.3122472763061523,
      "step": 5036
    },
    {
      "epoch": 0.6591314304408277,
      "grad_norm": 8.434502601623535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2672794461250305,
      "logits/rejected": 0.18368546664714813,
      "logps/chosen": -345.93017578125,
      "logps/rejected": -398.96429443359375,
      "loss": 0.5846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.757023811340332,
      "rewards/margins": 0.9522877335548401,
      "rewards/rejected": -1.709311604499817,
      "step": 5037
    },
    {
      "epoch": 0.6592622883781795,
      "grad_norm": 5.733286380767822,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09869281947612762,
      "logits/rejected": 0.022604744881391525,
      "logps/chosen": -422.27862548828125,
      "logps/rejected": -550.1709594726562,
      "loss": 0.2717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39908546209335327,
      "rewards/margins": 1.6120810508728027,
      "rewards/rejected": -2.011166572570801,
      "step": 5038
    },
    {
      "epoch": 0.6593931463155313,
      "grad_norm": 3.8604321479797363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0005377717316150665,
      "logits/rejected": 0.10671520978212357,
      "logps/chosen": -482.70220947265625,
      "logps/rejected": -398.03485107421875,
      "loss": 0.3282,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29715844988822937,
      "rewards/margins": 1.8110111951828003,
      "rewards/rejected": -2.1081697940826416,
      "step": 5039
    },
    {
      "epoch": 0.6595240042528829,
      "grad_norm": 7.200839042663574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3404155373573303,
      "logits/rejected": -0.0009500365704298019,
      "logps/chosen": -568.2637939453125,
      "logps/rejected": -427.84197998046875,
      "loss": 0.557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6284996271133423,
      "rewards/margins": 0.945660412311554,
      "rewards/rejected": -1.574160099029541,
      "step": 5040
    },
    {
      "epoch": 0.6596548621902347,
      "grad_norm": 8.537737846374512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.077259361743927,
      "logits/rejected": -0.058584004640579224,
      "logps/chosen": -588.4937133789062,
      "logps/rejected": -704.4657592773438,
      "loss": 0.6507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6865863800048828,
      "rewards/margins": 0.35139256715774536,
      "rewards/rejected": -1.0379788875579834,
      "step": 5041
    },
    {
      "epoch": 0.6597857201275865,
      "grad_norm": 11.008075714111328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2801066040992737,
      "logits/rejected": -0.0329948291182518,
      "logps/chosen": -693.6336059570312,
      "logps/rejected": -588.6931762695312,
      "loss": 0.797,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6703975200653076,
      "rewards/margins": 0.3776455521583557,
      "rewards/rejected": -1.048043131828308,
      "step": 5042
    },
    {
      "epoch": 0.6599165780649382,
      "grad_norm": 4.086154460906982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1247115433216095,
      "logits/rejected": -0.18165525794029236,
      "logps/chosen": -223.98973083496094,
      "logps/rejected": -254.00540161132812,
      "loss": 0.4147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6163593530654907,
      "rewards/margins": 1.047019362449646,
      "rewards/rejected": -1.6633784770965576,
      "step": 5043
    },
    {
      "epoch": 0.66004743600229,
      "grad_norm": 6.052452087402344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2741931974887848,
      "logits/rejected": 0.10230862349271774,
      "logps/chosen": -423.5990295410156,
      "logps/rejected": -409.6327819824219,
      "loss": 0.3821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41689950227737427,
      "rewards/margins": 1.4420148134231567,
      "rewards/rejected": -1.8589143753051758,
      "step": 5044
    },
    {
      "epoch": 0.6601782939396418,
      "grad_norm": 5.467554569244385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04764603450894356,
      "logits/rejected": -0.05185244232416153,
      "logps/chosen": -315.2873840332031,
      "logps/rejected": -408.9903869628906,
      "loss": 0.4237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5752414464950562,
      "rewards/margins": 1.2721248865127563,
      "rewards/rejected": -1.8473663330078125,
      "step": 5045
    },
    {
      "epoch": 0.6603091518769936,
      "grad_norm": 11.072876930236816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048536088317632675,
      "logits/rejected": -0.16560186445713043,
      "logps/chosen": -397.5538330078125,
      "logps/rejected": -498.46844482421875,
      "loss": 0.8331,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5389529466629028,
      "rewards/margins": 1.47421133518219,
      "rewards/rejected": -3.013164758682251,
      "step": 5046
    },
    {
      "epoch": 0.6604400098143453,
      "grad_norm": 5.952875137329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05456456542015076,
      "logits/rejected": -0.021061144769191742,
      "logps/chosen": -439.890380859375,
      "logps/rejected": -373.3268737792969,
      "loss": 0.3252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3089412748813629,
      "rewards/margins": 1.2769883871078491,
      "rewards/rejected": -1.5859296321868896,
      "step": 5047
    },
    {
      "epoch": 0.660570867751697,
      "grad_norm": 10.387556076049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1795676052570343,
      "logits/rejected": -0.23569078743457794,
      "logps/chosen": -509.4472351074219,
      "logps/rejected": -549.1028442382812,
      "loss": 0.6764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9525127410888672,
      "rewards/margins": 1.016247272491455,
      "rewards/rejected": -1.9687601327896118,
      "step": 5048
    },
    {
      "epoch": 0.6607017256890488,
      "grad_norm": 8.114564895629883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12626966834068298,
      "logits/rejected": 0.11024132370948792,
      "logps/chosen": -343.33026123046875,
      "logps/rejected": -456.22021484375,
      "loss": 0.5003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9525209665298462,
      "rewards/margins": 0.7773397564888,
      "rewards/rejected": -1.7298606634140015,
      "step": 5049
    },
    {
      "epoch": 0.6608325836264006,
      "grad_norm": 8.628814697265625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26401448249816895,
      "logits/rejected": 0.12904560565948486,
      "logps/chosen": -495.43035888671875,
      "logps/rejected": -491.10052490234375,
      "loss": 0.5197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8300005793571472,
      "rewards/margins": 0.7240476608276367,
      "rewards/rejected": -1.5540481805801392,
      "step": 5050
    },
    {
      "epoch": 0.6609634415637523,
      "grad_norm": 6.530557632446289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08101207762956619,
      "logits/rejected": 0.23415657877922058,
      "logps/chosen": -478.6134338378906,
      "logps/rejected": -419.2781982421875,
      "loss": 0.4423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9083985090255737,
      "rewards/margins": 0.9707814455032349,
      "rewards/rejected": -1.879179835319519,
      "step": 5051
    },
    {
      "epoch": 0.6610942995011041,
      "grad_norm": 5.4066338539123535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.059421606361866,
      "logits/rejected": -0.01158859208226204,
      "logps/chosen": -549.8047485351562,
      "logps/rejected": -484.2199401855469,
      "loss": 0.2798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7692155838012695,
      "rewards/margins": 1.2822554111480713,
      "rewards/rejected": -2.05147123336792,
      "step": 5052
    },
    {
      "epoch": 0.6612251574384559,
      "grad_norm": 4.261619567871094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27843713760375977,
      "logits/rejected": 0.044061653316020966,
      "logps/chosen": -513.763427734375,
      "logps/rejected": -472.3097229003906,
      "loss": 0.2528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3781505525112152,
      "rewards/margins": 2.226017713546753,
      "rewards/rejected": -2.604168176651001,
      "step": 5053
    },
    {
      "epoch": 0.6613560153758077,
      "grad_norm": 3.7974936962127686,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23848123848438263,
      "logits/rejected": -0.2513332664966583,
      "logps/chosen": -392.4406433105469,
      "logps/rejected": -395.292236328125,
      "loss": 0.1743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12970180809497833,
      "rewards/margins": 2.0522186756134033,
      "rewards/rejected": -2.1819205284118652,
      "step": 5054
    },
    {
      "epoch": 0.6614868733131594,
      "grad_norm": 6.193364143371582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2516010105609894,
      "logits/rejected": -0.038229551166296005,
      "logps/chosen": -360.64361572265625,
      "logps/rejected": -346.62689208984375,
      "loss": 0.4818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9321922659873962,
      "rewards/margins": 0.8686927556991577,
      "rewards/rejected": -1.8008849620819092,
      "step": 5055
    },
    {
      "epoch": 0.6616177312505112,
      "grad_norm": 6.576940536499023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0019478872418403625,
      "logits/rejected": -0.088841512799263,
      "logps/chosen": -605.551513671875,
      "logps/rejected": -370.57611083984375,
      "loss": 0.4709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43771669268608093,
      "rewards/margins": 1.2143375873565674,
      "rewards/rejected": -1.6520540714263916,
      "step": 5056
    },
    {
      "epoch": 0.661748589187863,
      "grad_norm": 3.142066478729248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10909191519021988,
      "logits/rejected": -0.15258191525936127,
      "logps/chosen": -505.69830322265625,
      "logps/rejected": -424.9467468261719,
      "loss": 0.3048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.025588631629943848,
      "rewards/margins": 1.7377585172653198,
      "rewards/rejected": -1.7121697664260864,
      "step": 5057
    },
    {
      "epoch": 0.6618794471252147,
      "grad_norm": 11.080619812011719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03134618699550629,
      "logits/rejected": -0.29953503608703613,
      "logps/chosen": -394.9761657714844,
      "logps/rejected": -456.7674560546875,
      "loss": 0.7464,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.536550760269165,
      "rewards/margins": 0.4785843789577484,
      "rewards/rejected": -2.0151352882385254,
      "step": 5058
    },
    {
      "epoch": 0.6620103050625664,
      "grad_norm": 9.97709846496582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1354934275150299,
      "logits/rejected": -0.0004322472959756851,
      "logps/chosen": -394.13836669921875,
      "logps/rejected": -416.9833984375,
      "loss": 0.7748,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2621113061904907,
      "rewards/margins": 0.25470662117004395,
      "rewards/rejected": -1.5168179273605347,
      "step": 5059
    },
    {
      "epoch": 0.6621411629999182,
      "grad_norm": 8.356001853942871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05475550889968872,
      "logits/rejected": 0.008539550006389618,
      "logps/chosen": -450.918701171875,
      "logps/rejected": -383.17291259765625,
      "loss": 0.67,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7935051321983337,
      "rewards/margins": 0.21634657680988312,
      "rewards/rejected": -1.0098516941070557,
      "step": 5060
    },
    {
      "epoch": 0.66227202093727,
      "grad_norm": 5.878708839416504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13429777324199677,
      "logits/rejected": -0.09857359528541565,
      "logps/chosen": -348.2930603027344,
      "logps/rejected": -356.9683837890625,
      "loss": 0.4631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4021310806274414,
      "rewards/margins": 1.5935373306274414,
      "rewards/rejected": -1.9956684112548828,
      "step": 5061
    },
    {
      "epoch": 0.6624028788746218,
      "grad_norm": 4.427967548370361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3221031427383423,
      "logits/rejected": 0.04077883064746857,
      "logps/chosen": -663.34521484375,
      "logps/rejected": -786.6638793945312,
      "loss": 0.1592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09694422036409378,
      "rewards/margins": 2.2483108043670654,
      "rewards/rejected": -2.345255136489868,
      "step": 5062
    },
    {
      "epoch": 0.6625337368119735,
      "grad_norm": 12.139296531677246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18649844825267792,
      "logits/rejected": -0.3539252281188965,
      "logps/chosen": -458.7827453613281,
      "logps/rejected": -456.0890197753906,
      "loss": 0.5778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6359761953353882,
      "rewards/margins": 1.8057430982589722,
      "rewards/rejected": -2.4417190551757812,
      "step": 5063
    },
    {
      "epoch": 0.6626645947493253,
      "grad_norm": 14.166108131408691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23915590345859528,
      "logits/rejected": -0.1351335346698761,
      "logps/chosen": -665.7483520507812,
      "logps/rejected": -612.1857299804688,
      "loss": 0.9848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3199363946914673,
      "rewards/margins": 0.30874407291412354,
      "rewards/rejected": -1.6286804676055908,
      "step": 5064
    },
    {
      "epoch": 0.6627954526866771,
      "grad_norm": 7.421176433563232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3390209376811981,
      "logits/rejected": -0.2503381371498108,
      "logps/chosen": -239.62611389160156,
      "logps/rejected": -315.1560974121094,
      "loss": 0.659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4559428691864014,
      "rewards/margins": 0.09101903438568115,
      "rewards/rejected": -1.5469619035720825,
      "step": 5065
    },
    {
      "epoch": 0.6629263106240287,
      "grad_norm": 5.910504341125488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19916822016239166,
      "logits/rejected": -0.009200604632496834,
      "logps/chosen": -482.465576171875,
      "logps/rejected": -287.3436584472656,
      "loss": 0.3177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6175193786621094,
      "rewards/margins": 1.1620594263076782,
      "rewards/rejected": -1.779578685760498,
      "step": 5066
    },
    {
      "epoch": 0.6630571685613805,
      "grad_norm": 3.572913408279419,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15907835960388184,
      "logits/rejected": -0.11078101396560669,
      "logps/chosen": -527.1746826171875,
      "logps/rejected": -428.3236999511719,
      "loss": 0.2327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5107560753822327,
      "rewards/margins": 1.5148992538452148,
      "rewards/rejected": -2.0256552696228027,
      "step": 5067
    },
    {
      "epoch": 0.6631880264987323,
      "grad_norm": 7.57879638671875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1971077024936676,
      "logits/rejected": 0.378756582736969,
      "logps/chosen": -391.79443359375,
      "logps/rejected": -338.54620361328125,
      "loss": 0.5173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4802572429180145,
      "rewards/margins": 0.5646002292633057,
      "rewards/rejected": -1.044857382774353,
      "step": 5068
    },
    {
      "epoch": 0.6633188844360841,
      "grad_norm": 7.761619567871094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05029652267694473,
      "logits/rejected": 0.004783932119607925,
      "logps/chosen": -583.8609619140625,
      "logps/rejected": -392.7013854980469,
      "loss": 0.458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06747454404830933,
      "rewards/margins": 1.1524683237075806,
      "rewards/rejected": -1.084993839263916,
      "step": 5069
    },
    {
      "epoch": 0.6634497423734358,
      "grad_norm": 6.829669952392578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2881179451942444,
      "logits/rejected": 0.15997347235679626,
      "logps/chosen": -515.9754638671875,
      "logps/rejected": -562.8787841796875,
      "loss": 0.4649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5234684348106384,
      "rewards/margins": 1.1716278791427612,
      "rewards/rejected": -1.6950961351394653,
      "step": 5070
    },
    {
      "epoch": 0.6635806003107876,
      "grad_norm": 7.138078212738037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08799893409013748,
      "logits/rejected": -0.025165587663650513,
      "logps/chosen": -522.1336059570312,
      "logps/rejected": -619.80322265625,
      "loss": 0.5133,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06408633291721344,
      "rewards/margins": 1.0686731338500977,
      "rewards/rejected": -1.13275945186615,
      "step": 5071
    },
    {
      "epoch": 0.6637114582481394,
      "grad_norm": 6.503811359405518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06224197521805763,
      "logits/rejected": -0.08764252811670303,
      "logps/chosen": -330.3753356933594,
      "logps/rejected": -303.65765380859375,
      "loss": 0.7183,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9773104190826416,
      "rewards/margins": 0.18106330931186676,
      "rewards/rejected": -1.1583738327026367,
      "step": 5072
    },
    {
      "epoch": 0.6638423161854912,
      "grad_norm": 10.174497604370117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044032059609889984,
      "logits/rejected": 0.04835202544927597,
      "logps/chosen": -501.01806640625,
      "logps/rejected": -475.6600036621094,
      "loss": 0.4508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42054876685142517,
      "rewards/margins": 1.03905189037323,
      "rewards/rejected": -1.4596006870269775,
      "step": 5073
    },
    {
      "epoch": 0.6639731741228428,
      "grad_norm": 8.105079650878906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18505041301250458,
      "logits/rejected": -0.34961751103401184,
      "logps/chosen": -266.3619384765625,
      "logps/rejected": -258.3695068359375,
      "loss": 0.4378,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0884616374969482,
      "rewards/margins": 1.1487784385681152,
      "rewards/rejected": -2.2372403144836426,
      "step": 5074
    },
    {
      "epoch": 0.6641040320601946,
      "grad_norm": 9.242082595825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04659489542245865,
      "logits/rejected": 0.08467286825180054,
      "logps/chosen": -609.3524780273438,
      "logps/rejected": -590.207275390625,
      "loss": 0.5603,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2766767740249634,
      "rewards/margins": 0.5776174664497375,
      "rewards/rejected": -0.8542941808700562,
      "step": 5075
    },
    {
      "epoch": 0.6642348899975464,
      "grad_norm": 4.471347808837891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07939192652702332,
      "logits/rejected": 0.0710344910621643,
      "logps/chosen": -525.1571655273438,
      "logps/rejected": -472.861083984375,
      "loss": 0.2824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6890495419502258,
      "rewards/margins": 1.7864789962768555,
      "rewards/rejected": -2.4755287170410156,
      "step": 5076
    },
    {
      "epoch": 0.6643657479348982,
      "grad_norm": 6.117456436157227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.045369163155555725,
      "logits/rejected": 0.002374541014432907,
      "logps/chosen": -662.9166870117188,
      "logps/rejected": -537.5928955078125,
      "loss": 0.3345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.541973888874054,
      "rewards/margins": 1.590865135192871,
      "rewards/rejected": -1.048891305923462,
      "step": 5077
    },
    {
      "epoch": 0.6644966058722499,
      "grad_norm": 9.856104850769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1108182966709137,
      "logits/rejected": 0.05328284949064255,
      "logps/chosen": -324.22705078125,
      "logps/rejected": -380.2112731933594,
      "loss": 0.7178,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18128147721290588,
      "rewards/margins": 0.16630461812019348,
      "rewards/rejected": -0.347586065530777,
      "step": 5078
    },
    {
      "epoch": 0.6646274638096017,
      "grad_norm": 6.2055840492248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4109029173851013,
      "logits/rejected": 0.25744321942329407,
      "logps/chosen": -650.4356079101562,
      "logps/rejected": -459.10498046875,
      "loss": 0.4456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14375938475131989,
      "rewards/margins": 1.583988070487976,
      "rewards/rejected": -1.7277474403381348,
      "step": 5079
    },
    {
      "epoch": 0.6647583217469535,
      "grad_norm": 5.250490188598633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11075697094202042,
      "logits/rejected": -0.05968470871448517,
      "logps/chosen": -514.3934326171875,
      "logps/rejected": -573.0979614257812,
      "loss": 0.3046,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3094007968902588,
      "rewards/margins": 1.5982776880264282,
      "rewards/rejected": -1.288877010345459,
      "step": 5080
    },
    {
      "epoch": 0.6648891796843053,
      "grad_norm": 5.225468635559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20155034959316254,
      "logits/rejected": -0.07801081985235214,
      "logps/chosen": -430.22137451171875,
      "logps/rejected": -350.294921875,
      "loss": 0.4534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47642651200294495,
      "rewards/margins": 0.8451554775238037,
      "rewards/rejected": -1.3215820789337158,
      "step": 5081
    },
    {
      "epoch": 0.665020037621657,
      "grad_norm": 8.31527328491211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09111801534891129,
      "logits/rejected": -0.18229110538959503,
      "logps/chosen": -438.2756042480469,
      "logps/rejected": -479.415283203125,
      "loss": 0.5955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4203628599643707,
      "rewards/margins": 0.6020423769950867,
      "rewards/rejected": -1.0224052667617798,
      "step": 5082
    },
    {
      "epoch": 0.6651508955590087,
      "grad_norm": 4.33783483505249,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027911825105547905,
      "logits/rejected": 0.020349573343992233,
      "logps/chosen": -360.0010986328125,
      "logps/rejected": -450.429931640625,
      "loss": 0.2741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37676525115966797,
      "rewards/margins": 1.9414135217666626,
      "rewards/rejected": -1.5646482706069946,
      "step": 5083
    },
    {
      "epoch": 0.6652817534963605,
      "grad_norm": 5.884021282196045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0552600622177124,
      "logits/rejected": -0.12349773943424225,
      "logps/chosen": -318.6827392578125,
      "logps/rejected": -297.7398986816406,
      "loss": 0.4687,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2650585174560547,
      "rewards/margins": 0.9348982572555542,
      "rewards/rejected": -1.1999566555023193,
      "step": 5084
    },
    {
      "epoch": 0.6654126114337123,
      "grad_norm": 8.87759780883789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24740591645240784,
      "logits/rejected": 0.36039015650749207,
      "logps/chosen": -368.75775146484375,
      "logps/rejected": -418.8183288574219,
      "loss": 0.685,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0051767826080322,
      "rewards/margins": 0.9548564553260803,
      "rewards/rejected": -1.9600332975387573,
      "step": 5085
    },
    {
      "epoch": 0.665543469371064,
      "grad_norm": 4.371127605438232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0841117799282074,
      "logits/rejected": -0.06953597813844681,
      "logps/chosen": -519.404296875,
      "logps/rejected": -334.2428894042969,
      "loss": 0.3306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11563242971897125,
      "rewards/margins": 1.4048044681549072,
      "rewards/rejected": -1.2891720533370972,
      "step": 5086
    },
    {
      "epoch": 0.6656743273084158,
      "grad_norm": 6.648928165435791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07451172173023224,
      "logits/rejected": -0.07834140956401825,
      "logps/chosen": -393.3135070800781,
      "logps/rejected": -468.28814697265625,
      "loss": 0.3712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49004536867141724,
      "rewards/margins": 1.0993629693984985,
      "rewards/rejected": -1.589408278465271,
      "step": 5087
    },
    {
      "epoch": 0.6658051852457676,
      "grad_norm": 7.149631023406982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17651399970054626,
      "logits/rejected": -0.15746404230594635,
      "logps/chosen": -453.007568359375,
      "logps/rejected": -339.380126953125,
      "loss": 0.5244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8033113479614258,
      "rewards/margins": 1.1427202224731445,
      "rewards/rejected": -1.9460315704345703,
      "step": 5088
    },
    {
      "epoch": 0.6659360431831193,
      "grad_norm": 7.529919147491455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30667316913604736,
      "logits/rejected": -0.36578476428985596,
      "logps/chosen": -620.1233520507812,
      "logps/rejected": -586.67919921875,
      "loss": 0.4993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3585803806781769,
      "rewards/margins": 0.6195446848869324,
      "rewards/rejected": -0.9781249761581421,
      "step": 5089
    },
    {
      "epoch": 0.6660669011204711,
      "grad_norm": 5.189237594604492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08465378731489182,
      "logits/rejected": 0.00028304755687713623,
      "logps/chosen": -366.22662353515625,
      "logps/rejected": -511.49847412109375,
      "loss": 0.4155,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07110041379928589,
      "rewards/margins": 1.411357045173645,
      "rewards/rejected": -1.340256690979004,
      "step": 5090
    },
    {
      "epoch": 0.6661977590578229,
      "grad_norm": 8.113350868225098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05684274435043335,
      "logits/rejected": -0.08631590008735657,
      "logps/chosen": -495.2507629394531,
      "logps/rejected": -575.0875244140625,
      "loss": 0.4911,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8571939468383789,
      "rewards/margins": 1.3785367012023926,
      "rewards/rejected": -2.2357308864593506,
      "step": 5091
    },
    {
      "epoch": 0.6663286169951746,
      "grad_norm": 3.3181183338165283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2098386287689209,
      "logits/rejected": -0.0004678971308749169,
      "logps/chosen": -531.74609375,
      "logps/rejected": -369.90655517578125,
      "loss": 0.2512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2055511474609375,
      "rewards/margins": 2.0554864406585693,
      "rewards/rejected": -1.8499352931976318,
      "step": 5092
    },
    {
      "epoch": 0.6664594749325263,
      "grad_norm": 11.481614112854004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07893157005310059,
      "logits/rejected": -0.10121666640043259,
      "logps/chosen": -356.3105163574219,
      "logps/rejected": -567.5223999023438,
      "loss": 0.6127,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.839232325553894,
      "rewards/margins": 1.1601754426956177,
      "rewards/rejected": -1.9994075298309326,
      "step": 5093
    },
    {
      "epoch": 0.6665903328698781,
      "grad_norm": 9.387288093566895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04526441916823387,
      "logits/rejected": -0.07823330163955688,
      "logps/chosen": -451.99896240234375,
      "logps/rejected": -632.5078125,
      "loss": 0.8067,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7005651593208313,
      "rewards/margins": 0.30016952753067017,
      "rewards/rejected": -1.0007346868515015,
      "step": 5094
    },
    {
      "epoch": 0.6667211908072299,
      "grad_norm": 7.07838249206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33317437767982483,
      "logits/rejected": 0.4038459062576294,
      "logps/chosen": -396.69287109375,
      "logps/rejected": -462.8741149902344,
      "loss": 0.5035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0662206411361694,
      "rewards/margins": 0.9055676460266113,
      "rewards/rejected": -1.9717882871627808,
      "step": 5095
    },
    {
      "epoch": 0.6668520487445817,
      "grad_norm": 4.268917083740234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0908445417881012,
      "logits/rejected": 0.03732817620038986,
      "logps/chosen": -495.7093200683594,
      "logps/rejected": -459.2071838378906,
      "loss": 0.2514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1631649136543274,
      "rewards/margins": 2.1586546897888184,
      "rewards/rejected": -1.9954897165298462,
      "step": 5096
    },
    {
      "epoch": 0.6669829066819334,
      "grad_norm": 8.998357772827148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40794622898101807,
      "logits/rejected": -0.3611212372779846,
      "logps/chosen": -599.9971923828125,
      "logps/rejected": -470.494140625,
      "loss": 0.5285,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6702839136123657,
      "rewards/margins": 0.6936966180801392,
      "rewards/rejected": -1.3639805316925049,
      "step": 5097
    },
    {
      "epoch": 0.6671137646192852,
      "grad_norm": 5.766686916351318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10492546856403351,
      "logits/rejected": 0.17421957850456238,
      "logps/chosen": -377.6728820800781,
      "logps/rejected": -432.3594970703125,
      "loss": 0.4857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36024191975593567,
      "rewards/margins": 0.7001485824584961,
      "rewards/rejected": -1.0603904724121094,
      "step": 5098
    },
    {
      "epoch": 0.667244622556637,
      "grad_norm": 8.41041374206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06624630093574524,
      "logits/rejected": 0.08243679255247116,
      "logps/chosen": -476.65728759765625,
      "logps/rejected": -486.87530517578125,
      "loss": 0.7182,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6230508089065552,
      "rewards/margins": 0.3235960900783539,
      "rewards/rejected": -0.9466469287872314,
      "step": 5099
    },
    {
      "epoch": 0.6673754804939888,
      "grad_norm": 11.023568153381348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07834672182798386,
      "logits/rejected": -0.1747625172138214,
      "logps/chosen": -317.8862609863281,
      "logps/rejected": -369.6451110839844,
      "loss": 1.0071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4497987926006317,
      "rewards/margins": -0.13760128617286682,
      "rewards/rejected": -0.3121974766254425,
      "step": 5100
    },
    {
      "epoch": 0.6675063384313404,
      "grad_norm": 4.840396881103516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12768597900867462,
      "logits/rejected": -0.10509967803955078,
      "logps/chosen": -395.4266357421875,
      "logps/rejected": -357.61065673828125,
      "loss": 0.444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31112995743751526,
      "rewards/margins": 0.6602796912193298,
      "rewards/rejected": -0.9714096188545227,
      "step": 5101
    },
    {
      "epoch": 0.6676371963686922,
      "grad_norm": 6.218153953552246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13834859430789948,
      "logits/rejected": -0.09700934588909149,
      "logps/chosen": -284.9758605957031,
      "logps/rejected": -347.8974304199219,
      "loss": 0.3737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6741796135902405,
      "rewards/margins": 0.9186456799507141,
      "rewards/rejected": -1.5928252935409546,
      "step": 5102
    },
    {
      "epoch": 0.667768054306044,
      "grad_norm": 5.482421398162842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2812199890613556,
      "logits/rejected": 0.15063631534576416,
      "logps/chosen": -364.5919494628906,
      "logps/rejected": -422.4664001464844,
      "loss": 0.3921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6339874267578125,
      "rewards/margins": 1.7405414581298828,
      "rewards/rejected": -2.3745288848876953,
      "step": 5103
    },
    {
      "epoch": 0.6678989122433958,
      "grad_norm": 7.349590301513672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2346946895122528,
      "logits/rejected": -0.28904497623443604,
      "logps/chosen": -374.5383605957031,
      "logps/rejected": -484.7383728027344,
      "loss": 0.5124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4678611755371094,
      "rewards/margins": 1.0979046821594238,
      "rewards/rejected": -1.5657660961151123,
      "step": 5104
    },
    {
      "epoch": 0.6680297701807475,
      "grad_norm": 6.245404243469238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.131437748670578,
      "logits/rejected": 0.024227863177657127,
      "logps/chosen": -516.6153564453125,
      "logps/rejected": -504.96917724609375,
      "loss": 0.3702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6704890131950378,
      "rewards/margins": 1.8339576721191406,
      "rewards/rejected": -2.5044467449188232,
      "step": 5105
    },
    {
      "epoch": 0.6681606281180993,
      "grad_norm": 10.397805213928223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04108349233865738,
      "logits/rejected": -0.007176993880420923,
      "logps/chosen": -307.4334716796875,
      "logps/rejected": -508.3292541503906,
      "loss": 0.6272,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3659103810787201,
      "rewards/margins": 0.7844920754432678,
      "rewards/rejected": -1.1504024267196655,
      "step": 5106
    },
    {
      "epoch": 0.6682914860554511,
      "grad_norm": 4.8467116355896,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0428374744951725,
      "logits/rejected": 0.05806049332022667,
      "logps/chosen": -400.588134765625,
      "logps/rejected": -311.4293518066406,
      "loss": 0.2413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15091820061206818,
      "rewards/margins": 2.2661900520324707,
      "rewards/rejected": -2.4171078205108643,
      "step": 5107
    },
    {
      "epoch": 0.6684223439928028,
      "grad_norm": 9.566768646240234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08294753730297089,
      "logits/rejected": -0.02990877628326416,
      "logps/chosen": -534.1253662109375,
      "logps/rejected": -686.5616455078125,
      "loss": 0.8812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.140893578529358,
      "rewards/margins": -0.1647987961769104,
      "rewards/rejected": -0.9760948419570923,
      "step": 5108
    },
    {
      "epoch": 0.6685532019301546,
      "grad_norm": 6.484427452087402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11935140192508698,
      "logits/rejected": 0.12752759456634521,
      "logps/chosen": -546.008056640625,
      "logps/rejected": -587.5576782226562,
      "loss": 0.3505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24191685020923615,
      "rewards/margins": 1.8084359169006348,
      "rewards/rejected": -1.5665191411972046,
      "step": 5109
    },
    {
      "epoch": 0.6686840598675063,
      "grad_norm": 8.558494567871094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20788823068141937,
      "logits/rejected": -0.03389847278594971,
      "logps/chosen": -465.38128662109375,
      "logps/rejected": -451.53082275390625,
      "loss": 0.5303,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2706955671310425,
      "rewards/margins": 0.7158765196800232,
      "rewards/rejected": -1.9865721464157104,
      "step": 5110
    },
    {
      "epoch": 0.6688149178048581,
      "grad_norm": 11.928383827209473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0701778382062912,
      "logits/rejected": -0.15095585584640503,
      "logps/chosen": -575.560546875,
      "logps/rejected": -473.1800842285156,
      "loss": 0.6134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6639776229858398,
      "rewards/margins": 0.5617954730987549,
      "rewards/rejected": -1.2257729768753052,
      "step": 5111
    },
    {
      "epoch": 0.6689457757422098,
      "grad_norm": 9.652698516845703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20965127646923065,
      "logits/rejected": 0.1261410266160965,
      "logps/chosen": -546.6973876953125,
      "logps/rejected": -489.84735107421875,
      "loss": 0.6577,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27067604660987854,
      "rewards/margins": 0.27543142437934875,
      "rewards/rejected": -0.5461075305938721,
      "step": 5112
    },
    {
      "epoch": 0.6690766336795616,
      "grad_norm": 7.528936862945557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07597887516021729,
      "logits/rejected": -0.0029828455299139023,
      "logps/chosen": -400.28094482421875,
      "logps/rejected": -385.36767578125,
      "loss": 0.4971,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2683066129684448,
      "rewards/margins": 0.7437801361083984,
      "rewards/rejected": -2.012086868286133,
      "step": 5113
    },
    {
      "epoch": 0.6692074916169134,
      "grad_norm": 9.035944938659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44950535893440247,
      "logits/rejected": -0.5749959945678711,
      "logps/chosen": -513.6170043945312,
      "logps/rejected": -409.58343505859375,
      "loss": 0.7229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.186408519744873,
      "rewards/margins": 0.7344034314155579,
      "rewards/rejected": -1.9208118915557861,
      "step": 5114
    },
    {
      "epoch": 0.6693383495542652,
      "grad_norm": 10.749815940856934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15304221212863922,
      "logits/rejected": 0.0965433269739151,
      "logps/chosen": -467.4344177246094,
      "logps/rejected": -487.2391357421875,
      "loss": 0.7304,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32669633626937866,
      "rewards/margins": 0.855740487575531,
      "rewards/rejected": -1.1824370622634888,
      "step": 5115
    },
    {
      "epoch": 0.6694692074916169,
      "grad_norm": 7.842963695526123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09150940179824829,
      "logits/rejected": -0.05956742912530899,
      "logps/chosen": -389.91973876953125,
      "logps/rejected": -537.1353149414062,
      "loss": 0.3986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.799968957901001,
      "rewards/margins": 1.4494669437408447,
      "rewards/rejected": -2.2494359016418457,
      "step": 5116
    },
    {
      "epoch": 0.6696000654289687,
      "grad_norm": 10.467133522033691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17499986290931702,
      "logits/rejected": 0.2505144774913788,
      "logps/chosen": -391.0990905761719,
      "logps/rejected": -463.94366455078125,
      "loss": 0.9984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0205360651016235,
      "rewards/margins": -0.2794731557369232,
      "rewards/rejected": -0.7410629391670227,
      "step": 5117
    },
    {
      "epoch": 0.6697309233663205,
      "grad_norm": 8.913544654846191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13972364366054535,
      "logits/rejected": 0.21256119012832642,
      "logps/chosen": -347.3730773925781,
      "logps/rejected": -367.1439208984375,
      "loss": 0.7399,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9282209277153015,
      "rewards/margins": 0.6016305685043335,
      "rewards/rejected": -1.5298514366149902,
      "step": 5118
    },
    {
      "epoch": 0.6698617813036722,
      "grad_norm": 6.908047676086426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33422037959098816,
      "logits/rejected": 0.21857337653636932,
      "logps/chosen": -503.8463134765625,
      "logps/rejected": -614.1710815429688,
      "loss": 0.3497,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33218157291412354,
      "rewards/margins": 2.2548115253448486,
      "rewards/rejected": -2.5869932174682617,
      "step": 5119
    },
    {
      "epoch": 0.6699926392410239,
      "grad_norm": 7.495398044586182,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07748855650424957,
      "logits/rejected": -0.3364327549934387,
      "logps/chosen": -376.4874267578125,
      "logps/rejected": -341.86260986328125,
      "loss": 0.5176,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7819700837135315,
      "rewards/margins": 1.0087566375732422,
      "rewards/rejected": -1.7907267808914185,
      "step": 5120
    },
    {
      "epoch": 0.6701234971783757,
      "grad_norm": 8.124361038208008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00962386280298233,
      "logits/rejected": 0.22225451469421387,
      "logps/chosen": -349.1337890625,
      "logps/rejected": -275.76409912109375,
      "loss": 0.6789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09987544268369675,
      "rewards/margins": 0.7679336667060852,
      "rewards/rejected": -0.6680582761764526,
      "step": 5121
    },
    {
      "epoch": 0.6702543551157275,
      "grad_norm": 9.83918571472168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010843856260180473,
      "logits/rejected": -0.1749064028263092,
      "logps/chosen": -483.7401123046875,
      "logps/rejected": -500.23931884765625,
      "loss": 0.7205,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7247918248176575,
      "rewards/margins": 0.26464468240737915,
      "rewards/rejected": -0.9894363880157471,
      "step": 5122
    },
    {
      "epoch": 0.6703852130530793,
      "grad_norm": 6.815192222595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.162639319896698,
      "logits/rejected": -0.02311065047979355,
      "logps/chosen": -313.7934265136719,
      "logps/rejected": -299.3401184082031,
      "loss": 0.5542,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3625952005386353,
      "rewards/margins": 0.42706552147865295,
      "rewards/rejected": -1.7896606922149658,
      "step": 5123
    },
    {
      "epoch": 0.670516070990431,
      "grad_norm": 5.672943115234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3887641429901123,
      "logits/rejected": -0.36504390835762024,
      "logps/chosen": -378.5861511230469,
      "logps/rejected": -358.95855712890625,
      "loss": 0.3977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6172437071800232,
      "rewards/margins": 0.9596068859100342,
      "rewards/rejected": -1.5768505334854126,
      "step": 5124
    },
    {
      "epoch": 0.6706469289277828,
      "grad_norm": 7.433385372161865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35212862491607666,
      "logits/rejected": 0.04563459753990173,
      "logps/chosen": -605.656005859375,
      "logps/rejected": -522.1587524414062,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3096240758895874,
      "rewards/margins": 0.7202385067939758,
      "rewards/rejected": -1.0298625230789185,
      "step": 5125
    },
    {
      "epoch": 0.6707777868651346,
      "grad_norm": 9.973357200622559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04640297591686249,
      "logits/rejected": -0.027297034859657288,
      "logps/chosen": -296.5799560546875,
      "logps/rejected": -295.0123596191406,
      "loss": 0.7344,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5095946788787842,
      "rewards/margins": 0.7258049845695496,
      "rewards/rejected": -1.235399603843689,
      "step": 5126
    },
    {
      "epoch": 0.6709086448024864,
      "grad_norm": 7.766532897949219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23373019695281982,
      "logits/rejected": 0.29305779933929443,
      "logps/chosen": -514.4248657226562,
      "logps/rejected": -393.6290588378906,
      "loss": 0.5029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29699137806892395,
      "rewards/margins": 1.2572215795516968,
      "rewards/rejected": -1.5542129278182983,
      "step": 5127
    },
    {
      "epoch": 0.671039502739838,
      "grad_norm": 7.1836628913879395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13583552837371826,
      "logits/rejected": -0.07117632031440735,
      "logps/chosen": -448.0337829589844,
      "logps/rejected": -372.6123046875,
      "loss": 0.504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.045023370534181595,
      "rewards/margins": 1.1497598886489868,
      "rewards/rejected": -1.1947832107543945,
      "step": 5128
    },
    {
      "epoch": 0.6711703606771898,
      "grad_norm": 7.794274806976318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04320331662893295,
      "logits/rejected": -0.13729053735733032,
      "logps/chosen": -736.4857788085938,
      "logps/rejected": -830.5347900390625,
      "loss": 0.4012,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8458728790283203,
      "rewards/margins": 1.133751630783081,
      "rewards/rejected": -1.979624629020691,
      "step": 5129
    },
    {
      "epoch": 0.6713012186145416,
      "grad_norm": 4.769452095031738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13062764704227448,
      "logits/rejected": 0.1193271204829216,
      "logps/chosen": -378.4241943359375,
      "logps/rejected": -374.5245361328125,
      "loss": 0.3504,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1753925085067749,
      "rewards/margins": 1.6334946155548096,
      "rewards/rejected": -1.458101749420166,
      "step": 5130
    },
    {
      "epoch": 0.6714320765518933,
      "grad_norm": 4.479739665985107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11557231098413467,
      "logits/rejected": 0.13274019956588745,
      "logps/chosen": -436.4172668457031,
      "logps/rejected": -551.7809448242188,
      "loss": 0.3212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5747141242027283,
      "rewards/margins": 1.4364068508148193,
      "rewards/rejected": -2.0111207962036133,
      "step": 5131
    },
    {
      "epoch": 0.6715629344892451,
      "grad_norm": 7.7958269119262695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4814899265766144,
      "logits/rejected": -0.4050700068473816,
      "logps/chosen": -310.6222229003906,
      "logps/rejected": -326.16583251953125,
      "loss": 0.6436,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0747647285461426,
      "rewards/margins": 0.17745837569236755,
      "rewards/rejected": -1.2522231340408325,
      "step": 5132
    },
    {
      "epoch": 0.6716937924265969,
      "grad_norm": 9.1182222366333,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13977691531181335,
      "logits/rejected": -0.17112340033054352,
      "logps/chosen": -819.97265625,
      "logps/rejected": -522.720458984375,
      "loss": 0.6228,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5174956917762756,
      "rewards/margins": 0.3312442898750305,
      "rewards/rejected": -0.8487400412559509,
      "step": 5133
    },
    {
      "epoch": 0.6718246503639487,
      "grad_norm": 10.389673233032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15898573398590088,
      "logits/rejected": -0.0024142060428857803,
      "logps/chosen": -466.1346740722656,
      "logps/rejected": -565.7288208007812,
      "loss": 0.8178,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2809118032455444,
      "rewards/margins": 0.15430547297000885,
      "rewards/rejected": -1.4352173805236816,
      "step": 5134
    },
    {
      "epoch": 0.6719555083013004,
      "grad_norm": 6.729360580444336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11828203499317169,
      "logits/rejected": 0.07311215996742249,
      "logps/chosen": -468.0360412597656,
      "logps/rejected": -489.9686584472656,
      "loss": 0.4602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3817465007305145,
      "rewards/margins": 0.7283015847206116,
      "rewards/rejected": -1.1100480556488037,
      "step": 5135
    },
    {
      "epoch": 0.6720863662386521,
      "grad_norm": 10.89133358001709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01582026481628418,
      "logits/rejected": 0.07738948613405228,
      "logps/chosen": -567.7085571289062,
      "logps/rejected": -614.568603515625,
      "loss": 0.656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6272453665733337,
      "rewards/margins": 0.6482452154159546,
      "rewards/rejected": -1.275490641593933,
      "step": 5136
    },
    {
      "epoch": 0.6722172241760039,
      "grad_norm": 7.986497402191162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11024727672338486,
      "logits/rejected": -0.05229012668132782,
      "logps/chosen": -526.6451416015625,
      "logps/rejected": -421.7123107910156,
      "loss": 0.4431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03832623362541199,
      "rewards/margins": 1.5850322246551514,
      "rewards/rejected": -1.5467058420181274,
      "step": 5137
    },
    {
      "epoch": 0.6723480821133557,
      "grad_norm": 3.452895164489746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1278059482574463,
      "logits/rejected": 0.04671507328748703,
      "logps/chosen": -451.676025390625,
      "logps/rejected": -529.8834228515625,
      "loss": 0.1609,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22611960768699646,
      "rewards/margins": 2.946288585662842,
      "rewards/rejected": -3.172408103942871,
      "step": 5138
    },
    {
      "epoch": 0.6724789400507074,
      "grad_norm": 7.596905708312988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05496683344244957,
      "logits/rejected": 0.05229534953832626,
      "logps/chosen": -885.708251953125,
      "logps/rejected": -812.4852905273438,
      "loss": 0.3908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.020740456879138947,
      "rewards/margins": 2.045328378677368,
      "rewards/rejected": -2.0660691261291504,
      "step": 5139
    },
    {
      "epoch": 0.6726097979880592,
      "grad_norm": 5.078707218170166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29780906438827515,
      "logits/rejected": 0.16805137693881989,
      "logps/chosen": -596.14892578125,
      "logps/rejected": -380.11456298828125,
      "loss": 0.3045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.932170569896698,
      "rewards/margins": 1.5612826347351074,
      "rewards/rejected": -2.493453025817871,
      "step": 5140
    },
    {
      "epoch": 0.672740655925411,
      "grad_norm": 8.8068208694458,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14029347896575928,
      "logits/rejected": 0.026992395520210266,
      "logps/chosen": -466.932861328125,
      "logps/rejected": -575.9440307617188,
      "loss": 0.5546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5249174237251282,
      "rewards/margins": 1.2949918508529663,
      "rewards/rejected": -1.8199093341827393,
      "step": 5141
    },
    {
      "epoch": 0.6728715138627628,
      "grad_norm": 4.969098091125488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08236570656299591,
      "logits/rejected": -0.055649228394031525,
      "logps/chosen": -450.13348388671875,
      "logps/rejected": -576.0921630859375,
      "loss": 0.1951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0395292192697525,
      "rewards/margins": 1.9508888721466064,
      "rewards/rejected": -1.9113597869873047,
      "step": 5142
    },
    {
      "epoch": 0.6730023718001145,
      "grad_norm": 6.048758029937744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039774924516677856,
      "logits/rejected": -0.02620657905936241,
      "logps/chosen": -498.65185546875,
      "logps/rejected": -471.8997802734375,
      "loss": 0.3644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.025492286309599876,
      "rewards/margins": 1.7086774110794067,
      "rewards/rejected": -1.68318510055542,
      "step": 5143
    },
    {
      "epoch": 0.6731332297374663,
      "grad_norm": 3.036052703857422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4474547803401947,
      "logits/rejected": -0.4560614228248596,
      "logps/chosen": -333.47802734375,
      "logps/rejected": -362.77740478515625,
      "loss": 0.2356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0523293018341064,
      "rewards/margins": 2.0444633960723877,
      "rewards/rejected": -3.096792697906494,
      "step": 5144
    },
    {
      "epoch": 0.673264087674818,
      "grad_norm": 6.453623294830322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07874661684036255,
      "logits/rejected": -0.15299415588378906,
      "logps/chosen": -338.2382507324219,
      "logps/rejected": -417.3009948730469,
      "loss": 0.3302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14680786430835724,
      "rewards/margins": 1.337591290473938,
      "rewards/rejected": -1.4843993186950684,
      "step": 5145
    },
    {
      "epoch": 0.6733949456121698,
      "grad_norm": 8.288037300109863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14883659780025482,
      "logits/rejected": -0.1198749914765358,
      "logps/chosen": -409.66015625,
      "logps/rejected": -335.04107666015625,
      "loss": 0.5387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4388485252857208,
      "rewards/margins": 0.5449120402336121,
      "rewards/rejected": -0.9837605953216553,
      "step": 5146
    },
    {
      "epoch": 0.6735258035495215,
      "grad_norm": 7.629393100738525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39404889941215515,
      "logits/rejected": 0.26091229915618896,
      "logps/chosen": -405.0466003417969,
      "logps/rejected": -367.2264709472656,
      "loss": 0.3973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22614306211471558,
      "rewards/margins": 1.3060835599899292,
      "rewards/rejected": -1.5322265625,
      "step": 5147
    },
    {
      "epoch": 0.6736566614868733,
      "grad_norm": 7.817178726196289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41572821140289307,
      "logits/rejected": -0.28621476888656616,
      "logps/chosen": -542.3540649414062,
      "logps/rejected": -499.4088134765625,
      "loss": 0.396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8433172106742859,
      "rewards/margins": 0.9995965957641602,
      "rewards/rejected": -1.8429138660430908,
      "step": 5148
    },
    {
      "epoch": 0.6737875194242251,
      "grad_norm": 4.706846714019775,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023892179131507874,
      "logits/rejected": -0.19054177403450012,
      "logps/chosen": -553.2081298828125,
      "logps/rejected": -453.62322998046875,
      "loss": 0.2392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35383090376853943,
      "rewards/margins": 1.7100956439971924,
      "rewards/rejected": -2.0639266967773438,
      "step": 5149
    },
    {
      "epoch": 0.6739183773615769,
      "grad_norm": 7.844920635223389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03934238478541374,
      "logits/rejected": 0.06839805096387863,
      "logps/chosen": -412.1339111328125,
      "logps/rejected": -440.4036560058594,
      "loss": 0.4693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.221926212310791,
      "rewards/margins": 1.0991504192352295,
      "rewards/rejected": -2.3210768699645996,
      "step": 5150
    },
    {
      "epoch": 0.6740492352989286,
      "grad_norm": 7.405709266662598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3218766152858734,
      "logits/rejected": -0.1797400265932083,
      "logps/chosen": -328.2624816894531,
      "logps/rejected": -481.3105773925781,
      "loss": 0.3545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9603904485702515,
      "rewards/margins": 1.1719763278961182,
      "rewards/rejected": -2.132366895675659,
      "step": 5151
    },
    {
      "epoch": 0.6741800932362804,
      "grad_norm": 12.130176544189453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25726044178009033,
      "logits/rejected": 0.11764498800039291,
      "logps/chosen": -668.5978393554688,
      "logps/rejected": -518.4608764648438,
      "loss": 0.8397,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3261692523956299,
      "rewards/margins": 0.3680133819580078,
      "rewards/rejected": -0.6941825747489929,
      "step": 5152
    },
    {
      "epoch": 0.6743109511736322,
      "grad_norm": 9.865975379943848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12352146208286285,
      "logits/rejected": -0.024724766612052917,
      "logps/chosen": -477.89105224609375,
      "logps/rejected": -492.641357421875,
      "loss": 0.5858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4040721654891968,
      "rewards/margins": 0.5316526889801025,
      "rewards/rejected": -0.9357248544692993,
      "step": 5153
    },
    {
      "epoch": 0.6744418091109838,
      "grad_norm": 9.857503890991211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17843562364578247,
      "logits/rejected": 0.1143786609172821,
      "logps/chosen": -649.8942260742188,
      "logps/rejected": -545.2273559570312,
      "loss": 0.8005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8924733996391296,
      "rewards/margins": 0.20591431856155396,
      "rewards/rejected": -1.0983878374099731,
      "step": 5154
    },
    {
      "epoch": 0.6745726670483356,
      "grad_norm": 11.410110473632812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1870427280664444,
      "logits/rejected": -0.14789697527885437,
      "logps/chosen": -572.0346069335938,
      "logps/rejected": -496.3999328613281,
      "loss": 0.6006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3935904502868652,
      "rewards/margins": 0.8814874887466431,
      "rewards/rejected": -2.2750778198242188,
      "step": 5155
    },
    {
      "epoch": 0.6747035249856874,
      "grad_norm": 8.043933868408203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32791730761528015,
      "logits/rejected": 0.22145715355873108,
      "logps/chosen": -469.4670715332031,
      "logps/rejected": -478.679931640625,
      "loss": 0.494,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3359800577163696,
      "rewards/margins": 0.8646341562271118,
      "rewards/rejected": -2.2006142139434814,
      "step": 5156
    },
    {
      "epoch": 0.6748343829230392,
      "grad_norm": 9.031270980834961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07432469725608826,
      "logits/rejected": 0.07326717674732208,
      "logps/chosen": -372.3416442871094,
      "logps/rejected": -436.24444580078125,
      "loss": 0.7674,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7319412231445312,
      "rewards/margins": 0.4547806680202484,
      "rewards/rejected": -1.1867220401763916,
      "step": 5157
    },
    {
      "epoch": 0.6749652408603909,
      "grad_norm": 5.879412651062012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07993073761463165,
      "logits/rejected": -0.17823585867881775,
      "logps/chosen": -382.308349609375,
      "logps/rejected": -449.6717529296875,
      "loss": 0.4372,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3560667634010315,
      "rewards/margins": 1.1336930990219116,
      "rewards/rejected": -1.489759922027588,
      "step": 5158
    },
    {
      "epoch": 0.6750960987977427,
      "grad_norm": 7.0028839111328125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24840408563613892,
      "logits/rejected": 0.263688862323761,
      "logps/chosen": -302.8831787109375,
      "logps/rejected": -478.07958984375,
      "loss": 0.6184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7305565476417542,
      "rewards/margins": 0.8557389974594116,
      "rewards/rejected": -1.5862956047058105,
      "step": 5159
    },
    {
      "epoch": 0.6752269567350945,
      "grad_norm": 7.483874320983887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14814916253089905,
      "logits/rejected": -0.4257011413574219,
      "logps/chosen": -531.8056030273438,
      "logps/rejected": -612.130859375,
      "loss": 0.594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7813150882720947,
      "rewards/margins": 1.1224700212478638,
      "rewards/rejected": -1.903784990310669,
      "step": 5160
    },
    {
      "epoch": 0.6753578146724463,
      "grad_norm": 4.913893699645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09777875244617462,
      "logits/rejected": -0.18244361877441406,
      "logps/chosen": -372.1844177246094,
      "logps/rejected": -320.74761962890625,
      "loss": 0.4394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2350742518901825,
      "rewards/margins": 1.758320927619934,
      "rewards/rejected": -1.993395447731018,
      "step": 5161
    },
    {
      "epoch": 0.675488672609798,
      "grad_norm": 20.630783081054688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02323097735643387,
      "logits/rejected": -0.23938189446926117,
      "logps/chosen": -574.191650390625,
      "logps/rejected": -456.61053466796875,
      "loss": 1.0564,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4544256925582886,
      "rewards/margins": -0.09695661067962646,
      "rewards/rejected": -1.3574692010879517,
      "step": 5162
    },
    {
      "epoch": 0.6756195305471497,
      "grad_norm": 7.487783432006836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11552242934703827,
      "logits/rejected": -0.005797140300273895,
      "logps/chosen": -305.9978942871094,
      "logps/rejected": -384.5205078125,
      "loss": 0.5693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.184773325920105,
      "rewards/margins": 0.7731927633285522,
      "rewards/rejected": -1.9579662084579468,
      "step": 5163
    },
    {
      "epoch": 0.6757503884845015,
      "grad_norm": 12.194127082824707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06782132387161255,
      "logits/rejected": 0.021072303876280785,
      "logps/chosen": -600.4571533203125,
      "logps/rejected": -596.5203247070312,
      "loss": 0.781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6565632224082947,
      "rewards/margins": 0.11488687992095947,
      "rewards/rejected": -0.7714501023292542,
      "step": 5164
    },
    {
      "epoch": 0.6758812464218533,
      "grad_norm": 5.7536492347717285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0665784627199173,
      "logits/rejected": -0.11113020777702332,
      "logps/chosen": -331.639404296875,
      "logps/rejected": -300.45843505859375,
      "loss": 0.2779,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3977546691894531,
      "rewards/margins": 1.5261774063110352,
      "rewards/rejected": -1.9239320755004883,
      "step": 5165
    },
    {
      "epoch": 0.676012104359205,
      "grad_norm": 8.053242683410645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15169531106948853,
      "logits/rejected": 0.2989707589149475,
      "logps/chosen": -596.8378295898438,
      "logps/rejected": -618.15478515625,
      "loss": 0.7452,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3832300007343292,
      "rewards/margins": 0.06527121365070343,
      "rewards/rejected": -0.44850119948387146,
      "step": 5166
    },
    {
      "epoch": 0.6761429622965568,
      "grad_norm": 6.184628009796143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39938291907310486,
      "logits/rejected": 0.2293730080127716,
      "logps/chosen": -483.9748229980469,
      "logps/rejected": -581.5158081054688,
      "loss": 0.3302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25865820050239563,
      "rewards/margins": 1.4653279781341553,
      "rewards/rejected": -1.7239861488342285,
      "step": 5167
    },
    {
      "epoch": 0.6762738202339086,
      "grad_norm": 9.85637092590332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.038265034556388855,
      "logits/rejected": 0.041605979204177856,
      "logps/chosen": -688.5511474609375,
      "logps/rejected": -511.68438720703125,
      "loss": 0.69,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0091651678085327,
      "rewards/margins": 0.17064708471298218,
      "rewards/rejected": -1.1798123121261597,
      "step": 5168
    },
    {
      "epoch": 0.6764046781712604,
      "grad_norm": 10.817008018493652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0141303651034832,
      "logits/rejected": -0.1300925612449646,
      "logps/chosen": -496.2626037597656,
      "logps/rejected": -460.1081848144531,
      "loss": 0.8191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0727688074111938,
      "rewards/margins": 0.40710964798927307,
      "rewards/rejected": -1.479878544807434,
      "step": 5169
    },
    {
      "epoch": 0.6765355361086121,
      "grad_norm": 12.052055358886719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07711229473352432,
      "logits/rejected": 0.16542844474315643,
      "logps/chosen": -419.1896057128906,
      "logps/rejected": -398.0809326171875,
      "loss": 1.2207,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.561266303062439,
      "rewards/margins": -0.506076455116272,
      "rewards/rejected": -1.055189847946167,
      "step": 5170
    },
    {
      "epoch": 0.6766663940459638,
      "grad_norm": 5.725978374481201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024440689012408257,
      "logits/rejected": 0.07359900325536728,
      "logps/chosen": -527.9021606445312,
      "logps/rejected": -569.6644897460938,
      "loss": 0.402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0401710569858551,
      "rewards/margins": 1.0062413215637207,
      "rewards/rejected": -0.966070294380188,
      "step": 5171
    },
    {
      "epoch": 0.6767972519833156,
      "grad_norm": 11.186911582946777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15358787775039673,
      "logits/rejected": -0.0732368528842926,
      "logps/chosen": -516.8796997070312,
      "logps/rejected": -491.3638000488281,
      "loss": 0.6381,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6490788459777832,
      "rewards/margins": 0.5979037284851074,
      "rewards/rejected": -2.2469825744628906,
      "step": 5172
    },
    {
      "epoch": 0.6769281099206673,
      "grad_norm": 7.572180271148682,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0922025591135025,
      "logits/rejected": -0.21084126830101013,
      "logps/chosen": -576.74462890625,
      "logps/rejected": -501.9215087890625,
      "loss": 0.4723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7634553909301758,
      "rewards/margins": 0.944990336894989,
      "rewards/rejected": -1.7084457874298096,
      "step": 5173
    },
    {
      "epoch": 0.6770589678580191,
      "grad_norm": 7.0058722496032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2939356565475464,
      "logits/rejected": 0.025824442505836487,
      "logps/chosen": -305.64501953125,
      "logps/rejected": -366.365966796875,
      "loss": 0.4449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5258934497833252,
      "rewards/margins": 1.0747900009155273,
      "rewards/rejected": -1.6006834506988525,
      "step": 5174
    },
    {
      "epoch": 0.6771898257953709,
      "grad_norm": 7.119475841522217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1347188949584961,
      "logits/rejected": -0.18872015178203583,
      "logps/chosen": -603.9822998046875,
      "logps/rejected": -489.9796142578125,
      "loss": 0.5135,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3639981746673584,
      "rewards/margins": 0.5681751370429993,
      "rewards/rejected": -1.932173490524292,
      "step": 5175
    },
    {
      "epoch": 0.6773206837327227,
      "grad_norm": 7.599984645843506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09848999977111816,
      "logits/rejected": -0.015305481851100922,
      "logps/chosen": -440.2685546875,
      "logps/rejected": -433.3492431640625,
      "loss": 0.5766,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.35512855648994446,
      "rewards/margins": 1.2523773908615112,
      "rewards/rejected": -0.8972488045692444,
      "step": 5176
    },
    {
      "epoch": 0.6774515416700744,
      "grad_norm": 7.082450866699219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2093748152256012,
      "logits/rejected": 0.12404663860797882,
      "logps/chosen": -505.4082336425781,
      "logps/rejected": -356.17120361328125,
      "loss": 0.5336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004587081260979176,
      "rewards/margins": 1.0083084106445312,
      "rewards/rejected": -1.0037212371826172,
      "step": 5177
    },
    {
      "epoch": 0.6775823996074262,
      "grad_norm": 9.493426322937012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.046702902764081955,
      "logits/rejected": -0.09483025223016739,
      "logps/chosen": -477.09649658203125,
      "logps/rejected": -475.71136474609375,
      "loss": 0.5158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.382223516702652,
      "rewards/margins": 0.9746319651603699,
      "rewards/rejected": -1.3568556308746338,
      "step": 5178
    },
    {
      "epoch": 0.677713257544778,
      "grad_norm": 6.508005142211914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23456069827079773,
      "logits/rejected": 0.007544348016381264,
      "logps/chosen": -307.305908203125,
      "logps/rejected": -343.3193359375,
      "loss": 0.5396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3835611343383789,
      "rewards/margins": 0.8717156052589417,
      "rewards/rejected": -1.2552767992019653,
      "step": 5179
    },
    {
      "epoch": 0.6778441154821297,
      "grad_norm": 11.806346893310547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017225654795765877,
      "logits/rejected": 0.06519033759832382,
      "logps/chosen": -347.8083190917969,
      "logps/rejected": -449.8796691894531,
      "loss": 0.9649,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.366774320602417,
      "rewards/margins": -0.1565048098564148,
      "rewards/rejected": -1.2102696895599365,
      "step": 5180
    },
    {
      "epoch": 0.6779749734194814,
      "grad_norm": 6.796186923980713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14093926548957825,
      "logits/rejected": 0.2334449291229248,
      "logps/chosen": -546.570068359375,
      "logps/rejected": -504.6686096191406,
      "loss": 0.4866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40978795289993286,
      "rewards/margins": 0.8228239417076111,
      "rewards/rejected": -1.2326117753982544,
      "step": 5181
    },
    {
      "epoch": 0.6781058313568332,
      "grad_norm": 9.537161827087402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1130487322807312,
      "logits/rejected": -0.016184944659471512,
      "logps/chosen": -391.33367919921875,
      "logps/rejected": -462.6537780761719,
      "loss": 0.776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7221875786781311,
      "rewards/margins": 0.14724348485469818,
      "rewards/rejected": -0.8694310188293457,
      "step": 5182
    },
    {
      "epoch": 0.678236689294185,
      "grad_norm": 5.840165615081787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1470613330602646,
      "logits/rejected": 0.08275168389081955,
      "logps/chosen": -407.91229248046875,
      "logps/rejected": -470.9330749511719,
      "loss": 0.3027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5589599013328552,
      "rewards/margins": 1.6910746097564697,
      "rewards/rejected": -1.1321148872375488,
      "step": 5183
    },
    {
      "epoch": 0.6783675472315368,
      "grad_norm": 6.962063312530518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00676080584526062,
      "logits/rejected": 0.1334657222032547,
      "logps/chosen": -417.3441162109375,
      "logps/rejected": -476.7545166015625,
      "loss": 0.3143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.189176544547081,
      "rewards/margins": 1.737352728843689,
      "rewards/rejected": -1.9265294075012207,
      "step": 5184
    },
    {
      "epoch": 0.6784984051688885,
      "grad_norm": 4.642251491546631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24112623929977417,
      "logits/rejected": -0.4848150312900543,
      "logps/chosen": -278.8988342285156,
      "logps/rejected": -361.8741455078125,
      "loss": 0.3568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.517388939857483,
      "rewards/margins": 1.2926934957504272,
      "rewards/rejected": -2.81008243560791,
      "step": 5185
    },
    {
      "epoch": 0.6786292631062403,
      "grad_norm": 6.237491607666016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1580849289894104,
      "logits/rejected": 0.018749846145510674,
      "logps/chosen": -432.7897033691406,
      "logps/rejected": -484.9693908691406,
      "loss": 0.3713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8685078024864197,
      "rewards/margins": 0.9022114276885986,
      "rewards/rejected": -1.770719289779663,
      "step": 5186
    },
    {
      "epoch": 0.6787601210435921,
      "grad_norm": 7.362175464630127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055283866822719574,
      "logits/rejected": -0.011401445604860783,
      "logps/chosen": -292.7071533203125,
      "logps/rejected": -358.42926025390625,
      "loss": 0.5161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7696405649185181,
      "rewards/margins": 1.106588363647461,
      "rewards/rejected": -1.876228928565979,
      "step": 5187
    },
    {
      "epoch": 0.6788909789809439,
      "grad_norm": 8.1512451171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06709708273410797,
      "logits/rejected": -0.0554535835981369,
      "logps/chosen": -436.2918395996094,
      "logps/rejected": -532.5772094726562,
      "loss": 0.6068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2282198667526245,
      "rewards/margins": 0.9498765468597412,
      "rewards/rejected": -1.1780964136123657,
      "step": 5188
    },
    {
      "epoch": 0.6790218369182955,
      "grad_norm": 5.826519966125488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047188885509967804,
      "logits/rejected": 0.26458966732025146,
      "logps/chosen": -504.40087890625,
      "logps/rejected": -597.1336669921875,
      "loss": 0.272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3518233895301819,
      "rewards/margins": 1.4989216327667236,
      "rewards/rejected": -1.8507449626922607,
      "step": 5189
    },
    {
      "epoch": 0.6791526948556473,
      "grad_norm": 7.562754154205322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09826576709747314,
      "logits/rejected": -0.08344372361898422,
      "logps/chosen": -330.0636901855469,
      "logps/rejected": -416.1009216308594,
      "loss": 0.5602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5871012806892395,
      "rewards/margins": 1.0203664302825928,
      "rewards/rejected": -1.607467532157898,
      "step": 5190
    },
    {
      "epoch": 0.6792835527929991,
      "grad_norm": 5.912304401397705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14048126339912415,
      "logits/rejected": -0.1313105672597885,
      "logps/chosen": -528.7083740234375,
      "logps/rejected": -476.16741943359375,
      "loss": 0.381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1354011595249176,
      "rewards/margins": 1.4638878107070923,
      "rewards/rejected": -1.599289059638977,
      "step": 5191
    },
    {
      "epoch": 0.6794144107303509,
      "grad_norm": 5.257744312286377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07792910933494568,
      "logits/rejected": 0.24609214067459106,
      "logps/chosen": -440.2658386230469,
      "logps/rejected": -542.1905517578125,
      "loss": 0.4003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19086669385433197,
      "rewards/margins": 1.3336504697799683,
      "rewards/rejected": -1.142783761024475,
      "step": 5192
    },
    {
      "epoch": 0.6795452686677026,
      "grad_norm": 5.157970905303955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024553045630455017,
      "logits/rejected": -0.19943398237228394,
      "logps/chosen": -530.0338134765625,
      "logps/rejected": -633.2345581054688,
      "loss": 0.3452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19345590472221375,
      "rewards/margins": 1.5196313858032227,
      "rewards/rejected": -1.7130872011184692,
      "step": 5193
    },
    {
      "epoch": 0.6796761266050544,
      "grad_norm": 5.669084548950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3564004898071289,
      "logits/rejected": 0.06818047910928726,
      "logps/chosen": -569.0045166015625,
      "logps/rejected": -490.25799560546875,
      "loss": 0.3551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24048271775245667,
      "rewards/margins": 1.2700059413909912,
      "rewards/rejected": -1.0295233726501465,
      "step": 5194
    },
    {
      "epoch": 0.6798069845424062,
      "grad_norm": 4.357198238372803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06122402846813202,
      "logits/rejected": -0.009381967596709728,
      "logps/chosen": -422.6291198730469,
      "logps/rejected": -399.9054870605469,
      "loss": 0.2493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06784134358167648,
      "rewards/margins": 1.8043606281280518,
      "rewards/rejected": -1.872201919555664,
      "step": 5195
    },
    {
      "epoch": 0.6799378424797579,
      "grad_norm": 7.219058513641357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11840110272169113,
      "logits/rejected": 0.12350542843341827,
      "logps/chosen": -351.17059326171875,
      "logps/rejected": -378.9901428222656,
      "loss": 0.4085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6041854619979858,
      "rewards/margins": 1.179384469985962,
      "rewards/rejected": -1.7835698127746582,
      "step": 5196
    },
    {
      "epoch": 0.6800687004171097,
      "grad_norm": 9.455032348632812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09092499315738678,
      "logits/rejected": -0.12703922390937805,
      "logps/chosen": -377.610595703125,
      "logps/rejected": -407.0433349609375,
      "loss": 0.5603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7975765466690063,
      "rewards/margins": 0.9755465984344482,
      "rewards/rejected": -1.7731231451034546,
      "step": 5197
    },
    {
      "epoch": 0.6801995583544614,
      "grad_norm": 10.892577171325684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13913196325302124,
      "logits/rejected": 0.005084700882434845,
      "logps/chosen": -596.034912109375,
      "logps/rejected": -533.6022338867188,
      "loss": 0.6516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8405309915542603,
      "rewards/margins": 0.42096370458602905,
      "rewards/rejected": -1.2614946365356445,
      "step": 5198
    },
    {
      "epoch": 0.6803304162918132,
      "grad_norm": 9.853713989257812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12396551668643951,
      "logits/rejected": 0.21109813451766968,
      "logps/chosen": -361.130126953125,
      "logps/rejected": -402.05560302734375,
      "loss": 0.6401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4778387248516083,
      "rewards/margins": 0.919681191444397,
      "rewards/rejected": -1.3975199460983276,
      "step": 5199
    },
    {
      "epoch": 0.6804612742291649,
      "grad_norm": 5.994625091552734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15459489822387695,
      "logits/rejected": -0.03506951034069061,
      "logps/chosen": -364.99957275390625,
      "logps/rejected": -354.21124267578125,
      "loss": 0.2749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8091720342636108,
      "rewards/margins": 1.6450941562652588,
      "rewards/rejected": -2.454266309738159,
      "step": 5200
    },
    {
      "epoch": 0.6805921321665167,
      "grad_norm": 6.713869094848633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.049475476145744324,
      "logits/rejected": 0.04805363342165947,
      "logps/chosen": -277.12628173828125,
      "logps/rejected": -375.9568786621094,
      "loss": 0.4665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07732335478067398,
      "rewards/margins": 1.0312221050262451,
      "rewards/rejected": -1.1085455417633057,
      "step": 5201
    },
    {
      "epoch": 0.6807229901038685,
      "grad_norm": 13.426976203918457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42598631978034973,
      "logits/rejected": 0.24795660376548767,
      "logps/chosen": -604.3375854492188,
      "logps/rejected": -627.12841796875,
      "loss": 1.076,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7470806241035461,
      "rewards/margins": 0.13922077417373657,
      "rewards/rejected": -0.8863013982772827,
      "step": 5202
    },
    {
      "epoch": 0.6808538480412203,
      "grad_norm": 6.660983085632324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09869430959224701,
      "logits/rejected": 0.05961642414331436,
      "logps/chosen": -526.2265014648438,
      "logps/rejected": -447.8360595703125,
      "loss": 0.5332,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6570912599563599,
      "rewards/margins": 0.6809288263320923,
      "rewards/rejected": -1.3380202054977417,
      "step": 5203
    },
    {
      "epoch": 0.680984705978572,
      "grad_norm": 8.041563034057617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08057940006256104,
      "logits/rejected": -0.13723739981651306,
      "logps/chosen": -377.5981140136719,
      "logps/rejected": -369.34320068359375,
      "loss": 0.486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8224983215332031,
      "rewards/margins": 0.8593177795410156,
      "rewards/rejected": -1.6818161010742188,
      "step": 5204
    },
    {
      "epoch": 0.6811155639159238,
      "grad_norm": 4.805176258087158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21558822691440582,
      "logits/rejected": 0.01285870373249054,
      "logps/chosen": -461.11199951171875,
      "logps/rejected": -477.0259094238281,
      "loss": 0.2137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39934730529785156,
      "rewards/margins": 1.783254861831665,
      "rewards/rejected": -2.1826024055480957,
      "step": 5205
    },
    {
      "epoch": 0.6812464218532756,
      "grad_norm": 7.230103969573975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18167519569396973,
      "logits/rejected": -0.07167170941829681,
      "logps/chosen": -413.4972839355469,
      "logps/rejected": -455.0054626464844,
      "loss": 0.572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.165528655052185,
      "rewards/margins": 0.6581416130065918,
      "rewards/rejected": -1.8236702680587769,
      "step": 5206
    },
    {
      "epoch": 0.6813772797906273,
      "grad_norm": 5.228188991546631,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10386193543672562,
      "logits/rejected": -0.16579264402389526,
      "logps/chosen": -487.627685546875,
      "logps/rejected": -554.5545654296875,
      "loss": 0.293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08681695908308029,
      "rewards/margins": 2.0931169986724854,
      "rewards/rejected": -2.0062999725341797,
      "step": 5207
    },
    {
      "epoch": 0.681508137727979,
      "grad_norm": 5.383928298950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00021313130855560303,
      "logits/rejected": -0.029422752559185028,
      "logps/chosen": -504.5326232910156,
      "logps/rejected": -396.6837463378906,
      "loss": 0.4578,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2171218395233154,
      "rewards/margins": 0.8959084153175354,
      "rewards/rejected": -2.113030195236206,
      "step": 5208
    },
    {
      "epoch": 0.6816389956653308,
      "grad_norm": 12.585033416748047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08850936591625214,
      "logits/rejected": -0.27971503138542175,
      "logps/chosen": -376.4913330078125,
      "logps/rejected": -353.5734558105469,
      "loss": 0.7826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4062693119049072,
      "rewards/margins": 0.8531241416931152,
      "rewards/rejected": -2.2593936920166016,
      "step": 5209
    },
    {
      "epoch": 0.6817698536026826,
      "grad_norm": 7.276096820831299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14896321296691895,
      "logits/rejected": 0.038103099912405014,
      "logps/chosen": -431.814453125,
      "logps/rejected": -378.68438720703125,
      "loss": 0.6413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.872772216796875,
      "rewards/margins": 0.6251662969589233,
      "rewards/rejected": -1.4979385137557983,
      "step": 5210
    },
    {
      "epoch": 0.6819007115400344,
      "grad_norm": 12.91381549835205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18183302879333496,
      "logits/rejected": -0.32300448417663574,
      "logps/chosen": -426.8161926269531,
      "logps/rejected": -311.0373229980469,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8312814235687256,
      "rewards/margins": 0.21289387345314026,
      "rewards/rejected": -1.044175386428833,
      "step": 5211
    },
    {
      "epoch": 0.6820315694773861,
      "grad_norm": 8.81730842590332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0958944708108902,
      "logits/rejected": 0.07053201645612717,
      "logps/chosen": -560.90673828125,
      "logps/rejected": -505.93408203125,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49667567014694214,
      "rewards/margins": 0.6718317270278931,
      "rewards/rejected": -1.1685073375701904,
      "step": 5212
    },
    {
      "epoch": 0.6821624274147379,
      "grad_norm": 8.351134300231934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22680021822452545,
      "logits/rejected": 0.09498609602451324,
      "logps/chosen": -348.35064697265625,
      "logps/rejected": -346.869384765625,
      "loss": 0.8251,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4124481678009033,
      "rewards/margins": 0.008709549903869629,
      "rewards/rejected": -1.421157717704773,
      "step": 5213
    },
    {
      "epoch": 0.6822932853520897,
      "grad_norm": 4.309448719024658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09135708212852478,
      "logits/rejected": -0.19455918669700623,
      "logps/chosen": -320.2023620605469,
      "logps/rejected": -360.2248840332031,
      "loss": 0.3083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0878299474716187,
      "rewards/margins": 2.164435625076294,
      "rewards/rejected": -3.2522659301757812,
      "step": 5214
    },
    {
      "epoch": 0.6824241432894415,
      "grad_norm": 5.973432540893555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10205140709877014,
      "logits/rejected": -0.32549163699150085,
      "logps/chosen": -689.6549072265625,
      "logps/rejected": -635.77880859375,
      "loss": 0.3571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5715495944023132,
      "rewards/margins": 1.4376693964004517,
      "rewards/rejected": -0.8661197423934937,
      "step": 5215
    },
    {
      "epoch": 0.6825550012267931,
      "grad_norm": 8.70772647857666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019547395408153534,
      "logits/rejected": 0.03619494289159775,
      "logps/chosen": -445.2413635253906,
      "logps/rejected": -513.9231567382812,
      "loss": 0.6078,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.896497905254364,
      "rewards/margins": 1.0681520700454712,
      "rewards/rejected": -1.96465003490448,
      "step": 5216
    },
    {
      "epoch": 0.6826858591641449,
      "grad_norm": 8.772865295410156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1638263612985611,
      "logits/rejected": -0.18102936446666718,
      "logps/chosen": -488.94891357421875,
      "logps/rejected": -491.8552551269531,
      "loss": 0.5795,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34313279390335083,
      "rewards/margins": 0.9207055568695068,
      "rewards/rejected": -1.2638384103775024,
      "step": 5217
    },
    {
      "epoch": 0.6828167171014967,
      "grad_norm": 5.693717956542969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19183072447776794,
      "logits/rejected": 0.09840112924575806,
      "logps/chosen": -345.7052307128906,
      "logps/rejected": -397.85198974609375,
      "loss": 0.3389,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3384862542152405,
      "rewards/margins": 1.2243003845214844,
      "rewards/rejected": -1.5627866983413696,
      "step": 5218
    },
    {
      "epoch": 0.6829475750388484,
      "grad_norm": 7.100103378295898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01739444024860859,
      "logits/rejected": 0.03257415071129799,
      "logps/chosen": -427.9866027832031,
      "logps/rejected": -415.88458251953125,
      "loss": 0.6363,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7825474143028259,
      "rewards/margins": 0.6485996246337891,
      "rewards/rejected": -1.4311469793319702,
      "step": 5219
    },
    {
      "epoch": 0.6830784329762002,
      "grad_norm": 5.5080437660217285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07587374001741409,
      "logits/rejected": -0.06834699958562851,
      "logps/chosen": -299.6600036621094,
      "logps/rejected": -348.29534912109375,
      "loss": 0.5561,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1605820655822754,
      "rewards/margins": 0.8977831602096558,
      "rewards/rejected": -1.0583652257919312,
      "step": 5220
    },
    {
      "epoch": 0.683209290913552,
      "grad_norm": 6.962337493896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13808122277259827,
      "logits/rejected": 0.20631250739097595,
      "logps/chosen": -431.26617431640625,
      "logps/rejected": -404.608642578125,
      "loss": 0.4995,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4188874363899231,
      "rewards/margins": 1.1565157175064087,
      "rewards/rejected": -1.5754032135009766,
      "step": 5221
    },
    {
      "epoch": 0.6833401488509038,
      "grad_norm": 5.091192722320557,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12565293908119202,
      "logits/rejected": -0.00997290387749672,
      "logps/chosen": -425.5328674316406,
      "logps/rejected": -509.69970703125,
      "loss": 0.3005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.025268904864788055,
      "rewards/margins": 1.4464908838272095,
      "rewards/rejected": -1.4212220907211304,
      "step": 5222
    },
    {
      "epoch": 0.6834710067882555,
      "grad_norm": 5.806907653808594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04916182532906532,
      "logits/rejected": -0.027471913024783134,
      "logps/chosen": -650.3306884765625,
      "logps/rejected": -453.8355712890625,
      "loss": 0.3397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16763611137866974,
      "rewards/margins": 1.4551892280578613,
      "rewards/rejected": -1.287553071975708,
      "step": 5223
    },
    {
      "epoch": 0.6836018647256072,
      "grad_norm": 7.088200569152832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1626121699810028,
      "logits/rejected": 0.30359509587287903,
      "logps/chosen": -470.14422607421875,
      "logps/rejected": -450.8233642578125,
      "loss": 0.5414,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5670689344406128,
      "rewards/margins": 1.096718192100525,
      "rewards/rejected": -1.6637870073318481,
      "step": 5224
    },
    {
      "epoch": 0.683732722662959,
      "grad_norm": 4.474249839782715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3105509281158447,
      "logits/rejected": 0.3035062849521637,
      "logps/chosen": -319.7595520019531,
      "logps/rejected": -383.2652587890625,
      "loss": 0.2625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20970706641674042,
      "rewards/margins": 1.6398124694824219,
      "rewards/rejected": -1.8495192527770996,
      "step": 5225
    },
    {
      "epoch": 0.6838635806003108,
      "grad_norm": 3.8566484451293945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06167452782392502,
      "logits/rejected": 0.12206774204969406,
      "logps/chosen": -425.7706298828125,
      "logps/rejected": -536.841064453125,
      "loss": 0.1623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1873074769973755,
      "rewards/margins": 1.9081565141677856,
      "rewards/rejected": -2.095463991165161,
      "step": 5226
    },
    {
      "epoch": 0.6839944385376625,
      "grad_norm": 4.884247779846191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017835011705756187,
      "logits/rejected": -0.0659196674823761,
      "logps/chosen": -385.943603515625,
      "logps/rejected": -545.2771606445312,
      "loss": 0.3689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2405833601951599,
      "rewards/margins": 1.4772889614105225,
      "rewards/rejected": -1.7178723812103271,
      "step": 5227
    },
    {
      "epoch": 0.6841252964750143,
      "grad_norm": 6.526413917541504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15625660121440887,
      "logits/rejected": 0.03308785706758499,
      "logps/chosen": -464.36395263671875,
      "logps/rejected": -505.76513671875,
      "loss": 0.4279,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8327412009239197,
      "rewards/margins": 1.0050411224365234,
      "rewards/rejected": -1.837782382965088,
      "step": 5228
    },
    {
      "epoch": 0.6842561544123661,
      "grad_norm": 5.65090274810791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14936001598834991,
      "logits/rejected": -0.12241937220096588,
      "logps/chosen": -384.9020080566406,
      "logps/rejected": -486.8737487792969,
      "loss": 0.3098,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3814665377140045,
      "rewards/margins": 1.7727841138839722,
      "rewards/rejected": -2.1542506217956543,
      "step": 5229
    },
    {
      "epoch": 0.6843870123497179,
      "grad_norm": 5.218435287475586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1543160080909729,
      "logits/rejected": 0.12661278247833252,
      "logps/chosen": -660.2269897460938,
      "logps/rejected": -590.6626586914062,
      "loss": 0.3313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2499658614397049,
      "rewards/margins": 1.3810495138168335,
      "rewards/rejected": -1.1310837268829346,
      "step": 5230
    },
    {
      "epoch": 0.6845178702870696,
      "grad_norm": 4.052697658538818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09907996654510498,
      "logits/rejected": -0.4913523495197296,
      "logps/chosen": -541.4817504882812,
      "logps/rejected": -367.8181457519531,
      "loss": 0.2381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.44034916162490845,
      "rewards/margins": 1.636771559715271,
      "rewards/rejected": -2.077120780944824,
      "step": 5231
    },
    {
      "epoch": 0.6846487282244214,
      "grad_norm": 9.27220344543457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2692054212093353,
      "logits/rejected": 0.30531376600265503,
      "logps/chosen": -375.29083251953125,
      "logps/rejected": -318.182861328125,
      "loss": 0.6303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8494728207588196,
      "rewards/margins": 0.7793363332748413,
      "rewards/rejected": -1.6288090944290161,
      "step": 5232
    },
    {
      "epoch": 0.6847795861617731,
      "grad_norm": 5.985505104064941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12522649765014648,
      "logits/rejected": 0.13933196663856506,
      "logps/chosen": -621.4027099609375,
      "logps/rejected": -751.8870239257812,
      "loss": 0.4018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49980583786964417,
      "rewards/margins": 1.3826096057891846,
      "rewards/rejected": -1.882415533065796,
      "step": 5233
    },
    {
      "epoch": 0.6849104440991249,
      "grad_norm": 6.3647541999816895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1000594049692154,
      "logits/rejected": -0.04660482704639435,
      "logps/chosen": -606.277099609375,
      "logps/rejected": -479.63720703125,
      "loss": 0.3276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6272915005683899,
      "rewards/margins": 1.4887495040893555,
      "rewards/rejected": -0.8614580631256104,
      "step": 5234
    },
    {
      "epoch": 0.6850413020364766,
      "grad_norm": 10.660538673400879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06028270721435547,
      "logits/rejected": -0.0752541720867157,
      "logps/chosen": -599.414306640625,
      "logps/rejected": -589.0969848632812,
      "loss": 0.705,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5676555633544922,
      "rewards/margins": 0.5531421899795532,
      "rewards/rejected": -1.1207977533340454,
      "step": 5235
    },
    {
      "epoch": 0.6851721599738284,
      "grad_norm": 6.802700996398926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28093278408050537,
      "logits/rejected": 0.13087090849876404,
      "logps/chosen": -335.2948913574219,
      "logps/rejected": -410.3641357421875,
      "loss": 0.6617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6869039535522461,
      "rewards/margins": 0.3882030248641968,
      "rewards/rejected": -1.0751069784164429,
      "step": 5236
    },
    {
      "epoch": 0.6853030179111802,
      "grad_norm": 7.665887832641602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.204444020986557,
      "logits/rejected": -0.1727353036403656,
      "logps/chosen": -604.471923828125,
      "logps/rejected": -375.50286865234375,
      "loss": 0.5852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7814203500747681,
      "rewards/margins": 0.5178815126419067,
      "rewards/rejected": -1.2993019819259644,
      "step": 5237
    },
    {
      "epoch": 0.685433875848532,
      "grad_norm": 5.913730621337891,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19634443521499634,
      "logits/rejected": 0.016799306496977806,
      "logps/chosen": -395.67864990234375,
      "logps/rejected": -411.27764892578125,
      "loss": 0.3992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7936636209487915,
      "rewards/margins": 1.0596760511398315,
      "rewards/rejected": -1.853339672088623,
      "step": 5238
    },
    {
      "epoch": 0.6855647337858837,
      "grad_norm": 9.81069564819336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12038354575634003,
      "logits/rejected": -0.128400057554245,
      "logps/chosen": -558.71044921875,
      "logps/rejected": -490.252197265625,
      "loss": 0.568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2068587988615036,
      "rewards/margins": 1.4024510383605957,
      "rewards/rejected": -1.1955921649932861,
      "step": 5239
    },
    {
      "epoch": 0.6856955917232355,
      "grad_norm": 8.325868606567383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008404098451137543,
      "logits/rejected": 0.0020990371704101562,
      "logps/chosen": -466.8264465332031,
      "logps/rejected": -563.9212036132812,
      "loss": 0.6037,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07820091396570206,
      "rewards/margins": 1.1755975484848022,
      "rewards/rejected": -1.0973966121673584,
      "step": 5240
    },
    {
      "epoch": 0.6858264496605873,
      "grad_norm": 8.183879852294922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2695198655128479,
      "logits/rejected": 0.4367445409297943,
      "logps/chosen": -317.793212890625,
      "logps/rejected": -382.51171875,
      "loss": 0.6748,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9695240259170532,
      "rewards/margins": 0.5932276248931885,
      "rewards/rejected": -1.5627515316009521,
      "step": 5241
    },
    {
      "epoch": 0.6859573075979389,
      "grad_norm": 5.442204475402832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024201225489377975,
      "logits/rejected": -0.1761869490146637,
      "logps/chosen": -559.4608154296875,
      "logps/rejected": -540.8202514648438,
      "loss": 0.3229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21108777821063995,
      "rewards/margins": 1.7873210906982422,
      "rewards/rejected": -1.9984087944030762,
      "step": 5242
    },
    {
      "epoch": 0.6860881655352907,
      "grad_norm": 10.42838191986084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016538619995117188,
      "logits/rejected": -0.06989164650440216,
      "logps/chosen": -353.1095886230469,
      "logps/rejected": -365.6289367675781,
      "loss": 0.9212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4591362476348877,
      "rewards/margins": 0.3694307208061218,
      "rewards/rejected": -1.8285669088363647,
      "step": 5243
    },
    {
      "epoch": 0.6862190234726425,
      "grad_norm": 10.994431495666504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05889390408992767,
      "logits/rejected": -0.04822125658392906,
      "logps/chosen": -345.71575927734375,
      "logps/rejected": -418.6411437988281,
      "loss": 0.6437,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1303766965866089,
      "rewards/margins": 0.3923001289367676,
      "rewards/rejected": -1.5226765871047974,
      "step": 5244
    },
    {
      "epoch": 0.6863498814099943,
      "grad_norm": 8.4390230178833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10269379615783691,
      "logits/rejected": -0.05775214731693268,
      "logps/chosen": -723.8067626953125,
      "logps/rejected": -632.8236694335938,
      "loss": 0.5572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.305781751871109,
      "rewards/margins": 0.7420438528060913,
      "rewards/rejected": -1.0478256940841675,
      "step": 5245
    },
    {
      "epoch": 0.686480739347346,
      "grad_norm": 6.886566162109375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26210668683052063,
      "logits/rejected": 0.05822453647851944,
      "logps/chosen": -609.7716064453125,
      "logps/rejected": -643.5587158203125,
      "loss": 0.6207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.69460129737854,
      "rewards/margins": 1.018404245376587,
      "rewards/rejected": -1.7130054235458374,
      "step": 5246
    },
    {
      "epoch": 0.6866115972846978,
      "grad_norm": 8.324151039123535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043516889214515686,
      "logits/rejected": -0.1142033264040947,
      "logps/chosen": -395.3504333496094,
      "logps/rejected": -407.0597839355469,
      "loss": 0.6585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.749718189239502,
      "rewards/margins": 0.7875752449035645,
      "rewards/rejected": -1.5372931957244873,
      "step": 5247
    },
    {
      "epoch": 0.6867424552220496,
      "grad_norm": 8.876566886901855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.153996080160141,
      "logits/rejected": 0.021041139960289,
      "logps/chosen": -358.9489440917969,
      "logps/rejected": -494.833740234375,
      "loss": 0.5553,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8959737420082092,
      "rewards/margins": 0.8453071117401123,
      "rewards/rejected": -1.7412809133529663,
      "step": 5248
    },
    {
      "epoch": 0.6868733131594014,
      "grad_norm": 5.707967758178711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09270509332418442,
      "logits/rejected": -0.07239090651273727,
      "logps/chosen": -373.17510986328125,
      "logps/rejected": -454.07855224609375,
      "loss": 0.3137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24488431215286255,
      "rewards/margins": 1.4812188148498535,
      "rewards/rejected": -1.7261030673980713,
      "step": 5249
    },
    {
      "epoch": 0.687004171096753,
      "grad_norm": 8.729822158813477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07660258561372757,
      "logits/rejected": 0.09896545112133026,
      "logps/chosen": -776.1897583007812,
      "logps/rejected": -616.5753173828125,
      "loss": 0.615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9369684457778931,
      "rewards/margins": 0.9928423762321472,
      "rewards/rejected": -1.9298107624053955,
      "step": 5250
    },
    {
      "epoch": 0.6871350290341048,
      "grad_norm": 4.021740913391113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06465522944927216,
      "logits/rejected": 0.2582552433013916,
      "logps/chosen": -365.25396728515625,
      "logps/rejected": -457.41021728515625,
      "loss": 0.23,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3634730875492096,
      "rewards/margins": 1.9775030612945557,
      "rewards/rejected": -1.614029884338379,
      "step": 5251
    },
    {
      "epoch": 0.6872658869714566,
      "grad_norm": 7.659179210662842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2532169222831726,
      "logits/rejected": 0.044017717242240906,
      "logps/chosen": -541.4197998046875,
      "logps/rejected": -477.2008056640625,
      "loss": 0.5717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.630535900592804,
      "rewards/margins": 0.6982913017272949,
      "rewards/rejected": -1.3288272619247437,
      "step": 5252
    },
    {
      "epoch": 0.6873967449088084,
      "grad_norm": 6.0992536544799805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0713685005903244,
      "logits/rejected": 0.07650472223758698,
      "logps/chosen": -422.3703308105469,
      "logps/rejected": -576.025390625,
      "loss": 0.4574,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21197891235351562,
      "rewards/margins": 1.337440848350525,
      "rewards/rejected": -1.5494195222854614,
      "step": 5253
    },
    {
      "epoch": 0.6875276028461601,
      "grad_norm": 9.61310863494873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010056275874376297,
      "logits/rejected": -0.006886573508381844,
      "logps/chosen": -529.1397705078125,
      "logps/rejected": -507.0902404785156,
      "loss": 0.5493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6540645360946655,
      "rewards/margins": 1.1669319868087769,
      "rewards/rejected": -1.8209964036941528,
      "step": 5254
    },
    {
      "epoch": 0.6876584607835119,
      "grad_norm": 8.097651481628418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025307467207312584,
      "logits/rejected": -0.0568770095705986,
      "logps/chosen": -422.29345703125,
      "logps/rejected": -459.60693359375,
      "loss": 0.6052,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6570150852203369,
      "rewards/margins": 0.6888017654418945,
      "rewards/rejected": -1.3458168506622314,
      "step": 5255
    },
    {
      "epoch": 0.6877893187208637,
      "grad_norm": 8.218615531921387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07020115852355957,
      "logits/rejected": 0.04971180856227875,
      "logps/chosen": -387.8805236816406,
      "logps/rejected": -360.11151123046875,
      "loss": 0.5858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08590660989284515,
      "rewards/margins": 0.9650194644927979,
      "rewards/rejected": -0.8791128396987915,
      "step": 5256
    },
    {
      "epoch": 0.6879201766582155,
      "grad_norm": 10.395001411437988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15777024626731873,
      "logits/rejected": 0.003948293626308441,
      "logps/chosen": -318.3594055175781,
      "logps/rejected": -438.9375305175781,
      "loss": 0.574,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43892478942871094,
      "rewards/margins": 0.7200309634208679,
      "rewards/rejected": -1.1589558124542236,
      "step": 5257
    },
    {
      "epoch": 0.6880510345955672,
      "grad_norm": 10.311494827270508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16839587688446045,
      "logits/rejected": -0.24277518689632416,
      "logps/chosen": -672.321044921875,
      "logps/rejected": -375.7420959472656,
      "loss": 0.643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.330442190170288,
      "rewards/margins": 0.4345472455024719,
      "rewards/rejected": -1.7649893760681152,
      "step": 5258
    },
    {
      "epoch": 0.688181892532919,
      "grad_norm": 6.108273506164551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24748852849006653,
      "logits/rejected": -0.3059268891811371,
      "logps/chosen": -371.8519287109375,
      "logps/rejected": -449.32183837890625,
      "loss": 0.5437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5677741765975952,
      "rewards/margins": 0.6824613809585571,
      "rewards/rejected": -1.2502355575561523,
      "step": 5259
    },
    {
      "epoch": 0.6883127504702707,
      "grad_norm": 43.1685791015625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035354502499103546,
      "logits/rejected": -0.3445051610469818,
      "logps/chosen": -488.94110107421875,
      "logps/rejected": -397.83062744140625,
      "loss": 1.5608,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.236083745956421,
      "rewards/margins": -0.6183017492294312,
      "rewards/rejected": -1.6177818775177002,
      "step": 5260
    },
    {
      "epoch": 0.6884436084076224,
      "grad_norm": 8.88067626953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23469366133213043,
      "logits/rejected": -0.22470399737358093,
      "logps/chosen": -584.4197387695312,
      "logps/rejected": -354.6123046875,
      "loss": 0.5728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8473604321479797,
      "rewards/margins": 0.5480737686157227,
      "rewards/rejected": -1.3954342603683472,
      "step": 5261
    },
    {
      "epoch": 0.6885744663449742,
      "grad_norm": 10.85287857055664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2313264012336731,
      "logits/rejected": 0.17095758020877838,
      "logps/chosen": -545.4464111328125,
      "logps/rejected": -486.0307312011719,
      "loss": 1.1046,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3580738306045532,
      "rewards/margins": -0.005015939474105835,
      "rewards/rejected": -0.353057861328125,
      "step": 5262
    },
    {
      "epoch": 0.688705324282326,
      "grad_norm": 4.673874855041504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030081292614340782,
      "logits/rejected": 0.015244709327816963,
      "logps/chosen": -435.29473876953125,
      "logps/rejected": -357.9326477050781,
      "loss": 0.3329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1176425963640213,
      "rewards/margins": 1.6864274740219116,
      "rewards/rejected": -1.5687849521636963,
      "step": 5263
    },
    {
      "epoch": 0.6888361822196778,
      "grad_norm": 4.227728366851807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01641436666250229,
      "logits/rejected": -0.293628990650177,
      "logps/chosen": -631.224365234375,
      "logps/rejected": -448.58563232421875,
      "loss": 0.2434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27859288454055786,
      "rewards/margins": 1.7738553285598755,
      "rewards/rejected": -2.052448272705078,
      "step": 5264
    },
    {
      "epoch": 0.6889670401570295,
      "grad_norm": 10.246482849121094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1922338306903839,
      "logits/rejected": -0.10946404933929443,
      "logps/chosen": -635.4918212890625,
      "logps/rejected": -460.7496337890625,
      "loss": 0.5757,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9673072695732117,
      "rewards/margins": 0.6772609949111938,
      "rewards/rejected": -1.6445683240890503,
      "step": 5265
    },
    {
      "epoch": 0.6890978980943813,
      "grad_norm": 10.31712818145752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2415086030960083,
      "logits/rejected": -0.2846057415008545,
      "logps/chosen": -509.1015319824219,
      "logps/rejected": -449.93475341796875,
      "loss": 0.5636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.926944375038147,
      "rewards/margins": 0.8024656176567078,
      "rewards/rejected": -1.7294100522994995,
      "step": 5266
    },
    {
      "epoch": 0.6892287560317331,
      "grad_norm": 6.017200469970703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034355223178863525,
      "logits/rejected": 0.027756430208683014,
      "logps/chosen": -539.3635864257812,
      "logps/rejected": -560.2554321289062,
      "loss": 0.3668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3839142322540283,
      "rewards/margins": 1.652762770652771,
      "rewards/rejected": -1.2688486576080322,
      "step": 5267
    },
    {
      "epoch": 0.6893596139690849,
      "grad_norm": 7.884453296661377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08906345069408417,
      "logits/rejected": -0.018572837114334106,
      "logps/chosen": -390.02593994140625,
      "logps/rejected": -450.3874206542969,
      "loss": 0.5787,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1252317577600479,
      "rewards/margins": 0.7462305426597595,
      "rewards/rejected": -0.8714622259140015,
      "step": 5268
    },
    {
      "epoch": 0.6894904719064365,
      "grad_norm": 5.6683125495910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2779039144515991,
      "logits/rejected": -0.2363189160823822,
      "logps/chosen": -492.07171630859375,
      "logps/rejected": -451.9000244140625,
      "loss": 0.416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2817401885986328,
      "rewards/margins": 0.9525853991508484,
      "rewards/rejected": -1.2343255281448364,
      "step": 5269
    },
    {
      "epoch": 0.6896213298437883,
      "grad_norm": 10.205470085144043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029338765889406204,
      "logits/rejected": -0.23619148135185242,
      "logps/chosen": -529.0632934570312,
      "logps/rejected": -395.58624267578125,
      "loss": 0.6319,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5886795520782471,
      "rewards/margins": 0.5920646786689758,
      "rewards/rejected": -1.1807441711425781,
      "step": 5270
    },
    {
      "epoch": 0.6897521877811401,
      "grad_norm": 5.327991485595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02727137878537178,
      "logits/rejected": -0.08417756855487823,
      "logps/chosen": -504.1113586425781,
      "logps/rejected": -496.8243103027344,
      "loss": 0.4176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7401102185249329,
      "rewards/margins": 1.0270347595214844,
      "rewards/rejected": -1.767145037651062,
      "step": 5271
    },
    {
      "epoch": 0.6898830457184919,
      "grad_norm": 8.048744201660156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004561395384371281,
      "logits/rejected": 0.03369233012199402,
      "logps/chosen": -457.8721923828125,
      "logps/rejected": -487.69476318359375,
      "loss": 0.5041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1618175357580185,
      "rewards/margins": 1.090583086013794,
      "rewards/rejected": -1.2524006366729736,
      "step": 5272
    },
    {
      "epoch": 0.6900139036558436,
      "grad_norm": 6.583767414093018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1476297229528427,
      "logits/rejected": 0.017798669636249542,
      "logps/chosen": -586.704345703125,
      "logps/rejected": -420.53155517578125,
      "loss": 0.4602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21073150634765625,
      "rewards/margins": 0.9775718450546265,
      "rewards/rejected": -1.1883033514022827,
      "step": 5273
    },
    {
      "epoch": 0.6901447615931954,
      "grad_norm": 7.297778606414795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07041102647781372,
      "logits/rejected": 0.11647883802652359,
      "logps/chosen": -302.9036560058594,
      "logps/rejected": -312.1529235839844,
      "loss": 0.4695,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8529199361801147,
      "rewards/margins": 0.9439117312431335,
      "rewards/rejected": -1.796831727027893,
      "step": 5274
    },
    {
      "epoch": 0.6902756195305472,
      "grad_norm": 6.238838195800781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20523808896541595,
      "logits/rejected": 0.14937064051628113,
      "logps/chosen": -320.5183410644531,
      "logps/rejected": -581.9561767578125,
      "loss": 0.4172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5389421582221985,
      "rewards/margins": 1.6472384929656982,
      "rewards/rejected": -2.186180591583252,
      "step": 5275
    },
    {
      "epoch": 0.690406477467899,
      "grad_norm": 5.574509143829346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5651375651359558,
      "logits/rejected": -0.2820979356765747,
      "logps/chosen": -483.5539855957031,
      "logps/rejected": -462.0284423828125,
      "loss": 0.4087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49464669823646545,
      "rewards/margins": 1.1070795059204102,
      "rewards/rejected": -1.6017262935638428,
      "step": 5276
    },
    {
      "epoch": 0.6905373354052506,
      "grad_norm": 6.805235862731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23576843738555908,
      "logits/rejected": -0.2977822721004486,
      "logps/chosen": -331.9342041015625,
      "logps/rejected": -237.71554565429688,
      "loss": 0.5144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23736898601055145,
      "rewards/margins": 1.229812741279602,
      "rewards/rejected": -1.46718168258667,
      "step": 5277
    },
    {
      "epoch": 0.6906681933426024,
      "grad_norm": 7.5379862785339355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29446202516555786,
      "logits/rejected": 0.2529732882976532,
      "logps/chosen": -478.89068603515625,
      "logps/rejected": -558.0013427734375,
      "loss": 0.6027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3262140452861786,
      "rewards/margins": 0.5208231806755066,
      "rewards/rejected": -0.194609135389328,
      "step": 5278
    },
    {
      "epoch": 0.6907990512799542,
      "grad_norm": 14.57974910736084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1898295283317566,
      "logits/rejected": -0.27698081731796265,
      "logps/chosen": -366.7052001953125,
      "logps/rejected": -370.0311584472656,
      "loss": 0.8509,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5144864320755005,
      "rewards/margins": 0.5227402448654175,
      "rewards/rejected": -2.037226438522339,
      "step": 5279
    },
    {
      "epoch": 0.690929909217306,
      "grad_norm": 7.914735794067383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.032722096890211105,
      "logits/rejected": -0.17898818850517273,
      "logps/chosen": -391.06524658203125,
      "logps/rejected": -278.35565185546875,
      "loss": 0.4677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38939782977104187,
      "rewards/margins": 0.8409405946731567,
      "rewards/rejected": -1.2303383350372314,
      "step": 5280
    },
    {
      "epoch": 0.6910607671546577,
      "grad_norm": 7.473903656005859,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38135161995887756,
      "logits/rejected": 0.3383468687534332,
      "logps/chosen": -661.531982421875,
      "logps/rejected": -505.6771240234375,
      "loss": 0.5096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4188880920410156,
      "rewards/margins": 1.0824413299560547,
      "rewards/rejected": -1.5013294219970703,
      "step": 5281
    },
    {
      "epoch": 0.6911916250920095,
      "grad_norm": 7.2725830078125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1757211685180664,
      "logits/rejected": -0.08686866611242294,
      "logps/chosen": -347.1539611816406,
      "logps/rejected": -298.1325988769531,
      "loss": 0.5521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1357681304216385,
      "rewards/margins": 0.8491861820220947,
      "rewards/rejected": -0.9849542379379272,
      "step": 5282
    },
    {
      "epoch": 0.6913224830293613,
      "grad_norm": 6.708046913146973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03780345618724823,
      "logits/rejected": -0.01683611050248146,
      "logps/chosen": -431.68316650390625,
      "logps/rejected": -385.2337341308594,
      "loss": 0.4481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16934946179389954,
      "rewards/margins": 1.3428843021392822,
      "rewards/rejected": -1.173534870147705,
      "step": 5283
    },
    {
      "epoch": 0.691453340966713,
      "grad_norm": 5.912057399749756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15852287411689758,
      "logits/rejected": -0.26960253715515137,
      "logps/chosen": -346.68060302734375,
      "logps/rejected": -428.5091552734375,
      "loss": 0.414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.016406267881393433,
      "rewards/margins": 1.638300895690918,
      "rewards/rejected": -1.6218945980072021,
      "step": 5284
    },
    {
      "epoch": 0.6915841989040648,
      "grad_norm": 4.618724822998047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20085465908050537,
      "logits/rejected": 0.008613264188170433,
      "logps/chosen": -424.45770263671875,
      "logps/rejected": -432.7146911621094,
      "loss": 0.2673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2897239625453949,
      "rewards/margins": 1.507228970527649,
      "rewards/rejected": -1.2175049781799316,
      "step": 5285
    },
    {
      "epoch": 0.6917150568414165,
      "grad_norm": 11.876103401184082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030901972204446793,
      "logits/rejected": -0.09965866804122925,
      "logps/chosen": -380.0188293457031,
      "logps/rejected": -292.106689453125,
      "loss": 0.8615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8283591866493225,
      "rewards/margins": 0.4818456172943115,
      "rewards/rejected": -1.3102047443389893,
      "step": 5286
    },
    {
      "epoch": 0.6918459147787683,
      "grad_norm": 8.380817413330078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039244644343853,
      "logits/rejected": -0.1704595685005188,
      "logps/chosen": -602.2101440429688,
      "logps/rejected": -527.1114501953125,
      "loss": 0.6296,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.044268976897001266,
      "rewards/margins": 0.3921913206577301,
      "rewards/rejected": -0.43646031618118286,
      "step": 5287
    },
    {
      "epoch": 0.69197677271612,
      "grad_norm": 6.304755687713623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07403412461280823,
      "logits/rejected": 0.09323712438344955,
      "logps/chosen": -506.3564453125,
      "logps/rejected": -491.8013000488281,
      "loss": 0.3922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17705193161964417,
      "rewards/margins": 1.2122746706008911,
      "rewards/rejected": -1.3893266916275024,
      "step": 5288
    },
    {
      "epoch": 0.6921076306534718,
      "grad_norm": 9.441277503967285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11978844553232193,
      "logits/rejected": 0.06927276402711868,
      "logps/chosen": -433.0120544433594,
      "logps/rejected": -449.19647216796875,
      "loss": 0.452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.537834882736206,
      "rewards/margins": 1.3967325687408447,
      "rewards/rejected": -1.9345675706863403,
      "step": 5289
    },
    {
      "epoch": 0.6922384885908236,
      "grad_norm": 6.355869770050049,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06293371319770813,
      "logits/rejected": -0.22013916075229645,
      "logps/chosen": -404.4775390625,
      "logps/rejected": -349.64910888671875,
      "loss": 0.4042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1401350200176239,
      "rewards/margins": 1.1810940504074097,
      "rewards/rejected": -1.0409590005874634,
      "step": 5290
    },
    {
      "epoch": 0.6923693465281754,
      "grad_norm": 8.078731536865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01761331409215927,
      "logits/rejected": -0.14408233761787415,
      "logps/chosen": -303.4598388671875,
      "logps/rejected": -382.1621398925781,
      "loss": 0.8353,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4759180247783661,
      "rewards/margins": -0.028820812702178955,
      "rewards/rejected": -0.44709721207618713,
      "step": 5291
    },
    {
      "epoch": 0.6925002044655271,
      "grad_norm": 9.92989444732666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14065179228782654,
      "logits/rejected": 0.2136843502521515,
      "logps/chosen": -604.5098876953125,
      "logps/rejected": -447.1741027832031,
      "loss": 0.8535,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.27216053009033203,
      "rewards/margins": 0.681073784828186,
      "rewards/rejected": -0.408913254737854,
      "step": 5292
    },
    {
      "epoch": 0.6926310624028789,
      "grad_norm": 8.581769943237305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1931253969669342,
      "logits/rejected": -0.04918880760669708,
      "logps/chosen": -338.14422607421875,
      "logps/rejected": -381.26654052734375,
      "loss": 0.5999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3182031810283661,
      "rewards/margins": 0.7946270704269409,
      "rewards/rejected": -1.1128302812576294,
      "step": 5293
    },
    {
      "epoch": 0.6927619203402307,
      "grad_norm": 12.808972358703613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01836208999156952,
      "logits/rejected": 0.07804431766271591,
      "logps/chosen": -467.29302978515625,
      "logps/rejected": -593.8356323242188,
      "loss": 0.8836,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5761293768882751,
      "rewards/margins": -0.06345236301422119,
      "rewards/rejected": -0.512677013874054,
      "step": 5294
    },
    {
      "epoch": 0.6928927782775824,
      "grad_norm": 8.031856536865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31492161750793457,
      "logits/rejected": -0.30521920323371887,
      "logps/chosen": -478.7627868652344,
      "logps/rejected": -474.9599914550781,
      "loss": 0.5339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0394741296768188,
      "rewards/margins": 0.8506689071655273,
      "rewards/rejected": -1.8901430368423462,
      "step": 5295
    },
    {
      "epoch": 0.6930236362149341,
      "grad_norm": 6.206177711486816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018199607729911804,
      "logits/rejected": -0.020239852368831635,
      "logps/chosen": -458.46295166015625,
      "logps/rejected": -427.79901123046875,
      "loss": 0.5401,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18086358904838562,
      "rewards/margins": 0.6729324460029602,
      "rewards/rejected": -0.8537960052490234,
      "step": 5296
    },
    {
      "epoch": 0.6931544941522859,
      "grad_norm": 8.526658058166504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08922772109508514,
      "logits/rejected": -0.22044570744037628,
      "logps/chosen": -452.52215576171875,
      "logps/rejected": -630.740234375,
      "loss": 0.4455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19861489534378052,
      "rewards/margins": 1.4952013492584229,
      "rewards/rejected": -1.6938165426254272,
      "step": 5297
    },
    {
      "epoch": 0.6932853520896377,
      "grad_norm": 8.888457298278809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2203807383775711,
      "logits/rejected": -0.39443328976631165,
      "logps/chosen": -493.21514892578125,
      "logps/rejected": -366.8481750488281,
      "loss": 0.6956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6562172770500183,
      "rewards/margins": 0.7517641186714172,
      "rewards/rejected": -1.407981276512146,
      "step": 5298
    },
    {
      "epoch": 0.6934162100269895,
      "grad_norm": 10.50959587097168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0508221834897995,
      "logits/rejected": -0.14563243091106415,
      "logps/chosen": -733.9990234375,
      "logps/rejected": -636.7883911132812,
      "loss": 0.7373,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04527854919433594,
      "rewards/margins": 0.6868810653686523,
      "rewards/rejected": -0.6416025161743164,
      "step": 5299
    },
    {
      "epoch": 0.6935470679643412,
      "grad_norm": 18.02527618408203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025609396398067474,
      "logits/rejected": 0.01676301285624504,
      "logps/chosen": -516.2754516601562,
      "logps/rejected": -532.3973388671875,
      "loss": 0.9294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46927565336227417,
      "rewards/margins": 0.4170882999897003,
      "rewards/rejected": -0.8863639235496521,
      "step": 5300
    },
    {
      "epoch": 0.693677925901693,
      "grad_norm": 11.030963897705078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0036014486104249954,
      "logits/rejected": -0.25358256697654724,
      "logps/chosen": -529.9520874023438,
      "logps/rejected": -581.0372924804688,
      "loss": 0.7477,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04208182916045189,
      "rewards/margins": 0.3085487484931946,
      "rewards/rejected": -0.2664669156074524,
      "step": 5301
    },
    {
      "epoch": 0.6938087838390448,
      "grad_norm": 9.277127265930176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10135293751955032,
      "logits/rejected": -0.020574871450662613,
      "logps/chosen": -420.29437255859375,
      "logps/rejected": -327.42742919921875,
      "loss": 0.7221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3916158676147461,
      "rewards/margins": 0.7783514261245728,
      "rewards/rejected": -1.1699672937393188,
      "step": 5302
    },
    {
      "epoch": 0.6939396417763966,
      "grad_norm": 14.405532836914062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2846606373786926,
      "logits/rejected": -0.35457685589790344,
      "logps/chosen": -339.29620361328125,
      "logps/rejected": -554.9051513671875,
      "loss": 1.0494,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.056583285331726,
      "rewards/margins": 0.23884814977645874,
      "rewards/rejected": -1.29543137550354,
      "step": 5303
    },
    {
      "epoch": 0.6940704997137482,
      "grad_norm": 3.277214288711548,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24756477773189545,
      "logits/rejected": -0.11819274723529816,
      "logps/chosen": -237.97927856445312,
      "logps/rejected": -311.62420654296875,
      "loss": 0.3707,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3493780493736267,
      "rewards/margins": 1.1355149745941162,
      "rewards/rejected": -1.4848932027816772,
      "step": 5304
    },
    {
      "epoch": 0.6942013576511,
      "grad_norm": 5.927492618560791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025743959471583366,
      "logits/rejected": -0.047641828656196594,
      "logps/chosen": -508.09375,
      "logps/rejected": -589.6508178710938,
      "loss": 0.3331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3587896227836609,
      "rewards/margins": 1.2223029136657715,
      "rewards/rejected": -1.5810925960540771,
      "step": 5305
    },
    {
      "epoch": 0.6943322155884518,
      "grad_norm": 3.2243969440460205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3195715546607971,
      "logits/rejected": -0.2715129852294922,
      "logps/chosen": -381.0421142578125,
      "logps/rejected": -273.94830322265625,
      "loss": 0.2896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42971915006637573,
      "rewards/margins": 1.5679848194122314,
      "rewards/rejected": -1.1382657289505005,
      "step": 5306
    },
    {
      "epoch": 0.6944630735258035,
      "grad_norm": 7.949847221374512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06213969737291336,
      "logits/rejected": 0.06693577766418457,
      "logps/chosen": -417.1981201171875,
      "logps/rejected": -518.9732666015625,
      "loss": 0.5166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9080327749252319,
      "rewards/margins": 0.8565142154693604,
      "rewards/rejected": -1.7645471096038818,
      "step": 5307
    },
    {
      "epoch": 0.6945939314631553,
      "grad_norm": 14.294233322143555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2100187987089157,
      "logits/rejected": -0.17062902450561523,
      "logps/chosen": -434.28839111328125,
      "logps/rejected": -494.84808349609375,
      "loss": 1.2959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6510259509086609,
      "rewards/margins": -0.43257373571395874,
      "rewards/rejected": -0.21845225989818573,
      "step": 5308
    },
    {
      "epoch": 0.6947247894005071,
      "grad_norm": 10.360614776611328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1634734869003296,
      "logits/rejected": 0.05897656828165054,
      "logps/chosen": -508.2778015136719,
      "logps/rejected": -406.5440979003906,
      "loss": 0.8464,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0431534051895142,
      "rewards/margins": -0.14815154671669006,
      "rewards/rejected": -0.8950017690658569,
      "step": 5309
    },
    {
      "epoch": 0.6948556473378589,
      "grad_norm": 10.034891128540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10313761234283447,
      "logits/rejected": -0.2789529263973236,
      "logps/chosen": -430.64373779296875,
      "logps/rejected": -404.2890930175781,
      "loss": 0.6828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3919338583946228,
      "rewards/margins": 0.4684349298477173,
      "rewards/rejected": -0.8603687286376953,
      "step": 5310
    },
    {
      "epoch": 0.6949865052752106,
      "grad_norm": 7.6599884033203125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03736045956611633,
      "logits/rejected": 0.12139835953712463,
      "logps/chosen": -407.4928283691406,
      "logps/rejected": -533.2424926757812,
      "loss": 0.5521,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10679225623607635,
      "rewards/margins": 0.677939772605896,
      "rewards/rejected": -0.5711475014686584,
      "step": 5311
    },
    {
      "epoch": 0.6951173632125623,
      "grad_norm": 8.829991340637207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15009742975234985,
      "logits/rejected": -0.11689627170562744,
      "logps/chosen": -363.49981689453125,
      "logps/rejected": -469.97149658203125,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7128709554672241,
      "rewards/margins": 0.7282443046569824,
      "rewards/rejected": -1.441115140914917,
      "step": 5312
    },
    {
      "epoch": 0.6952482211499141,
      "grad_norm": 5.268552780151367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03817206621170044,
      "logits/rejected": -0.080022431910038,
      "logps/chosen": -626.339111328125,
      "logps/rejected": -396.3388977050781,
      "loss": 0.2829,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2691914141178131,
      "rewards/margins": 1.3846029043197632,
      "rewards/rejected": -1.653794527053833,
      "step": 5313
    },
    {
      "epoch": 0.6953790790872659,
      "grad_norm": 7.440009593963623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24226094782352448,
      "logits/rejected": 0.004740206524729729,
      "logps/chosen": -485.55914306640625,
      "logps/rejected": -374.30108642578125,
      "loss": 0.6143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48618069291114807,
      "rewards/margins": 0.4367593824863434,
      "rewards/rejected": -0.9229400753974915,
      "step": 5314
    },
    {
      "epoch": 0.6955099370246176,
      "grad_norm": 8.799225807189941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19025194644927979,
      "logits/rejected": -0.20988187193870544,
      "logps/chosen": -438.60858154296875,
      "logps/rejected": -313.20208740234375,
      "loss": 0.4233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5281427502632141,
      "rewards/margins": 1.9297502040863037,
      "rewards/rejected": -2.457892894744873,
      "step": 5315
    },
    {
      "epoch": 0.6956407949619694,
      "grad_norm": 7.648044586181641,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20235447585582733,
      "logits/rejected": 0.13747277855873108,
      "logps/chosen": -534.0381469726562,
      "logps/rejected": -549.8453369140625,
      "loss": 0.5315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27586573362350464,
      "rewards/margins": 0.7990423440933228,
      "rewards/rejected": -1.0749080181121826,
      "step": 5316
    },
    {
      "epoch": 0.6957716528993212,
      "grad_norm": 7.37006139755249,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10476671904325485,
      "logits/rejected": 0.05161437764763832,
      "logps/chosen": -568.69921875,
      "logps/rejected": -451.8587951660156,
      "loss": 0.4005,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07422085851430893,
      "rewards/margins": 1.5066845417022705,
      "rewards/rejected": -1.5809053182601929,
      "step": 5317
    },
    {
      "epoch": 0.695902510836673,
      "grad_norm": 9.044602394104004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11883403360843658,
      "logits/rejected": -0.07483045011758804,
      "logps/chosen": -541.1602783203125,
      "logps/rejected": -479.39849853515625,
      "loss": 0.8156,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5985321402549744,
      "rewards/margins": -0.07444535195827484,
      "rewards/rejected": -0.5240867733955383,
      "step": 5318
    },
    {
      "epoch": 0.6960333687740247,
      "grad_norm": 7.1686110496521,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13784871995449066,
      "logits/rejected": -0.044061098247766495,
      "logps/chosen": -428.373291015625,
      "logps/rejected": -461.7436828613281,
      "loss": 0.6204,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05444526672363281,
      "rewards/margins": 0.51810622215271,
      "rewards/rejected": -0.46366098523139954,
      "step": 5319
    },
    {
      "epoch": 0.6961642267113765,
      "grad_norm": 4.428394794464111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19334593415260315,
      "logits/rejected": -0.11651646345853806,
      "logps/chosen": -414.64837646484375,
      "logps/rejected": -394.94561767578125,
      "loss": 0.2851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5491158962249756,
      "rewards/margins": 1.4550293684005737,
      "rewards/rejected": -2.0041451454162598,
      "step": 5320
    },
    {
      "epoch": 0.6962950846487282,
      "grad_norm": 7.459418773651123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0018928635399788618,
      "logits/rejected": 0.048274919390678406,
      "logps/chosen": -273.0741271972656,
      "logps/rejected": -256.8544616699219,
      "loss": 0.5478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1779157519340515,
      "rewards/margins": 0.48094576597213745,
      "rewards/rejected": -0.658861517906189,
      "step": 5321
    },
    {
      "epoch": 0.69642594258608,
      "grad_norm": 10.194348335266113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11905941367149353,
      "logits/rejected": 0.010188020765781403,
      "logps/chosen": -379.8045349121094,
      "logps/rejected": -411.8595275878906,
      "loss": 0.6409,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6129644513130188,
      "rewards/margins": 0.34198933839797974,
      "rewards/rejected": -0.9549537897109985,
      "step": 5322
    },
    {
      "epoch": 0.6965568005234317,
      "grad_norm": 5.560659885406494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16580228507518768,
      "logits/rejected": -0.3757796287536621,
      "logps/chosen": -516.696533203125,
      "logps/rejected": -437.3666687011719,
      "loss": 0.3257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.011679932475090027,
      "rewards/margins": 1.470157504081726,
      "rewards/rejected": -1.481837511062622,
      "step": 5323
    },
    {
      "epoch": 0.6966876584607835,
      "grad_norm": 14.759791374206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10951940715312958,
      "logits/rejected": 0.06401630491018295,
      "logps/chosen": -556.2468872070312,
      "logps/rejected": -400.8597717285156,
      "loss": 0.9728,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7892502546310425,
      "rewards/margins": 0.4498555362224579,
      "rewards/rejected": -1.2391059398651123,
      "step": 5324
    },
    {
      "epoch": 0.6968185163981353,
      "grad_norm": 12.380719184875488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49739742279052734,
      "logits/rejected": -0.2004166841506958,
      "logps/chosen": -380.66583251953125,
      "logps/rejected": -433.9429931640625,
      "loss": 0.5124,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008468430489301682,
      "rewards/margins": 1.058807134628296,
      "rewards/rejected": -1.050338625907898,
      "step": 5325
    },
    {
      "epoch": 0.696949374335487,
      "grad_norm": 6.8422136306762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19726666808128357,
      "logits/rejected": 0.04209693521261215,
      "logps/chosen": -559.7709350585938,
      "logps/rejected": -399.17913818359375,
      "loss": 0.4525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3474315404891968,
      "rewards/margins": 0.806746780872345,
      "rewards/rejected": -1.1541783809661865,
      "step": 5326
    },
    {
      "epoch": 0.6970802322728388,
      "grad_norm": 10.371866226196289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07078243792057037,
      "logits/rejected": 0.035714633762836456,
      "logps/chosen": -363.97528076171875,
      "logps/rejected": -525.7459716796875,
      "loss": 0.8194,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3601927161216736,
      "rewards/margins": 0.2984127104282379,
      "rewards/rejected": -0.6586053967475891,
      "step": 5327
    },
    {
      "epoch": 0.6972110902101906,
      "grad_norm": 5.098017692565918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5639801621437073,
      "logits/rejected": -0.366801381111145,
      "logps/chosen": -689.1944580078125,
      "logps/rejected": -466.3526916503906,
      "loss": 0.3931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5193025469779968,
      "rewards/margins": 1.131028175354004,
      "rewards/rejected": -1.6503307819366455,
      "step": 5328
    },
    {
      "epoch": 0.6973419481475424,
      "grad_norm": 5.762172698974609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31934353709220886,
      "logits/rejected": -0.2632111608982086,
      "logps/chosen": -419.3626708984375,
      "logps/rejected": -341.0694580078125,
      "loss": 0.3691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3113832473754883,
      "rewards/margins": 1.1991623640060425,
      "rewards/rejected": -1.5105456113815308,
      "step": 5329
    },
    {
      "epoch": 0.697472806084894,
      "grad_norm": 10.220815658569336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.086204394698143,
      "logits/rejected": -0.012633733451366425,
      "logps/chosen": -326.0964050292969,
      "logps/rejected": -347.6112976074219,
      "loss": 0.8915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0546287298202515,
      "rewards/margins": -0.05645868182182312,
      "rewards/rejected": -0.9981701374053955,
      "step": 5330
    },
    {
      "epoch": 0.6976036640222458,
      "grad_norm": 3.088298797607422,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1939130425453186,
      "logits/rejected": 0.08542793989181519,
      "logps/chosen": -454.5816955566406,
      "logps/rejected": -419.187744140625,
      "loss": 0.2029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4157130718231201,
      "rewards/margins": 1.9633796215057373,
      "rewards/rejected": -1.5476664304733276,
      "step": 5331
    },
    {
      "epoch": 0.6977345219595976,
      "grad_norm": 5.428328990936279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08956257998943329,
      "logits/rejected": -0.18583008646965027,
      "logps/chosen": -343.9244689941406,
      "logps/rejected": -340.2554016113281,
      "loss": 0.4721,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.232672780752182,
      "rewards/margins": 0.8804138898849487,
      "rewards/rejected": -1.1130867004394531,
      "step": 5332
    },
    {
      "epoch": 0.6978653798969494,
      "grad_norm": 9.736763000488281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3169356882572174,
      "logits/rejected": 0.23713275790214539,
      "logps/chosen": -708.0519409179688,
      "logps/rejected": -482.550048828125,
      "loss": 0.5638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05997125059366226,
      "rewards/margins": 0.9434321522712708,
      "rewards/rejected": -1.0034034252166748,
      "step": 5333
    },
    {
      "epoch": 0.6979962378343011,
      "grad_norm": 7.846960067749023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.169528067111969,
      "logits/rejected": 0.06353417038917542,
      "logps/chosen": -322.8398132324219,
      "logps/rejected": -315.5349426269531,
      "loss": 0.86,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6619237065315247,
      "rewards/margins": 0.12670651078224182,
      "rewards/rejected": -0.7886302471160889,
      "step": 5334
    },
    {
      "epoch": 0.6981270957716529,
      "grad_norm": 3.7362334728240967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2877154052257538,
      "logits/rejected": -0.23064947128295898,
      "logps/chosen": -363.9295959472656,
      "logps/rejected": -404.3146057128906,
      "loss": 0.3075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5950451493263245,
      "rewards/margins": 1.4333730936050415,
      "rewards/rejected": -0.8383280038833618,
      "step": 5335
    },
    {
      "epoch": 0.6982579537090047,
      "grad_norm": 14.978008270263672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2771330177783966,
      "logits/rejected": -0.5004899501800537,
      "logps/chosen": -723.657470703125,
      "logps/rejected": -728.3871459960938,
      "loss": 0.9109,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.496482491493225,
      "rewards/margins": 0.3359735608100891,
      "rewards/rejected": -1.832456111907959,
      "step": 5336
    },
    {
      "epoch": 0.6983888116463565,
      "grad_norm": 7.440913677215576,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06036844477057457,
      "logits/rejected": 0.009389298036694527,
      "logps/chosen": -311.7904052734375,
      "logps/rejected": -353.2060546875,
      "loss": 0.5051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22193071246147156,
      "rewards/margins": 0.8630715012550354,
      "rewards/rejected": -1.0850021839141846,
      "step": 5337
    },
    {
      "epoch": 0.6985196695837081,
      "grad_norm": 12.474543571472168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1683742105960846,
      "logits/rejected": 0.12716174125671387,
      "logps/chosen": -691.0291748046875,
      "logps/rejected": -515.13916015625,
      "loss": 1.0396,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03945945203304291,
      "rewards/margins": -0.496385395526886,
      "rewards/rejected": 0.4569259285926819,
      "step": 5338
    },
    {
      "epoch": 0.6986505275210599,
      "grad_norm": 8.066317558288574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06124110519886017,
      "logits/rejected": -0.20202572643756866,
      "logps/chosen": -555.9892578125,
      "logps/rejected": -423.394775390625,
      "loss": 0.4752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17564833164215088,
      "rewards/margins": 1.0894775390625,
      "rewards/rejected": -0.9138292074203491,
      "step": 5339
    },
    {
      "epoch": 0.6987813854584117,
      "grad_norm": 8.394421577453613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09672154486179352,
      "logits/rejected": -0.16554667055606842,
      "logps/chosen": -412.2606506347656,
      "logps/rejected": -511.431640625,
      "loss": 0.565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2825613021850586,
      "rewards/margins": 1.1115469932556152,
      "rewards/rejected": -1.394108533859253,
      "step": 5340
    },
    {
      "epoch": 0.6989122433957635,
      "grad_norm": 4.497807502746582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15944412350654602,
      "logits/rejected": 0.06928947567939758,
      "logps/chosen": -318.8143005371094,
      "logps/rejected": -340.88946533203125,
      "loss": 0.3054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6982017755508423,
      "rewards/margins": 1.6429909467697144,
      "rewards/rejected": -0.9447893500328064,
      "step": 5341
    },
    {
      "epoch": 0.6990431013331152,
      "grad_norm": 7.809135437011719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05887209251523018,
      "logits/rejected": -0.05782776698470116,
      "logps/chosen": -376.4403076171875,
      "logps/rejected": -487.1075744628906,
      "loss": 0.3865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6356658935546875,
      "rewards/margins": 1.076791763305664,
      "rewards/rejected": -1.7124577760696411,
      "step": 5342
    },
    {
      "epoch": 0.699173959270467,
      "grad_norm": 8.643874168395996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18408967554569244,
      "logits/rejected": 0.18545018136501312,
      "logps/chosen": -598.8216552734375,
      "logps/rejected": -653.1236572265625,
      "loss": 0.6144,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.835522472858429,
      "rewards/margins": 0.3831455111503601,
      "rewards/rejected": -1.218667984008789,
      "step": 5343
    },
    {
      "epoch": 0.6993048172078188,
      "grad_norm": 6.950796604156494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39869239926338196,
      "logits/rejected": 0.4634522497653961,
      "logps/chosen": -596.2239990234375,
      "logps/rejected": -418.2402038574219,
      "loss": 0.5263,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3372463285923004,
      "rewards/margins": 1.247249960899353,
      "rewards/rejected": -0.9100034832954407,
      "step": 5344
    },
    {
      "epoch": 0.6994356751451706,
      "grad_norm": 8.43760871887207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07691720128059387,
      "logits/rejected": -0.01984097808599472,
      "logps/chosen": -421.2919616699219,
      "logps/rejected": -440.86981201171875,
      "loss": 0.5779,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6392191052436829,
      "rewards/margins": 1.103851556777954,
      "rewards/rejected": -1.7430706024169922,
      "step": 5345
    },
    {
      "epoch": 0.6995665330825223,
      "grad_norm": 12.12806224822998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03349849581718445,
      "logits/rejected": -0.28655508160591125,
      "logps/chosen": -386.49444580078125,
      "logps/rejected": -466.24859619140625,
      "loss": 1.0041,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.743320107460022,
      "rewards/margins": 0.1559240221977234,
      "rewards/rejected": -0.8992441892623901,
      "step": 5346
    },
    {
      "epoch": 0.699697391019874,
      "grad_norm": 8.013341903686523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24925699830055237,
      "logits/rejected": -0.1926109939813614,
      "logps/chosen": -571.0805053710938,
      "logps/rejected": -597.74072265625,
      "loss": 0.3289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5967836380004883,
      "rewards/margins": 1.8179632425308228,
      "rewards/rejected": -1.2211796045303345,
      "step": 5347
    },
    {
      "epoch": 0.6998282489572258,
      "grad_norm": 6.588342189788818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4302557110786438,
      "logits/rejected": -0.369512677192688,
      "logps/chosen": -327.28729248046875,
      "logps/rejected": -336.6590270996094,
      "loss": 0.4034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6222387552261353,
      "rewards/margins": 1.058413028717041,
      "rewards/rejected": -1.6806516647338867,
      "step": 5348
    },
    {
      "epoch": 0.6999591068945775,
      "grad_norm": 8.770639419555664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14145541191101074,
      "logits/rejected": 0.0685531347990036,
      "logps/chosen": -393.99114990234375,
      "logps/rejected": -367.867919921875,
      "loss": 0.5444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.221602663397789,
      "rewards/margins": 0.8766956329345703,
      "rewards/rejected": -0.6550930142402649,
      "step": 5349
    },
    {
      "epoch": 0.7000899648319293,
      "grad_norm": 6.8312554359436035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19369015097618103,
      "logits/rejected": -0.33405303955078125,
      "logps/chosen": -389.74188232421875,
      "logps/rejected": -560.295654296875,
      "loss": 0.4388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5612159967422485,
      "rewards/margins": 0.9307787418365479,
      "rewards/rejected": -1.4919947385787964,
      "step": 5350
    },
    {
      "epoch": 0.7002208227692811,
      "grad_norm": 6.561341762542725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11311416327953339,
      "logits/rejected": 0.11119823902845383,
      "logps/chosen": -395.9112548828125,
      "logps/rejected": -514.6103515625,
      "loss": 0.4201,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1330888569355011,
      "rewards/margins": 1.0492544174194336,
      "rewards/rejected": -0.9161654710769653,
      "step": 5351
    },
    {
      "epoch": 0.7003516807066329,
      "grad_norm": 8.657323837280273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29352036118507385,
      "logits/rejected": -0.09401006251573563,
      "logps/chosen": -419.21160888671875,
      "logps/rejected": -449.3782958984375,
      "loss": 0.8058,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8995967507362366,
      "rewards/margins": 0.5489529371261597,
      "rewards/rejected": -1.4485496282577515,
      "step": 5352
    },
    {
      "epoch": 0.7004825386439846,
      "grad_norm": 9.059147834777832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2361951321363449,
      "logits/rejected": 0.21500855684280396,
      "logps/chosen": -472.30841064453125,
      "logps/rejected": -504.4884033203125,
      "loss": 0.9254,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43247488141059875,
      "rewards/margins": 0.015058130025863647,
      "rewards/rejected": -0.4475330710411072,
      "step": 5353
    },
    {
      "epoch": 0.7006133965813364,
      "grad_norm": 12.729036331176758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18875479698181152,
      "logits/rejected": 0.08078548312187195,
      "logps/chosen": -385.64068603515625,
      "logps/rejected": -374.28118896484375,
      "loss": 1.0576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7479526996612549,
      "rewards/margins": 0.1653118133544922,
      "rewards/rejected": -0.9132644534111023,
      "step": 5354
    },
    {
      "epoch": 0.7007442545186882,
      "grad_norm": 10.908914566040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11848660558462143,
      "logits/rejected": 0.05694860219955444,
      "logps/chosen": -658.3573608398438,
      "logps/rejected": -547.0215454101562,
      "loss": 0.5587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5817773938179016,
      "rewards/margins": 0.7526960372924805,
      "rewards/rejected": -1.3344733715057373,
      "step": 5355
    },
    {
      "epoch": 0.70087511245604,
      "grad_norm": 6.139784812927246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1862497627735138,
      "logits/rejected": -0.06262281537055969,
      "logps/chosen": -448.0628967285156,
      "logps/rejected": -409.83953857421875,
      "loss": 0.339,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2675449550151825,
      "rewards/margins": 1.1891299486160278,
      "rewards/rejected": -0.9215848445892334,
      "step": 5356
    },
    {
      "epoch": 0.7010059703933916,
      "grad_norm": 10.289085388183594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.458578497171402,
      "logits/rejected": -0.30497652292251587,
      "logps/chosen": -341.89703369140625,
      "logps/rejected": -608.8072509765625,
      "loss": 0.8953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9703125953674316,
      "rewards/margins": -0.13563990592956543,
      "rewards/rejected": -0.834672749042511,
      "step": 5357
    },
    {
      "epoch": 0.7011368283307434,
      "grad_norm": 5.508246421813965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20311413705348969,
      "logits/rejected": 0.20256152749061584,
      "logps/chosen": -393.80560302734375,
      "logps/rejected": -524.1737060546875,
      "loss": 0.3164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35533350706100464,
      "rewards/margins": 1.5254396200180054,
      "rewards/rejected": -1.170106053352356,
      "step": 5358
    },
    {
      "epoch": 0.7012676862680952,
      "grad_norm": 7.537223815917969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049546875059604645,
      "logits/rejected": 0.04047369956970215,
      "logps/chosen": -484.6720886230469,
      "logps/rejected": -497.5024108886719,
      "loss": 0.5598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35639041662216187,
      "rewards/margins": 1.0340745449066162,
      "rewards/rejected": -1.3904646635055542,
      "step": 5359
    },
    {
      "epoch": 0.701398544205447,
      "grad_norm": 7.569166660308838,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07315508276224136,
      "logits/rejected": -0.15227988362312317,
      "logps/chosen": -656.9210815429688,
      "logps/rejected": -517.901611328125,
      "loss": 0.5528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020777691155672073,
      "rewards/margins": 1.0398443937301636,
      "rewards/rejected": -1.019066572189331,
      "step": 5360
    },
    {
      "epoch": 0.7015294021427987,
      "grad_norm": 6.185677528381348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10389813035726547,
      "logits/rejected": -0.1003875732421875,
      "logps/chosen": -596.7695922851562,
      "logps/rejected": -433.09747314453125,
      "loss": 0.395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7783377170562744,
      "rewards/margins": 0.9489758014678955,
      "rewards/rejected": -0.1706380993127823,
      "step": 5361
    },
    {
      "epoch": 0.7016602600801505,
      "grad_norm": 7.08810567855835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17214569449424744,
      "logits/rejected": -0.041939448565244675,
      "logps/chosen": -436.70037841796875,
      "logps/rejected": -457.3041687011719,
      "loss": 0.5448,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11721666902303696,
      "rewards/margins": 0.7526623010635376,
      "rewards/rejected": -0.8698789477348328,
      "step": 5362
    },
    {
      "epoch": 0.7017911180175023,
      "grad_norm": 8.90517520904541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0540701299905777,
      "logits/rejected": -0.08409222215414047,
      "logps/chosen": -404.22930908203125,
      "logps/rejected": -413.1900939941406,
      "loss": 0.7864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39731693267822266,
      "rewards/margins": 0.10467777401208878,
      "rewards/rejected": -0.5019947290420532,
      "step": 5363
    },
    {
      "epoch": 0.7019219759548541,
      "grad_norm": 8.002346992492676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44733431935310364,
      "logits/rejected": 0.22714561223983765,
      "logps/chosen": -454.2886047363281,
      "logps/rejected": -391.0372314453125,
      "loss": 0.511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2701479196548462,
      "rewards/margins": 0.8086594343185425,
      "rewards/rejected": -1.0788073539733887,
      "step": 5364
    },
    {
      "epoch": 0.7020528338922057,
      "grad_norm": 6.27659797668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13319383561611176,
      "logits/rejected": -0.07308951020240784,
      "logps/chosen": -491.9223937988281,
      "logps/rejected": -419.7920227050781,
      "loss": 0.3938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11606771498918533,
      "rewards/margins": 1.1889371871948242,
      "rewards/rejected": -1.0728696584701538,
      "step": 5365
    },
    {
      "epoch": 0.7021836918295575,
      "grad_norm": 7.8656744956970215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2377324253320694,
      "logits/rejected": -0.05859101563692093,
      "logps/chosen": -504.21783447265625,
      "logps/rejected": -451.4602355957031,
      "loss": 0.4633,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31606578826904297,
      "rewards/margins": 0.838361382484436,
      "rewards/rejected": -0.5222955942153931,
      "step": 5366
    },
    {
      "epoch": 0.7023145497669093,
      "grad_norm": 8.190816879272461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10503360629081726,
      "logits/rejected": -0.15102627873420715,
      "logps/chosen": -467.7417907714844,
      "logps/rejected": -322.2086181640625,
      "loss": 0.6595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7427666187286377,
      "rewards/margins": 0.3823930621147156,
      "rewards/rejected": -1.1251598596572876,
      "step": 5367
    },
    {
      "epoch": 0.7024454077042611,
      "grad_norm": 7.21642541885376,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28422120213508606,
      "logits/rejected": 0.06289564818143845,
      "logps/chosen": -485.49993896484375,
      "logps/rejected": -442.9711608886719,
      "loss": 0.583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11351563781499863,
      "rewards/margins": 0.4533451199531555,
      "rewards/rejected": -0.5668608546257019,
      "step": 5368
    },
    {
      "epoch": 0.7025762656416128,
      "grad_norm": 9.598971366882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1613970696926117,
      "logits/rejected": -0.23432090878486633,
      "logps/chosen": -284.34588623046875,
      "logps/rejected": -378.1431579589844,
      "loss": 0.7215,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16440880298614502,
      "rewards/margins": 0.4645158052444458,
      "rewards/rejected": -0.6289246082305908,
      "step": 5369
    },
    {
      "epoch": 0.7027071235789646,
      "grad_norm": 6.8263092041015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08301378041505814,
      "logits/rejected": -0.3856901228427887,
      "logps/chosen": -557.505859375,
      "logps/rejected": -436.3000793457031,
      "loss": 0.5058,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16372451186180115,
      "rewards/margins": 0.8646302223205566,
      "rewards/rejected": -1.0283546447753906,
      "step": 5370
    },
    {
      "epoch": 0.7028379815163164,
      "grad_norm": 6.010069370269775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10929850488901138,
      "logits/rejected": -0.4931008219718933,
      "logps/chosen": -517.9035034179688,
      "logps/rejected": -566.0751342773438,
      "loss": 0.2615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3036985695362091,
      "rewards/margins": 1.6667592525482178,
      "rewards/rejected": -1.970457911491394,
      "step": 5371
    },
    {
      "epoch": 0.7029688394536681,
      "grad_norm": 7.146902084350586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1569875180721283,
      "logits/rejected": -0.12879544496536255,
      "logps/chosen": -507.06060791015625,
      "logps/rejected": -423.5250549316406,
      "loss": 0.733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14533062279224396,
      "rewards/margins": 0.27196937799453735,
      "rewards/rejected": -0.4173000454902649,
      "step": 5372
    },
    {
      "epoch": 0.7030996973910199,
      "grad_norm": 8.58960247039795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07539525628089905,
      "logits/rejected": 0.045918744057416916,
      "logps/chosen": -491.33428955078125,
      "logps/rejected": -419.8623352050781,
      "loss": 0.6239,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3243391513824463,
      "rewards/margins": 0.7575073838233948,
      "rewards/rejected": -1.0818464756011963,
      "step": 5373
    },
    {
      "epoch": 0.7032305553283716,
      "grad_norm": 6.882262706756592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035706184804439545,
      "logits/rejected": 0.23147347569465637,
      "logps/chosen": -620.738525390625,
      "logps/rejected": -518.2486572265625,
      "loss": 0.4953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09308473020792007,
      "rewards/margins": 1.454512596130371,
      "rewards/rejected": -1.361427664756775,
      "step": 5374
    },
    {
      "epoch": 0.7033614132657234,
      "grad_norm": 6.635053634643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2674187421798706,
      "logits/rejected": 0.010618500411510468,
      "logps/chosen": -310.1917419433594,
      "logps/rejected": -390.5228271484375,
      "loss": 0.5917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21466636657714844,
      "rewards/margins": 0.8137001395225525,
      "rewards/rejected": -1.0283665657043457,
      "step": 5375
    },
    {
      "epoch": 0.7034922712030751,
      "grad_norm": 10.228157043457031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12370873242616653,
      "logits/rejected": -0.17613673210144043,
      "logps/chosen": -500.39813232421875,
      "logps/rejected": -461.7154846191406,
      "loss": 0.8715,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5716361999511719,
      "rewards/margins": 0.3541155457496643,
      "rewards/rejected": -0.925751805305481,
      "step": 5376
    },
    {
      "epoch": 0.7036231291404269,
      "grad_norm": 9.833330154418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11408576369285583,
      "logits/rejected": -0.16521655023097992,
      "logps/chosen": -320.71661376953125,
      "logps/rejected": -331.1265563964844,
      "loss": 0.8367,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6099947094917297,
      "rewards/margins": -0.051308680325746536,
      "rewards/rejected": -0.5586860775947571,
      "step": 5377
    },
    {
      "epoch": 0.7037539870777787,
      "grad_norm": 11.082355499267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0601784884929657,
      "logits/rejected": 0.14953070878982544,
      "logps/chosen": -363.96087646484375,
      "logps/rejected": -456.2783203125,
      "loss": 0.9246,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9239685535430908,
      "rewards/margins": -0.00043007731437683105,
      "rewards/rejected": -0.9235383868217468,
      "step": 5378
    },
    {
      "epoch": 0.7038848450151305,
      "grad_norm": 7.869922161102295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25399699807167053,
      "logits/rejected": 0.4970830976963043,
      "logps/chosen": -450.5939636230469,
      "logps/rejected": -451.1695556640625,
      "loss": 0.5743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2836202383041382,
      "rewards/margins": 0.4675582945346832,
      "rewards/rejected": -0.751178503036499,
      "step": 5379
    },
    {
      "epoch": 0.7040157029524822,
      "grad_norm": 6.0185699462890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14536626636981964,
      "logits/rejected": -0.031449951231479645,
      "logps/chosen": -427.7825012207031,
      "logps/rejected": -314.3507080078125,
      "loss": 0.4068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2156578004360199,
      "rewards/margins": 1.385960578918457,
      "rewards/rejected": -1.1703027486801147,
      "step": 5380
    },
    {
      "epoch": 0.704146560889834,
      "grad_norm": 6.00665283203125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2923995852470398,
      "logits/rejected": 0.3699164092540741,
      "logps/chosen": -578.447021484375,
      "logps/rejected": -498.6530456542969,
      "loss": 0.2925,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19107913970947266,
      "rewards/margins": 1.8772523403167725,
      "rewards/rejected": -2.068331480026245,
      "step": 5381
    },
    {
      "epoch": 0.7042774188271858,
      "grad_norm": 7.1350908279418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19012822210788727,
      "logits/rejected": -0.14126189053058624,
      "logps/chosen": -324.3168640136719,
      "logps/rejected": -490.3126525878906,
      "loss": 0.7375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.566684901714325,
      "rewards/margins": 0.31450575590133667,
      "rewards/rejected": -0.8811907172203064,
      "step": 5382
    },
    {
      "epoch": 0.7044082767645375,
      "grad_norm": 9.836654663085938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2564847767353058,
      "logits/rejected": -0.052417393773794174,
      "logps/chosen": -669.226806640625,
      "logps/rejected": -470.1505126953125,
      "loss": 0.6059,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.35676440596580505,
      "rewards/margins": 0.945628821849823,
      "rewards/rejected": -0.5888643264770508,
      "step": 5383
    },
    {
      "epoch": 0.7045391347018892,
      "grad_norm": 5.169053077697754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12407376617193222,
      "logits/rejected": 0.014603279531002045,
      "logps/chosen": -322.0203552246094,
      "logps/rejected": -473.4356994628906,
      "loss": 0.3749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07961394637823105,
      "rewards/margins": 1.6986924409866333,
      "rewards/rejected": -1.778306484222412,
      "step": 5384
    },
    {
      "epoch": 0.704669992639241,
      "grad_norm": 9.582653045654297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08335170149803162,
      "logits/rejected": 0.06091751530766487,
      "logps/chosen": -510.8770446777344,
      "logps/rejected": -515.1439208984375,
      "loss": 0.6259,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0474414825439453,
      "rewards/margins": 0.32430994510650635,
      "rewards/rejected": -1.3717515468597412,
      "step": 5385
    },
    {
      "epoch": 0.7048008505765928,
      "grad_norm": 6.571849822998047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11849550902843475,
      "logits/rejected": -0.15811163187026978,
      "logps/chosen": -396.9951171875,
      "logps/rejected": -359.0658264160156,
      "loss": 0.4779,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.23038579523563385,
      "rewards/margins": 0.8141279220581055,
      "rewards/rejected": -0.5837421417236328,
      "step": 5386
    },
    {
      "epoch": 0.7049317085139446,
      "grad_norm": 7.879774570465088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1967877596616745,
      "logits/rejected": -0.27447211742401123,
      "logps/chosen": -454.422607421875,
      "logps/rejected": -444.00897216796875,
      "loss": 0.6376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7036223411560059,
      "rewards/margins": 1.0626144409179688,
      "rewards/rejected": -1.7662367820739746,
      "step": 5387
    },
    {
      "epoch": 0.7050625664512963,
      "grad_norm": 5.718347072601318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1363430917263031,
      "logits/rejected": 0.02552017569541931,
      "logps/chosen": -330.4452209472656,
      "logps/rejected": -524.1168212890625,
      "loss": 0.4017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11919420957565308,
      "rewards/margins": 1.127048373222351,
      "rewards/rejected": -1.246242642402649,
      "step": 5388
    },
    {
      "epoch": 0.7051934243886481,
      "grad_norm": 8.541855812072754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14769117534160614,
      "logits/rejected": -0.20649483799934387,
      "logps/chosen": -383.0360107421875,
      "logps/rejected": -567.2372436523438,
      "loss": 0.6049,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3438487946987152,
      "rewards/margins": 1.0554522275924683,
      "rewards/rejected": -1.3993011713027954,
      "step": 5389
    },
    {
      "epoch": 0.7053242823259999,
      "grad_norm": 7.410342216491699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01548635121434927,
      "logits/rejected": 0.2430751621723175,
      "logps/chosen": -457.8214416503906,
      "logps/rejected": -480.41192626953125,
      "loss": 0.5427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48933547735214233,
      "rewards/margins": 0.4226188659667969,
      "rewards/rejected": -0.9119542837142944,
      "step": 5390
    },
    {
      "epoch": 0.7054551402633517,
      "grad_norm": 4.793086528778076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02916990965604782,
      "logits/rejected": -0.09466246515512466,
      "logps/chosen": -284.195556640625,
      "logps/rejected": -250.02415466308594,
      "loss": 0.4807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.356564998626709,
      "rewards/margins": 0.5914350152015686,
      "rewards/rejected": -0.9480000138282776,
      "step": 5391
    },
    {
      "epoch": 0.7055859982007033,
      "grad_norm": 9.123006820678711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12566377222537994,
      "logits/rejected": -0.13946279883384705,
      "logps/chosen": -437.4808349609375,
      "logps/rejected": -528.4622192382812,
      "loss": 0.5624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04246560111641884,
      "rewards/margins": 0.726249098777771,
      "rewards/rejected": -0.7687147259712219,
      "step": 5392
    },
    {
      "epoch": 0.7057168561380551,
      "grad_norm": 5.0160675048828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06759629398584366,
      "logits/rejected": 0.3076225817203522,
      "logps/chosen": -375.4132080078125,
      "logps/rejected": -408.8135986328125,
      "loss": 0.3096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14583149552345276,
      "rewards/margins": 1.3813927173614502,
      "rewards/rejected": -1.2355612516403198,
      "step": 5393
    },
    {
      "epoch": 0.7058477140754069,
      "grad_norm": 8.680999755859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18439511954784393,
      "logits/rejected": -0.3873896300792694,
      "logps/chosen": -389.4115905761719,
      "logps/rejected": -356.64642333984375,
      "loss": 0.7745,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5947599411010742,
      "rewards/margins": 0.21362414956092834,
      "rewards/rejected": -0.8083841800689697,
      "step": 5394
    },
    {
      "epoch": 0.7059785720127586,
      "grad_norm": 7.33700704574585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11712558567523956,
      "logits/rejected": -0.18295198678970337,
      "logps/chosen": -626.0515747070312,
      "logps/rejected": -566.9981689453125,
      "loss": 0.369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4714786112308502,
      "rewards/margins": 1.4628956317901611,
      "rewards/rejected": -1.934374451637268,
      "step": 5395
    },
    {
      "epoch": 0.7061094299501104,
      "grad_norm": 8.620305061340332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0022626370191574097,
      "logits/rejected": -0.2122572660446167,
      "logps/chosen": -433.47845458984375,
      "logps/rejected": -606.8081665039062,
      "loss": 0.4528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37918397784233093,
      "rewards/margins": 1.4048196077346802,
      "rewards/rejected": -1.784003496170044,
      "step": 5396
    },
    {
      "epoch": 0.7062402878874622,
      "grad_norm": 10.109850883483887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10157792270183563,
      "logits/rejected": 0.0177178755402565,
      "logps/chosen": -445.5197448730469,
      "logps/rejected": -357.0649108886719,
      "loss": 0.8216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5560062527656555,
      "rewards/margins": 0.5028254985809326,
      "rewards/rejected": -1.0588316917419434,
      "step": 5397
    },
    {
      "epoch": 0.706371145824814,
      "grad_norm": 7.771424293518066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3178940415382385,
      "logits/rejected": 0.16518433392047882,
      "logps/chosen": -446.5265808105469,
      "logps/rejected": -389.43841552734375,
      "loss": 0.55,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5475265383720398,
      "rewards/margins": 0.6979233026504517,
      "rewards/rejected": -1.2454499006271362,
      "step": 5398
    },
    {
      "epoch": 0.7065020037621657,
      "grad_norm": 5.497337818145752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1384120136499405,
      "logits/rejected": 0.35255494713783264,
      "logps/chosen": -475.0401306152344,
      "logps/rejected": -452.7004699707031,
      "loss": 0.3247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6390214562416077,
      "rewards/margins": 1.3364285230636597,
      "rewards/rejected": -0.6974071264266968,
      "step": 5399
    },
    {
      "epoch": 0.7066328616995174,
      "grad_norm": 13.27474594116211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01966923102736473,
      "logits/rejected": -0.044243305921554565,
      "logps/chosen": -740.0033569335938,
      "logps/rejected": -478.0251770019531,
      "loss": 0.6845,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10372984409332275,
      "rewards/margins": 0.6521685123443604,
      "rewards/rejected": -0.7558982968330383,
      "step": 5400
    },
    {
      "epoch": 0.7067637196368692,
      "grad_norm": 4.634669780731201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021543052047491074,
      "logits/rejected": -0.19678276777267456,
      "logps/chosen": -445.3089599609375,
      "logps/rejected": -341.5313720703125,
      "loss": 0.3578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06549321115016937,
      "rewards/margins": 1.3295831680297852,
      "rewards/rejected": -1.2640900611877441,
      "step": 5401
    },
    {
      "epoch": 0.706894577574221,
      "grad_norm": 6.8887481689453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05250854417681694,
      "logits/rejected": 0.06422063708305359,
      "logps/chosen": -577.6846313476562,
      "logps/rejected": -515.0972290039062,
      "loss": 0.4936,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.604144811630249,
      "rewards/margins": 0.628989040851593,
      "rewards/rejected": -1.2331339120864868,
      "step": 5402
    },
    {
      "epoch": 0.7070254355115727,
      "grad_norm": 10.889799118041992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013403177261352539,
      "logits/rejected": 0.0936204269528389,
      "logps/chosen": -558.2416381835938,
      "logps/rejected": -618.1958618164062,
      "loss": 0.8387,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39366400241851807,
      "rewards/margins": -0.10531100630760193,
      "rewards/rejected": -0.28835296630859375,
      "step": 5403
    },
    {
      "epoch": 0.7071562934489245,
      "grad_norm": 11.626490592956543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24461379647254944,
      "logits/rejected": -0.2692715525627136,
      "logps/chosen": -499.896484375,
      "logps/rejected": -506.0455017089844,
      "loss": 0.7082,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4083269238471985,
      "rewards/margins": 0.7908182144165039,
      "rewards/rejected": -1.1991450786590576,
      "step": 5404
    },
    {
      "epoch": 0.7072871513862763,
      "grad_norm": 6.70949125289917,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09298063069581985,
      "logits/rejected": -0.29009002447128296,
      "logps/chosen": -456.30975341796875,
      "logps/rejected": -377.65350341796875,
      "loss": 0.2296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5082796216011047,
      "rewards/margins": 2.0605337619781494,
      "rewards/rejected": -2.5688133239746094,
      "step": 5405
    },
    {
      "epoch": 0.7074180093236281,
      "grad_norm": 14.678651809692383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21317972242832184,
      "logits/rejected": -0.0021811509504914284,
      "logps/chosen": -792.3218994140625,
      "logps/rejected": -491.95965576171875,
      "loss": 0.5929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7989635467529297,
      "rewards/margins": 1.6728516817092896,
      "rewards/rejected": -2.4718151092529297,
      "step": 5406
    },
    {
      "epoch": 0.7075488672609798,
      "grad_norm": 9.62317180633545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11408297717571259,
      "logits/rejected": -0.2432180941104889,
      "logps/chosen": -725.1868286132812,
      "logps/rejected": -539.6140747070312,
      "loss": 0.5469,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3287174701690674,
      "rewards/margins": 0.9790655374526978,
      "rewards/rejected": -2.3077831268310547,
      "step": 5407
    },
    {
      "epoch": 0.7076797251983316,
      "grad_norm": 6.298984527587891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09069199860095978,
      "logits/rejected": 0.13523364067077637,
      "logps/chosen": -487.18817138671875,
      "logps/rejected": -407.919677734375,
      "loss": 0.3901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0956231951713562,
      "rewards/margins": 1.25944185256958,
      "rewards/rejected": -1.355065107345581,
      "step": 5408
    },
    {
      "epoch": 0.7078105831356833,
      "grad_norm": 10.51370906829834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2347867488861084,
      "logits/rejected": -0.14224188029766083,
      "logps/chosen": -438.11846923828125,
      "logps/rejected": -440.3028564453125,
      "loss": 0.6296,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41234731674194336,
      "rewards/margins": 0.5728991627693176,
      "rewards/rejected": -0.985246479511261,
      "step": 5409
    },
    {
      "epoch": 0.7079414410730351,
      "grad_norm": 5.796631336212158,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031167060136795044,
      "logits/rejected": -0.030675679445266724,
      "logps/chosen": -401.2566833496094,
      "logps/rejected": -440.3020324707031,
      "loss": 0.415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09991245716810226,
      "rewards/margins": 1.1496338844299316,
      "rewards/rejected": -1.2495462894439697,
      "step": 5410
    },
    {
      "epoch": 0.7080722990103868,
      "grad_norm": 8.378512382507324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1288275271654129,
      "logits/rejected": 0.007697895169258118,
      "logps/chosen": -394.01904296875,
      "logps/rejected": -368.01202392578125,
      "loss": 0.4747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08194200694561005,
      "rewards/margins": 1.1759004592895508,
      "rewards/rejected": -1.0939584970474243,
      "step": 5411
    },
    {
      "epoch": 0.7082031569477386,
      "grad_norm": 8.273974418640137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07447007298469543,
      "logits/rejected": -0.2156820148229599,
      "logps/chosen": -635.3192138671875,
      "logps/rejected": -531.7221069335938,
      "loss": 0.583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.27633267641067505,
      "rewards/margins": 0.567616879940033,
      "rewards/rejected": -0.2912842035293579,
      "step": 5412
    },
    {
      "epoch": 0.7083340148850904,
      "grad_norm": 5.449856758117676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0729612484574318,
      "logits/rejected": -0.1759961098432541,
      "logps/chosen": -521.88818359375,
      "logps/rejected": -679.0574340820312,
      "loss": 0.2737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46971434354782104,
      "rewards/margins": 2.5084006786346436,
      "rewards/rejected": -2.9781150817871094,
      "step": 5413
    },
    {
      "epoch": 0.7084648728224421,
      "grad_norm": 5.438948154449463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17919732630252838,
      "logits/rejected": -0.26564642786979675,
      "logps/chosen": -272.38775634765625,
      "logps/rejected": -306.9306640625,
      "loss": 0.496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26152288913726807,
      "rewards/margins": 1.1747907400131226,
      "rewards/rejected": -1.4363137483596802,
      "step": 5414
    },
    {
      "epoch": 0.7085957307597939,
      "grad_norm": 14.123044967651367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09464074671268463,
      "logits/rejected": 0.10213448107242584,
      "logps/chosen": -396.2969970703125,
      "logps/rejected": -416.6955871582031,
      "loss": 1.3079,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -1.227431058883667,
      "rewards/margins": -0.7234391570091248,
      "rewards/rejected": -0.5039918422698975,
      "step": 5415
    },
    {
      "epoch": 0.7087265886971457,
      "grad_norm": 9.638771057128906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47543150186538696,
      "logits/rejected": 0.21781164407730103,
      "logps/chosen": -502.4313659667969,
      "logps/rejected": -444.6370544433594,
      "loss": 0.6686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1794659048318863,
      "rewards/margins": 1.1117382049560547,
      "rewards/rejected": -1.2912039756774902,
      "step": 5416
    },
    {
      "epoch": 0.7088574466344975,
      "grad_norm": 5.483335018157959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00901968963444233,
      "logits/rejected": -0.24902641773223877,
      "logps/chosen": -359.789794921875,
      "logps/rejected": -384.8127746582031,
      "loss": 0.3612,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7157986164093018,
      "rewards/margins": 1.1363554000854492,
      "rewards/rejected": -1.8521541357040405,
      "step": 5417
    },
    {
      "epoch": 0.7089883045718491,
      "grad_norm": 5.046161651611328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2774604558944702,
      "logits/rejected": -0.07689543068408966,
      "logps/chosen": -369.7476501464844,
      "logps/rejected": -361.2718505859375,
      "loss": 0.2283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2685484290122986,
      "rewards/margins": 1.8567864894866943,
      "rewards/rejected": -1.5882381200790405,
      "step": 5418
    },
    {
      "epoch": 0.7091191625092009,
      "grad_norm": 7.471691131591797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11452432721853256,
      "logits/rejected": -0.07747262716293335,
      "logps/chosen": -439.9427795410156,
      "logps/rejected": -345.3255615234375,
      "loss": 0.5596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13767299056053162,
      "rewards/margins": 1.1809720993041992,
      "rewards/rejected": -1.0432991981506348,
      "step": 5419
    },
    {
      "epoch": 0.7092500204465527,
      "grad_norm": 5.27079963684082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0979541465640068,
      "logits/rejected": 0.15465742349624634,
      "logps/chosen": -537.1041259765625,
      "logps/rejected": -449.983642578125,
      "loss": 0.3193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23014602065086365,
      "rewards/margins": 1.0675137042999268,
      "rewards/rejected": -1.2976596355438232,
      "step": 5420
    },
    {
      "epoch": 0.7093808783839045,
      "grad_norm": 9.84031867980957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28135624527931213,
      "logits/rejected": 0.3363807499408722,
      "logps/chosen": -524.0166015625,
      "logps/rejected": -553.7117919921875,
      "loss": 0.9191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07270357012748718,
      "rewards/margins": 0.012997284531593323,
      "rewards/rejected": -0.08570080250501633,
      "step": 5421
    },
    {
      "epoch": 0.7095117363212562,
      "grad_norm": 6.8425798416137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29005855321884155,
      "logits/rejected": -0.11750717461109161,
      "logps/chosen": -427.8863525390625,
      "logps/rejected": -502.9214172363281,
      "loss": 0.4392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4345029890537262,
      "rewards/margins": 1.0298715829849243,
      "rewards/rejected": -0.5953686237335205,
      "step": 5422
    },
    {
      "epoch": 0.709642594258608,
      "grad_norm": 9.20198917388916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2261568158864975,
      "logits/rejected": 0.14352670311927795,
      "logps/chosen": -423.2543640136719,
      "logps/rejected": -591.5025634765625,
      "loss": 0.5153,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25975820422172546,
      "rewards/margins": 1.088584303855896,
      "rewards/rejected": -1.3483425378799438,
      "step": 5423
    },
    {
      "epoch": 0.7097734521959598,
      "grad_norm": 6.180551528930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18654265999794006,
      "logits/rejected": -0.13360688090324402,
      "logps/chosen": -713.0240478515625,
      "logps/rejected": -561.4885864257812,
      "loss": 0.2622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0774572342634201,
      "rewards/margins": 1.7703293561935425,
      "rewards/rejected": -1.847786784172058,
      "step": 5424
    },
    {
      "epoch": 0.7099043101333116,
      "grad_norm": 10.192596435546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022215710952878,
      "logits/rejected": 0.0007565133273601532,
      "logps/chosen": -390.7571716308594,
      "logps/rejected": -467.1981201171875,
      "loss": 0.9177,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0227844715118408,
      "rewards/margins": 0.349331259727478,
      "rewards/rejected": -1.3721156120300293,
      "step": 5425
    },
    {
      "epoch": 0.7100351680706632,
      "grad_norm": 5.478194713592529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22704416513442993,
      "logits/rejected": 0.05174170807003975,
      "logps/chosen": -430.9136047363281,
      "logps/rejected": -391.1122741699219,
      "loss": 0.4016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6315603852272034,
      "rewards/margins": 0.9069518446922302,
      "rewards/rejected": -1.5385122299194336,
      "step": 5426
    },
    {
      "epoch": 0.710166026008015,
      "grad_norm": 9.091182708740234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0908043384552002,
      "logits/rejected": -0.15491023659706116,
      "logps/chosen": -449.9718933105469,
      "logps/rejected": -439.24713134765625,
      "loss": 0.7813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8684085607528687,
      "rewards/margins": 0.46997666358947754,
      "rewards/rejected": -1.3383853435516357,
      "step": 5427
    },
    {
      "epoch": 0.7102968839453668,
      "grad_norm": 5.047056674957275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2189575433731079,
      "logits/rejected": -0.1478949934244156,
      "logps/chosen": -494.1056823730469,
      "logps/rejected": -457.7618713378906,
      "loss": 0.3071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28379061818122864,
      "rewards/margins": 1.2530206441879272,
      "rewards/rejected": -1.536811351776123,
      "step": 5428
    },
    {
      "epoch": 0.7104277418827186,
      "grad_norm": 3.4497013092041016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09650015830993652,
      "logits/rejected": 0.00925709679722786,
      "logps/chosen": -417.18780517578125,
      "logps/rejected": -368.7476501464844,
      "loss": 0.1579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5394853353500366,
      "rewards/margins": 2.410734176635742,
      "rewards/rejected": -1.871248722076416,
      "step": 5429
    },
    {
      "epoch": 0.7105585998200703,
      "grad_norm": 9.5350923538208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33340245485305786,
      "logits/rejected": -0.2729872465133667,
      "logps/chosen": -456.34674072265625,
      "logps/rejected": -413.0981750488281,
      "loss": 0.7349,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5798847079277039,
      "rewards/margins": 0.37784746289253235,
      "rewards/rejected": -0.9577322006225586,
      "step": 5430
    },
    {
      "epoch": 0.7106894577574221,
      "grad_norm": 10.25025463104248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0684303566813469,
      "logits/rejected": 0.1476038098335266,
      "logps/chosen": -296.22161865234375,
      "logps/rejected": -505.0960693359375,
      "loss": 0.9168,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6255804300308228,
      "rewards/margins": 0.1596025824546814,
      "rewards/rejected": -0.7851830124855042,
      "step": 5431
    },
    {
      "epoch": 0.7108203156947739,
      "grad_norm": 11.038894653320312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19675180315971375,
      "logits/rejected": 0.24983282387256622,
      "logps/chosen": -403.5237121582031,
      "logps/rejected": -385.41064453125,
      "loss": 0.7063,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21329174935817719,
      "rewards/margins": 0.8140701055526733,
      "rewards/rejected": -1.0273618698120117,
      "step": 5432
    },
    {
      "epoch": 0.7109511736321257,
      "grad_norm": 4.116122245788574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17609739303588867,
      "logits/rejected": -0.08623957633972168,
      "logps/chosen": -458.6516418457031,
      "logps/rejected": -316.6705322265625,
      "loss": 0.3205,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2446962147951126,
      "rewards/margins": 1.2823303937911987,
      "rewards/rejected": -1.0376341342926025,
      "step": 5433
    },
    {
      "epoch": 0.7110820315694774,
      "grad_norm": 8.420581817626953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07513921707868576,
      "logits/rejected": -0.18525384366512299,
      "logps/chosen": -591.1237182617188,
      "logps/rejected": -535.7544555664062,
      "loss": 0.4939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19622915983200073,
      "rewards/margins": 1.0757019519805908,
      "rewards/rejected": -1.2719311714172363,
      "step": 5434
    },
    {
      "epoch": 0.7112128895068291,
      "grad_norm": 11.719704627990723,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008117333054542542,
      "logits/rejected": 0.01875494047999382,
      "logps/chosen": -555.6969604492188,
      "logps/rejected": -632.8643188476562,
      "loss": 0.8143,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27700313925743103,
      "rewards/margins": -0.006047099828720093,
      "rewards/rejected": -0.27095603942871094,
      "step": 5435
    },
    {
      "epoch": 0.7113437474441809,
      "grad_norm": 4.779643535614014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004196299239993095,
      "logits/rejected": 0.023843998089432716,
      "logps/chosen": -503.1024169921875,
      "logps/rejected": -423.2362060546875,
      "loss": 0.2787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.019405946135520935,
      "rewards/margins": 1.7981984615325928,
      "rewards/rejected": -1.7787925004959106,
      "step": 5436
    },
    {
      "epoch": 0.7114746053815326,
      "grad_norm": 6.699634075164795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10893071442842484,
      "logits/rejected": -0.06198728457093239,
      "logps/chosen": -375.19720458984375,
      "logps/rejected": -382.653076171875,
      "loss": 0.4344,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.689643144607544,
      "rewards/margins": 1.2190909385681152,
      "rewards/rejected": -1.9087339639663696,
      "step": 5437
    },
    {
      "epoch": 0.7116054633188844,
      "grad_norm": 6.623092174530029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17297378182411194,
      "logits/rejected": 0.003187529742717743,
      "logps/chosen": -331.5779724121094,
      "logps/rejected": -352.1889953613281,
      "loss": 0.5468,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7308159470558167,
      "rewards/margins": 0.4736255705356598,
      "rewards/rejected": -1.2044414281845093,
      "step": 5438
    },
    {
      "epoch": 0.7117363212562362,
      "grad_norm": 6.472012519836426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22418534755706787,
      "logits/rejected": 0.14835186302661896,
      "logps/chosen": -521.7898559570312,
      "logps/rejected": -412.06573486328125,
      "loss": 0.4386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39828112721443176,
      "rewards/margins": 0.9490984678268433,
      "rewards/rejected": -1.3473795652389526,
      "step": 5439
    },
    {
      "epoch": 0.711867179193588,
      "grad_norm": 6.386105060577393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03125942498445511,
      "logits/rejected": -0.05558886379003525,
      "logps/chosen": -872.0868530273438,
      "logps/rejected": -647.4714965820312,
      "loss": 0.4548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8767067193984985,
      "rewards/margins": 0.8379729986190796,
      "rewards/rejected": 0.038733676075935364,
      "step": 5440
    },
    {
      "epoch": 0.7119980371309397,
      "grad_norm": 8.477668762207031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1221693903207779,
      "logits/rejected": 0.12768273055553436,
      "logps/chosen": -360.6556091308594,
      "logps/rejected": -285.9749450683594,
      "loss": 0.7959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.116762399673462,
      "rewards/margins": 0.3489423990249634,
      "rewards/rejected": -1.4657047986984253,
      "step": 5441
    },
    {
      "epoch": 0.7121288950682915,
      "grad_norm": 9.190070152282715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09177158772945404,
      "logits/rejected": 0.15069158375263214,
      "logps/chosen": -340.9556884765625,
      "logps/rejected": -374.286376953125,
      "loss": 0.6538,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5830522775650024,
      "rewards/margins": 1.0556293725967407,
      "rewards/rejected": -1.6386816501617432,
      "step": 5442
    },
    {
      "epoch": 0.7122597530056433,
      "grad_norm": 14.560189247131348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10694071650505066,
      "logits/rejected": 0.008811788633465767,
      "logps/chosen": -483.8841552734375,
      "logps/rejected": -414.1398620605469,
      "loss": 0.7328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7751398682594299,
      "rewards/margins": 1.4788427352905273,
      "rewards/rejected": -2.2539825439453125,
      "step": 5443
    },
    {
      "epoch": 0.712390610942995,
      "grad_norm": 4.034416198730469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08052989840507507,
      "logits/rejected": -0.33703523874282837,
      "logps/chosen": -267.583740234375,
      "logps/rejected": -527.0330810546875,
      "loss": 0.2363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12937182188034058,
      "rewards/margins": 2.0183908939361572,
      "rewards/rejected": -1.8890190124511719,
      "step": 5444
    },
    {
      "epoch": 0.7125214688803467,
      "grad_norm": 12.057404518127441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.150347501039505,
      "logits/rejected": 0.10342917591333389,
      "logps/chosen": -458.8775939941406,
      "logps/rejected": -286.8148193359375,
      "loss": 0.7019,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6602855920791626,
      "rewards/margins": 0.3236568570137024,
      "rewards/rejected": -0.9839423894882202,
      "step": 5445
    },
    {
      "epoch": 0.7126523268176985,
      "grad_norm": 6.838479995727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05060174688696861,
      "logits/rejected": -0.0777624174952507,
      "logps/chosen": -457.80999755859375,
      "logps/rejected": -358.19415283203125,
      "loss": 0.5136,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5023237466812134,
      "rewards/margins": 0.9151334166526794,
      "rewards/rejected": -1.4174572229385376,
      "step": 5446
    },
    {
      "epoch": 0.7127831847550503,
      "grad_norm": 10.856661796569824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04976924508810043,
      "logits/rejected": -0.2504730224609375,
      "logps/chosen": -564.8590087890625,
      "logps/rejected": -418.6453552246094,
      "loss": 0.9291,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5195714235305786,
      "rewards/margins": 0.15084397792816162,
      "rewards/rejected": -1.6704152822494507,
      "step": 5447
    },
    {
      "epoch": 0.7129140426924021,
      "grad_norm": 5.445229530334473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15396741032600403,
      "logits/rejected": 0.1133744940161705,
      "logps/chosen": -329.57568359375,
      "logps/rejected": -379.15716552734375,
      "loss": 0.4658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5498449206352234,
      "rewards/margins": 0.7389445900917053,
      "rewards/rejected": -1.2887896299362183,
      "step": 5448
    },
    {
      "epoch": 0.7130449006297538,
      "grad_norm": 5.005159378051758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18523263931274414,
      "logits/rejected": -0.1545109748840332,
      "logps/chosen": -513.033935546875,
      "logps/rejected": -537.8758544921875,
      "loss": 0.4152,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19766351580619812,
      "rewards/margins": 0.8722255825996399,
      "rewards/rejected": -1.0698890686035156,
      "step": 5449
    },
    {
      "epoch": 0.7131757585671056,
      "grad_norm": 8.460380554199219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030728120356798172,
      "logits/rejected": -0.019462646916508675,
      "logps/chosen": -325.99169921875,
      "logps/rejected": -293.14410400390625,
      "loss": 0.661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3969366252422333,
      "rewards/margins": 0.18720190227031708,
      "rewards/rejected": -0.5841385126113892,
      "step": 5450
    },
    {
      "epoch": 0.7133066165044574,
      "grad_norm": 8.343657493591309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.038031965494155884,
      "logits/rejected": -0.08373942226171494,
      "logps/chosen": -574.7002563476562,
      "logps/rejected": -448.3524475097656,
      "loss": 0.6569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20923066139221191,
      "rewards/margins": 0.6758054494857788,
      "rewards/rejected": -0.8850361108779907,
      "step": 5451
    },
    {
      "epoch": 0.7134374744418092,
      "grad_norm": 6.318385124206543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0304434671998024,
      "logits/rejected": 0.011860482394695282,
      "logps/chosen": -510.1079406738281,
      "logps/rejected": -514.45068359375,
      "loss": 0.4656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6800400614738464,
      "rewards/margins": 0.6879539489746094,
      "rewards/rejected": -0.007913976907730103,
      "step": 5452
    },
    {
      "epoch": 0.7135683323791608,
      "grad_norm": 6.137302875518799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2817119061946869,
      "logits/rejected": 0.1703297644853592,
      "logps/chosen": -468.31842041015625,
      "logps/rejected": -436.07147216796875,
      "loss": 0.5763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3055454194545746,
      "rewards/margins": 0.9847447872161865,
      "rewards/rejected": -0.6791993975639343,
      "step": 5453
    },
    {
      "epoch": 0.7136991903165126,
      "grad_norm": 5.918217658996582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25810936093330383,
      "logits/rejected": 0.3548918068408966,
      "logps/chosen": -559.503662109375,
      "logps/rejected": -569.0322265625,
      "loss": 0.3997,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05391388386487961,
      "rewards/margins": 1.221358299255371,
      "rewards/rejected": -1.1674442291259766,
      "step": 5454
    },
    {
      "epoch": 0.7138300482538644,
      "grad_norm": 6.932978630065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02907959371805191,
      "logits/rejected": -0.15340927243232727,
      "logps/chosen": -594.7439575195312,
      "logps/rejected": -465.21673583984375,
      "loss": 0.3996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03388652205467224,
      "rewards/margins": 0.9621959924697876,
      "rewards/rejected": -0.9283095002174377,
      "step": 5455
    },
    {
      "epoch": 0.7139609061912162,
      "grad_norm": 4.163232326507568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09103678911924362,
      "logits/rejected": -0.21912838518619537,
      "logps/chosen": -346.243408203125,
      "logps/rejected": -349.27716064453125,
      "loss": 0.2973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10747833549976349,
      "rewards/margins": 1.637183666229248,
      "rewards/rejected": -1.7446619272232056,
      "step": 5456
    },
    {
      "epoch": 0.7140917641285679,
      "grad_norm": 6.886384963989258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08664048463106155,
      "logits/rejected": -0.23705033957958221,
      "logps/chosen": -320.8857421875,
      "logps/rejected": -740.5805053710938,
      "loss": 0.5413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37073570489883423,
      "rewards/margins": 1.0947372913360596,
      "rewards/rejected": -1.465472936630249,
      "step": 5457
    },
    {
      "epoch": 0.7142226220659197,
      "grad_norm": 6.998867988586426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09614773094654083,
      "logits/rejected": -0.10828886926174164,
      "logps/chosen": -580.4442749023438,
      "logps/rejected": -505.05291748046875,
      "loss": 0.499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07103443890810013,
      "rewards/margins": 0.6945286393165588,
      "rewards/rejected": -0.6234941482543945,
      "step": 5458
    },
    {
      "epoch": 0.7143534800032715,
      "grad_norm": 7.667038440704346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19000892341136932,
      "logits/rejected": -0.1708388477563858,
      "logps/chosen": -410.5376281738281,
      "logps/rejected": -460.99395751953125,
      "loss": 0.4502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8418673276901245,
      "rewards/margins": 1.1532036066055298,
      "rewards/rejected": -1.9950709342956543,
      "step": 5459
    },
    {
      "epoch": 0.7144843379406232,
      "grad_norm": 4.649738788604736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0145636647939682,
      "logits/rejected": -0.10125404596328735,
      "logps/chosen": -814.1920166015625,
      "logps/rejected": -769.7728271484375,
      "loss": 0.2112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38042372465133667,
      "rewards/margins": 2.015315532684326,
      "rewards/rejected": -1.6348919868469238,
      "step": 5460
    },
    {
      "epoch": 0.714615195877975,
      "grad_norm": 9.677349090576172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03729238361120224,
      "logits/rejected": -0.25211721658706665,
      "logps/chosen": -433.7755126953125,
      "logps/rejected": -464.757568359375,
      "loss": 0.7618,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.233052060008049,
      "rewards/margins": 0.8849700093269348,
      "rewards/rejected": -1.1180222034454346,
      "step": 5461
    },
    {
      "epoch": 0.7147460538153267,
      "grad_norm": 4.176022529602051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15518265962600708,
      "logits/rejected": 0.09838897734880447,
      "logps/chosen": -328.96380615234375,
      "logps/rejected": -335.74053955078125,
      "loss": 0.3843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1294388771057129,
      "rewards/margins": 0.9456602334976196,
      "rewards/rejected": -0.8162212371826172,
      "step": 5462
    },
    {
      "epoch": 0.7148769117526785,
      "grad_norm": 8.368584632873535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23544928431510925,
      "logits/rejected": 0.1516581028699875,
      "logps/chosen": -470.9244689941406,
      "logps/rejected": -452.3521728515625,
      "loss": 0.6373,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7162922024726868,
      "rewards/margins": 0.3314319849014282,
      "rewards/rejected": -1.0477241277694702,
      "step": 5463
    },
    {
      "epoch": 0.7150077696900302,
      "grad_norm": 4.506972789764404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04765515774488449,
      "logits/rejected": -0.18776243925094604,
      "logps/chosen": -542.009521484375,
      "logps/rejected": -443.5857238769531,
      "loss": 0.3301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5655859112739563,
      "rewards/margins": 1.2117682695388794,
      "rewards/rejected": -0.6461824774742126,
      "step": 5464
    },
    {
      "epoch": 0.715138627627382,
      "grad_norm": 9.249966621398926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13621467351913452,
      "logits/rejected": -0.09982659667730331,
      "logps/chosen": -407.45587158203125,
      "logps/rejected": -268.17572021484375,
      "loss": 0.7114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7058203220367432,
      "rewards/margins": 0.7378803491592407,
      "rewards/rejected": -1.4437006711959839,
      "step": 5465
    },
    {
      "epoch": 0.7152694855647338,
      "grad_norm": 6.317105770111084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2738839089870453,
      "logits/rejected": -0.2628243863582611,
      "logps/chosen": -413.627685546875,
      "logps/rejected": -547.1309204101562,
      "loss": 0.3081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04892117902636528,
      "rewards/margins": 1.4447410106658936,
      "rewards/rejected": -1.4936624765396118,
      "step": 5466
    },
    {
      "epoch": 0.7154003435020856,
      "grad_norm": 3.8924357891082764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0406104251742363,
      "logits/rejected": -0.06951646506786346,
      "logps/chosen": -293.57177734375,
      "logps/rejected": -364.7496032714844,
      "loss": 0.2331,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09348221123218536,
      "rewards/margins": 1.6874936819076538,
      "rewards/rejected": -1.5940115451812744,
      "step": 5467
    },
    {
      "epoch": 0.7155312014394373,
      "grad_norm": 6.569014072418213,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17759984731674194,
      "logits/rejected": 0.11468765884637833,
      "logps/chosen": -373.89532470703125,
      "logps/rejected": -335.0856628417969,
      "loss": 0.5606,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.656163215637207,
      "rewards/margins": 0.9752092957496643,
      "rewards/rejected": -1.6313724517822266,
      "step": 5468
    },
    {
      "epoch": 0.7156620593767891,
      "grad_norm": 8.003219604492188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3319425880908966,
      "logits/rejected": 0.10985776782035828,
      "logps/chosen": -374.7722473144531,
      "logps/rejected": -440.35919189453125,
      "loss": 0.5864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05570259317755699,
      "rewards/margins": 0.8481577634811401,
      "rewards/rejected": -0.903860330581665,
      "step": 5469
    },
    {
      "epoch": 0.7157929173141409,
      "grad_norm": 9.66103744506836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17748059332370758,
      "logits/rejected": -0.3046398162841797,
      "logps/chosen": -420.58837890625,
      "logps/rejected": -407.27874755859375,
      "loss": 0.7898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7330343723297119,
      "rewards/margins": 0.3569728434085846,
      "rewards/rejected": -1.0900071859359741,
      "step": 5470
    },
    {
      "epoch": 0.7159237752514926,
      "grad_norm": 5.166697025299072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0996812954545021,
      "logits/rejected": 0.016775373369455338,
      "logps/chosen": -423.83343505859375,
      "logps/rejected": -427.91217041015625,
      "loss": 0.3649,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42225152254104614,
      "rewards/margins": 1.268066644668579,
      "rewards/rejected": -1.6903181076049805,
      "step": 5471
    },
    {
      "epoch": 0.7160546331888443,
      "grad_norm": 6.392922878265381,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12019345164299011,
      "logits/rejected": 0.22899262607097626,
      "logps/chosen": -323.99859619140625,
      "logps/rejected": -367.466796875,
      "loss": 0.4315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.045023173093795776,
      "rewards/margins": 0.962577760219574,
      "rewards/rejected": -0.9175547361373901,
      "step": 5472
    },
    {
      "epoch": 0.7161854911261961,
      "grad_norm": 9.711075782775879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28971707820892334,
      "logits/rejected": 0.27939945459365845,
      "logps/chosen": -573.39697265625,
      "logps/rejected": -545.078857421875,
      "loss": 0.6474,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.32723772525787354,
      "rewards/margins": 0.39962270855903625,
      "rewards/rejected": -0.0723850205540657,
      "step": 5473
    },
    {
      "epoch": 0.7163163490635479,
      "grad_norm": 9.226401329040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2373816817998886,
      "logits/rejected": -0.11975210905075073,
      "logps/chosen": -342.9836730957031,
      "logps/rejected": -384.7375793457031,
      "loss": 0.8847,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2216355800628662,
      "rewards/margins": 0.00663679838180542,
      "rewards/rejected": -1.2282721996307373,
      "step": 5474
    },
    {
      "epoch": 0.7164472070008997,
      "grad_norm": 6.845535755157471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017243817448616028,
      "logits/rejected": -0.25333115458488464,
      "logps/chosen": -479.021240234375,
      "logps/rejected": -553.6845703125,
      "loss": 0.5376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3741287291049957,
      "rewards/margins": 0.5860588550567627,
      "rewards/rejected": -0.960187554359436,
      "step": 5475
    },
    {
      "epoch": 0.7165780649382514,
      "grad_norm": 8.805550575256348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14379775524139404,
      "logits/rejected": 0.09549205750226974,
      "logps/chosen": -677.22705078125,
      "logps/rejected": -695.134765625,
      "loss": 0.5507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7065445184707642,
      "rewards/margins": 1.1407501697540283,
      "rewards/rejected": -1.8472946882247925,
      "step": 5476
    },
    {
      "epoch": 0.7167089228756032,
      "grad_norm": 7.7708516120910645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19377963244915009,
      "logits/rejected": 0.033512357622385025,
      "logps/chosen": -480.45770263671875,
      "logps/rejected": -513.3479614257812,
      "loss": 0.379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3381277322769165,
      "rewards/margins": 0.9691910147666931,
      "rewards/rejected": -1.3073186874389648,
      "step": 5477
    },
    {
      "epoch": 0.716839780812955,
      "grad_norm": 7.263584136962891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06655639410018921,
      "logits/rejected": 0.029536517336964607,
      "logps/chosen": -574.7581787109375,
      "logps/rejected": -433.49237060546875,
      "loss": 0.4543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8809974789619446,
      "rewards/margins": 0.8708290457725525,
      "rewards/rejected": -1.7518266439437866,
      "step": 5478
    },
    {
      "epoch": 0.7169706387503066,
      "grad_norm": 7.0573530197143555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30139368772506714,
      "logits/rejected": -0.12236210703849792,
      "logps/chosen": -399.4002990722656,
      "logps/rejected": -445.913330078125,
      "loss": 0.6549,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06545867025852203,
      "rewards/margins": 1.406558871269226,
      "rewards/rejected": -1.3411002159118652,
      "step": 5479
    },
    {
      "epoch": 0.7171014966876584,
      "grad_norm": 7.330504417419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21398577094078064,
      "logits/rejected": -0.04162358120083809,
      "logps/chosen": -402.2853088378906,
      "logps/rejected": -337.477783203125,
      "loss": 0.6914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6662675738334656,
      "rewards/margins": 0.441938579082489,
      "rewards/rejected": -1.108206033706665,
      "step": 5480
    },
    {
      "epoch": 0.7172323546250102,
      "grad_norm": 7.919708251953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.205468088388443,
      "logits/rejected": 0.09732215851545334,
      "logps/chosen": -526.5752563476562,
      "logps/rejected": -577.9564819335938,
      "loss": 0.5262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22055549919605255,
      "rewards/margins": 1.3284047842025757,
      "rewards/rejected": -1.5489600896835327,
      "step": 5481
    },
    {
      "epoch": 0.717363212562362,
      "grad_norm": 6.571430206298828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1649811565876007,
      "logits/rejected": -0.20530188083648682,
      "logps/chosen": -463.8813171386719,
      "logps/rejected": -364.9595642089844,
      "loss": 0.627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7861467599868774,
      "rewards/margins": 0.3742193579673767,
      "rewards/rejected": -1.1603660583496094,
      "step": 5482
    },
    {
      "epoch": 0.7174940704997137,
      "grad_norm": 7.011377334594727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05233491584658623,
      "logits/rejected": -0.2728913724422455,
      "logps/chosen": -512.4830322265625,
      "logps/rejected": -510.0513916015625,
      "loss": 0.3622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45013540983200073,
      "rewards/margins": 1.0578422546386719,
      "rewards/rejected": -1.5079777240753174,
      "step": 5483
    },
    {
      "epoch": 0.7176249284370655,
      "grad_norm": 5.873408317565918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07957132160663605,
      "logits/rejected": 0.09451339393854141,
      "logps/chosen": -484.30718994140625,
      "logps/rejected": -471.5992431640625,
      "loss": 0.2669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4085232615470886,
      "rewards/margins": 1.4915516376495361,
      "rewards/rejected": -1.90007483959198,
      "step": 5484
    },
    {
      "epoch": 0.7177557863744173,
      "grad_norm": 12.221027374267578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00519929826259613,
      "logits/rejected": 0.16567808389663696,
      "logps/chosen": -475.0181579589844,
      "logps/rejected": -551.3358764648438,
      "loss": 0.9024,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4569522142410278,
      "rewards/margins": 0.4944904148578644,
      "rewards/rejected": -1.9514425992965698,
      "step": 5485
    },
    {
      "epoch": 0.7178866443117691,
      "grad_norm": 4.29997444152832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10035645216703415,
      "logits/rejected": 0.08173119276762009,
      "logps/chosen": -414.66705322265625,
      "logps/rejected": -343.2615966796875,
      "loss": 0.3185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20829257369041443,
      "rewards/margins": 1.2951958179473877,
      "rewards/rejected": -1.503488302230835,
      "step": 5486
    },
    {
      "epoch": 0.7180175022491208,
      "grad_norm": 5.147304058074951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035059500485658646,
      "logits/rejected": -0.04015684127807617,
      "logps/chosen": -401.2877197265625,
      "logps/rejected": -427.6000061035156,
      "loss": 0.3895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.032651882618665695,
      "rewards/margins": 0.8989776968955994,
      "rewards/rejected": -0.9316295981407166,
      "step": 5487
    },
    {
      "epoch": 0.7181483601864725,
      "grad_norm": 10.259562492370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06991145014762878,
      "logits/rejected": -0.2988898456096649,
      "logps/chosen": -616.4844360351562,
      "logps/rejected": -403.3099060058594,
      "loss": 0.7389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.001481272280216217,
      "rewards/margins": 0.7849116325378418,
      "rewards/rejected": -0.7834302186965942,
      "step": 5488
    },
    {
      "epoch": 0.7182792181238243,
      "grad_norm": 9.622699737548828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25941920280456543,
      "logits/rejected": 0.06233539059758186,
      "logps/chosen": -533.2280883789062,
      "logps/rejected": -497.52362060546875,
      "loss": 0.7979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3985862731933594,
      "rewards/margins": 0.1014002114534378,
      "rewards/rejected": 0.29718607664108276,
      "step": 5489
    },
    {
      "epoch": 0.7184100760611761,
      "grad_norm": 3.6809651851654053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11762318015098572,
      "logits/rejected": -0.21955199539661407,
      "logps/chosen": -489.3186340332031,
      "logps/rejected": -504.905029296875,
      "loss": 0.1569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.014133349061012268,
      "rewards/margins": 2.5467476844787598,
      "rewards/rejected": -2.560880661010742,
      "step": 5490
    },
    {
      "epoch": 0.7185409339985278,
      "grad_norm": 8.341329574584961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26410794258117676,
      "logits/rejected": -0.06719285249710083,
      "logps/chosen": -588.9337158203125,
      "logps/rejected": -594.6448974609375,
      "loss": 0.5321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20942270755767822,
      "rewards/margins": 1.298359751701355,
      "rewards/rejected": -1.5077823400497437,
      "step": 5491
    },
    {
      "epoch": 0.7186717919358796,
      "grad_norm": 8.011913299560547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06274820864200592,
      "logits/rejected": -0.027330350130796432,
      "logps/chosen": -366.1658020019531,
      "logps/rejected": -397.4050598144531,
      "loss": 0.7066,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2522510290145874,
      "rewards/margins": 0.621150016784668,
      "rewards/rejected": -1.873401165008545,
      "step": 5492
    },
    {
      "epoch": 0.7188026498732314,
      "grad_norm": 6.083229064941406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4930252730846405,
      "logits/rejected": 0.32475966215133667,
      "logps/chosen": -702.104736328125,
      "logps/rejected": -539.6131591796875,
      "loss": 0.3328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5259418487548828,
      "rewards/margins": 1.951733946800232,
      "rewards/rejected": -1.4257922172546387,
      "step": 5493
    },
    {
      "epoch": 0.7189335078105832,
      "grad_norm": 7.148937702178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2780497074127197,
      "logits/rejected": -0.24837768077850342,
      "logps/chosen": -575.4058837890625,
      "logps/rejected": -428.7301330566406,
      "loss": 0.4389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0765236616134644,
      "rewards/margins": 0.7029151916503906,
      "rewards/rejected": -1.7794387340545654,
      "step": 5494
    },
    {
      "epoch": 0.7190643657479349,
      "grad_norm": 13.201696395874023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12470810860395432,
      "logits/rejected": 0.03668990358710289,
      "logps/chosen": -506.41357421875,
      "logps/rejected": -400.591552734375,
      "loss": 0.7546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9849386811256409,
      "rewards/margins": 0.2320711314678192,
      "rewards/rejected": -1.2170097827911377,
      "step": 5495
    },
    {
      "epoch": 0.7191952236852867,
      "grad_norm": 10.940451622009277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23350372910499573,
      "logits/rejected": -0.10497492551803589,
      "logps/chosen": -477.8191223144531,
      "logps/rejected": -437.33599853515625,
      "loss": 0.488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.97383052110672,
      "rewards/margins": 0.7421612739562988,
      "rewards/rejected": -1.7159918546676636,
      "step": 5496
    },
    {
      "epoch": 0.7193260816226384,
      "grad_norm": 7.266420841217041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01577281951904297,
      "logits/rejected": 0.03297600522637367,
      "logps/chosen": -453.5193176269531,
      "logps/rejected": -488.8173828125,
      "loss": 0.6616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5141012072563171,
      "rewards/margins": 0.24780979752540588,
      "rewards/rejected": -0.7619110345840454,
      "step": 5497
    },
    {
      "epoch": 0.7194569395599902,
      "grad_norm": 5.63301944732666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08726522326469421,
      "logits/rejected": -0.14125338196754456,
      "logps/chosen": -475.7368469238281,
      "logps/rejected": -400.0734558105469,
      "loss": 0.2965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.01743392087519169,
      "rewards/margins": 1.335386872291565,
      "rewards/rejected": -1.3179528713226318,
      "step": 5498
    },
    {
      "epoch": 0.7195877974973419,
      "grad_norm": 3.961725950241089,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09598147869110107,
      "logits/rejected": -0.0743747353553772,
      "logps/chosen": -415.0611572265625,
      "logps/rejected": -352.3009948730469,
      "loss": 0.3386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.230396568775177,
      "rewards/margins": 1.4190754890441895,
      "rewards/rejected": -1.1886789798736572,
      "step": 5499
    },
    {
      "epoch": 0.7197186554346937,
      "grad_norm": 4.820684909820557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.052242461591959,
      "logits/rejected": -0.10268515348434448,
      "logps/chosen": -591.5419921875,
      "logps/rejected": -458.3149719238281,
      "loss": 0.259,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3574642539024353,
      "rewards/margins": 1.823216199874878,
      "rewards/rejected": -1.4657518863677979,
      "step": 5500
    },
    {
      "epoch": 0.7198495133720455,
      "grad_norm": 5.454073429107666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09758773446083069,
      "logits/rejected": 0.11497699469327927,
      "logps/chosen": -331.2243347167969,
      "logps/rejected": -431.31781005859375,
      "loss": 0.3844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03544674813747406,
      "rewards/margins": 1.5655338764190674,
      "rewards/rejected": -1.5300873517990112,
      "step": 5501
    },
    {
      "epoch": 0.7199803713093972,
      "grad_norm": 13.041352272033691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2689601480960846,
      "logits/rejected": -0.18035244941711426,
      "logps/chosen": -396.0110168457031,
      "logps/rejected": -495.26678466796875,
      "loss": 1.1051,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5091056823730469,
      "rewards/margins": -0.4871070981025696,
      "rewards/rejected": -0.021998584270477295,
      "step": 5502
    },
    {
      "epoch": 0.720111229246749,
      "grad_norm": 3.511431932449341,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07119732350111008,
      "logits/rejected": -0.03071482852101326,
      "logps/chosen": -568.96533203125,
      "logps/rejected": -379.8075256347656,
      "loss": 0.3326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4190104007720947,
      "rewards/margins": 1.5767061710357666,
      "rewards/rejected": -1.1576958894729614,
      "step": 5503
    },
    {
      "epoch": 0.7202420871841008,
      "grad_norm": 7.337076663970947,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20708419382572174,
      "logits/rejected": -0.10463254153728485,
      "logps/chosen": -316.3337707519531,
      "logps/rejected": -355.9027404785156,
      "loss": 0.4946,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07698040455579758,
      "rewards/margins": 1.287200689315796,
      "rewards/rejected": -1.2102203369140625,
      "step": 5504
    },
    {
      "epoch": 0.7203729451214526,
      "grad_norm": 5.991674423217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2568766176700592,
      "logits/rejected": -0.2633536159992218,
      "logps/chosen": -479.85491943359375,
      "logps/rejected": -372.833984375,
      "loss": 0.4629,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3840879201889038,
      "rewards/margins": 0.9385564923286438,
      "rewards/rejected": -0.5544685125350952,
      "step": 5505
    },
    {
      "epoch": 0.7205038030588042,
      "grad_norm": 3.913144588470459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22384141385555267,
      "logits/rejected": -0.15381351113319397,
      "logps/chosen": -466.5654602050781,
      "logps/rejected": -358.9624328613281,
      "loss": 0.3669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48268720507621765,
      "rewards/margins": 1.0718669891357422,
      "rewards/rejected": -1.5545539855957031,
      "step": 5506
    },
    {
      "epoch": 0.720634660996156,
      "grad_norm": 5.042278289794922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12409894168376923,
      "logits/rejected": -0.17993983626365662,
      "logps/chosen": -503.61773681640625,
      "logps/rejected": -545.1611328125,
      "loss": 0.3119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.730738639831543,
      "rewards/margins": 1.2526893615722656,
      "rewards/rejected": -1.9834280014038086,
      "step": 5507
    },
    {
      "epoch": 0.7207655189335078,
      "grad_norm": 7.179806709289551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02429795265197754,
      "logits/rejected": 0.007205553352832794,
      "logps/chosen": -474.21405029296875,
      "logps/rejected": -497.77862548828125,
      "loss": 0.4886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017130862921476364,
      "rewards/margins": 1.0119751691818237,
      "rewards/rejected": -0.9948441982269287,
      "step": 5508
    },
    {
      "epoch": 0.7208963768708596,
      "grad_norm": 5.161820411682129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29800158739089966,
      "logits/rejected": 0.2799288332462311,
      "logps/chosen": -512.04150390625,
      "logps/rejected": -274.1846008300781,
      "loss": 0.4493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04534241557121277,
      "rewards/margins": 1.4838223457336426,
      "rewards/rejected": -1.5291647911071777,
      "step": 5509
    },
    {
      "epoch": 0.7210272348082113,
      "grad_norm": 10.153411865234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15122395753860474,
      "logits/rejected": 0.05432366579771042,
      "logps/chosen": -516.8904418945312,
      "logps/rejected": -647.236083984375,
      "loss": 0.8412,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.36316800117492676,
      "rewards/margins": 0.12842822074890137,
      "rewards/rejected": -0.4915962517261505,
      "step": 5510
    },
    {
      "epoch": 0.7211580927455631,
      "grad_norm": 8.975367546081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08967337012290955,
      "logits/rejected": 0.21665604412555695,
      "logps/chosen": -482.7938232421875,
      "logps/rejected": -432.05694580078125,
      "loss": 0.605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22236326336860657,
      "rewards/margins": 1.1957885026931763,
      "rewards/rejected": -0.9734251499176025,
      "step": 5511
    },
    {
      "epoch": 0.7212889506829149,
      "grad_norm": 7.64005708694458,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04894017428159714,
      "logits/rejected": 0.1766144335269928,
      "logps/chosen": -420.78778076171875,
      "logps/rejected": -453.3709411621094,
      "loss": 0.6643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006110004149377346,
      "rewards/margins": 0.3172492980957031,
      "rewards/rejected": -0.32335931062698364,
      "step": 5512
    },
    {
      "epoch": 0.7214198086202667,
      "grad_norm": 6.383911609649658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2817639112472534,
      "logits/rejected": -0.13052016496658325,
      "logps/chosen": -637.2109375,
      "logps/rejected": -427.33587646484375,
      "loss": 0.4507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1300378441810608,
      "rewards/margins": 1.4356470108032227,
      "rewards/rejected": -1.3056092262268066,
      "step": 5513
    },
    {
      "epoch": 0.7215506665576183,
      "grad_norm": 5.851563930511475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012078158557415009,
      "logits/rejected": 0.03563690185546875,
      "logps/chosen": -438.7109375,
      "logps/rejected": -453.73046875,
      "loss": 0.3499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5801721811294556,
      "rewards/margins": 1.3957959413528442,
      "rewards/rejected": -1.9759681224822998,
      "step": 5514
    },
    {
      "epoch": 0.7216815244949701,
      "grad_norm": 8.629935264587402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028914812952280045,
      "logits/rejected": -0.16040028631687164,
      "logps/chosen": -539.7793579101562,
      "logps/rejected": -586.2647094726562,
      "loss": 0.6797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03328229859471321,
      "rewards/margins": 0.9437814950942993,
      "rewards/rejected": -0.977063775062561,
      "step": 5515
    },
    {
      "epoch": 0.7218123824323219,
      "grad_norm": 8.406011581420898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09867441654205322,
      "logits/rejected": 0.11786006391048431,
      "logps/chosen": -560.4102783203125,
      "logps/rejected": -548.44384765625,
      "loss": 0.4718,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08983193337917328,
      "rewards/margins": 1.1726762056350708,
      "rewards/rejected": -1.0828444957733154,
      "step": 5516
    },
    {
      "epoch": 0.7219432403696737,
      "grad_norm": 7.626509666442871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046654779464006424,
      "logits/rejected": 0.09392823278903961,
      "logps/chosen": -408.1594543457031,
      "logps/rejected": -409.532958984375,
      "loss": 0.578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.020212776958942413,
      "rewards/margins": 0.5685970187187195,
      "rewards/rejected": -0.5483843088150024,
      "step": 5517
    },
    {
      "epoch": 0.7220740983070254,
      "grad_norm": 7.237539291381836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0421040914952755,
      "logits/rejected": -0.24163956940174103,
      "logps/chosen": -591.0178833007812,
      "logps/rejected": -577.3928833007812,
      "loss": 0.3902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22445526719093323,
      "rewards/margins": 0.9453802108764648,
      "rewards/rejected": -1.1698354482650757,
      "step": 5518
    },
    {
      "epoch": 0.7222049562443772,
      "grad_norm": 8.246326446533203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16126687824726105,
      "logits/rejected": -0.26161837577819824,
      "logps/chosen": -434.2289733886719,
      "logps/rejected": -389.1749572753906,
      "loss": 0.5878,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15891437232494354,
      "rewards/margins": 0.9328556060791016,
      "rewards/rejected": -1.0917699337005615,
      "step": 5519
    },
    {
      "epoch": 0.722335814181729,
      "grad_norm": 6.796332836151123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1702929139137268,
      "logits/rejected": -0.1366620808839798,
      "logps/chosen": -490.8085632324219,
      "logps/rejected": -455.9367980957031,
      "loss": 0.4135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5037622451782227,
      "rewards/margins": 1.4036028385162354,
      "rewards/rejected": -1.9073649644851685,
      "step": 5520
    },
    {
      "epoch": 0.7224666721190808,
      "grad_norm": 6.690834999084473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27790993452072144,
      "logits/rejected": 0.20467710494995117,
      "logps/chosen": -367.6100769042969,
      "logps/rejected": -397.8395690917969,
      "loss": 0.5703,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5413166284561157,
      "rewards/margins": 0.5023730993270874,
      "rewards/rejected": -1.0436897277832031,
      "step": 5521
    },
    {
      "epoch": 0.7225975300564325,
      "grad_norm": 5.914132595062256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1941043883562088,
      "logits/rejected": 0.06053200364112854,
      "logps/chosen": -488.39453125,
      "logps/rejected": -613.271484375,
      "loss": 0.3125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8768028616905212,
      "rewards/margins": 1.3661270141601562,
      "rewards/rejected": -0.4893242120742798,
      "step": 5522
    },
    {
      "epoch": 0.7227283879937842,
      "grad_norm": 4.191865921020508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.061837337911129,
      "logits/rejected": -0.24824225902557373,
      "logps/chosen": -425.2308654785156,
      "logps/rejected": -467.229736328125,
      "loss": 0.2165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5098770260810852,
      "rewards/margins": 1.9205420017242432,
      "rewards/rejected": -2.4304189682006836,
      "step": 5523
    },
    {
      "epoch": 0.722859245931136,
      "grad_norm": 7.165166854858398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.080083467066288,
      "logits/rejected": -0.03922351449728012,
      "logps/chosen": -637.36376953125,
      "logps/rejected": -495.78070068359375,
      "loss": 0.4977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4604254364967346,
      "rewards/margins": 0.8210481405258179,
      "rewards/rejected": -0.3606227934360504,
      "step": 5524
    },
    {
      "epoch": 0.7229901038684877,
      "grad_norm": 5.561097621917725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1115160882472992,
      "logits/rejected": -0.07127648591995239,
      "logps/chosen": -376.7503967285156,
      "logps/rejected": -440.25311279296875,
      "loss": 0.3873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07686404138803482,
      "rewards/margins": 1.1414458751678467,
      "rewards/rejected": -1.0645817518234253,
      "step": 5525
    },
    {
      "epoch": 0.7231209618058395,
      "grad_norm": 9.544022560119629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23084726929664612,
      "logits/rejected": -0.16983911395072937,
      "logps/chosen": -344.2131042480469,
      "logps/rejected": -393.52392578125,
      "loss": 0.5039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27992022037506104,
      "rewards/margins": 0.9217052459716797,
      "rewards/rejected": -1.2016255855560303,
      "step": 5526
    },
    {
      "epoch": 0.7232518197431913,
      "grad_norm": 7.156835079193115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16792690753936768,
      "logits/rejected": 0.13290753960609436,
      "logps/chosen": -455.46771240234375,
      "logps/rejected": -462.15057373046875,
      "loss": 0.4388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11657334864139557,
      "rewards/margins": 0.7851024866104126,
      "rewards/rejected": -0.9016758799552917,
      "step": 5527
    },
    {
      "epoch": 0.7233826776805431,
      "grad_norm": 6.597811222076416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21023109555244446,
      "logits/rejected": -0.00357886403799057,
      "logps/chosen": -543.8817138671875,
      "logps/rejected": -515.7647705078125,
      "loss": 0.3688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.17685067653656,
      "rewards/margins": 1.1994073390960693,
      "rewards/rejected": -2.376258134841919,
      "step": 5528
    },
    {
      "epoch": 0.7235135356178948,
      "grad_norm": 6.814995765686035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2282121479511261,
      "logits/rejected": 0.3951035737991333,
      "logps/chosen": -392.8581237792969,
      "logps/rejected": -434.7904052734375,
      "loss": 0.6399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33362123370170593,
      "rewards/margins": 0.5774823427200317,
      "rewards/rejected": -0.9111034870147705,
      "step": 5529
    },
    {
      "epoch": 0.7236443935552466,
      "grad_norm": 7.384458541870117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2756315767765045,
      "logits/rejected": -0.10416314005851746,
      "logps/chosen": -377.0824279785156,
      "logps/rejected": -268.99163818359375,
      "loss": 0.7534,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1504429578781128,
      "rewards/margins": 0.05236624926328659,
      "rewards/rejected": -1.20280921459198,
      "step": 5530
    },
    {
      "epoch": 0.7237752514925984,
      "grad_norm": 5.667468070983887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0532495453953743,
      "logits/rejected": 0.10815458744764328,
      "logps/chosen": -340.7552795410156,
      "logps/rejected": -430.638671875,
      "loss": 0.4435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46717509627342224,
      "rewards/margins": 0.8960129022598267,
      "rewards/rejected": -1.3631879091262817,
      "step": 5531
    },
    {
      "epoch": 0.7239061094299502,
      "grad_norm": 8.85827922821045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1176023930311203,
      "logits/rejected": -0.055243074893951416,
      "logps/chosen": -200.92184448242188,
      "logps/rejected": -329.77288818359375,
      "loss": 0.6266,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1878950148820877,
      "rewards/margins": 0.8731177449226379,
      "rewards/rejected": -1.0610127449035645,
      "step": 5532
    },
    {
      "epoch": 0.7240369673673018,
      "grad_norm": 7.154928207397461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18886876106262207,
      "logits/rejected": 0.12481188029050827,
      "logps/chosen": -386.1484375,
      "logps/rejected": -407.7109375,
      "loss": 0.5189,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.788434624671936,
      "rewards/margins": 0.8352420926094055,
      "rewards/rejected": -1.6236767768859863,
      "step": 5533
    },
    {
      "epoch": 0.7241678253046536,
      "grad_norm": 5.155317783355713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22196592390537262,
      "logits/rejected": -0.20166541635990143,
      "logps/chosen": -313.5342102050781,
      "logps/rejected": -280.423583984375,
      "loss": 0.3699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2701162099838257,
      "rewards/margins": 1.2155814170837402,
      "rewards/rejected": -1.485697627067566,
      "step": 5534
    },
    {
      "epoch": 0.7242986832420054,
      "grad_norm": 5.765632152557373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21602952480316162,
      "logits/rejected": 0.2856285572052002,
      "logps/chosen": -666.3465576171875,
      "logps/rejected": -551.4573364257812,
      "loss": 0.335,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5918669104576111,
      "rewards/margins": 1.9952248334884644,
      "rewards/rejected": -1.4033582210540771,
      "step": 5535
    },
    {
      "epoch": 0.7244295411793572,
      "grad_norm": 7.337310314178467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06464080512523651,
      "logits/rejected": 0.05062326416373253,
      "logps/chosen": -402.51885986328125,
      "logps/rejected": -371.57196044921875,
      "loss": 0.5951,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0166934728622437,
      "rewards/margins": 0.5825809240341187,
      "rewards/rejected": -1.5992745161056519,
      "step": 5536
    },
    {
      "epoch": 0.7245603991167089,
      "grad_norm": 7.836188793182373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.050906792283058167,
      "logits/rejected": -0.10327917337417603,
      "logps/chosen": -413.9915771484375,
      "logps/rejected": -369.0659484863281,
      "loss": 0.5737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6695413589477539,
      "rewards/margins": 0.6225897073745728,
      "rewards/rejected": -1.292130947113037,
      "step": 5537
    },
    {
      "epoch": 0.7246912570540607,
      "grad_norm": 5.954854965209961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1766248345375061,
      "logits/rejected": 0.34698301553726196,
      "logps/chosen": -406.5910949707031,
      "logps/rejected": -513.1931762695312,
      "loss": 0.3808,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16882438957691193,
      "rewards/margins": 1.0644466876983643,
      "rewards/rejected": -1.2332711219787598,
      "step": 5538
    },
    {
      "epoch": 0.7248221149914125,
      "grad_norm": 6.133982181549072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25637558102607727,
      "logits/rejected": -0.22819902002811432,
      "logps/chosen": -468.0482177734375,
      "logps/rejected": -448.239013671875,
      "loss": 0.3589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5479099154472351,
      "rewards/margins": 1.3575419187545776,
      "rewards/rejected": -0.8096319437026978,
      "step": 5539
    },
    {
      "epoch": 0.7249529729287643,
      "grad_norm": 8.225783348083496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060507554560899734,
      "logits/rejected": -0.03365221619606018,
      "logps/chosen": -375.47528076171875,
      "logps/rejected": -436.2300109863281,
      "loss": 0.7461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.36576420068740845,
      "rewards/margins": 0.0855613648891449,
      "rewards/rejected": -0.4513256251811981,
      "step": 5540
    },
    {
      "epoch": 0.7250838308661159,
      "grad_norm": 8.394696235656738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13956180214881897,
      "logits/rejected": -0.13706570863723755,
      "logps/chosen": -359.84344482421875,
      "logps/rejected": -436.50140380859375,
      "loss": 0.5023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7166626453399658,
      "rewards/margins": 0.9085292816162109,
      "rewards/rejected": -1.6251918077468872,
      "step": 5541
    },
    {
      "epoch": 0.7252146888034677,
      "grad_norm": 7.456146240234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13624078035354614,
      "logits/rejected": -0.1552712619304657,
      "logps/chosen": -510.34869384765625,
      "logps/rejected": -462.151123046875,
      "loss": 0.4622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23957481980323792,
      "rewards/margins": 1.0365650653839111,
      "rewards/rejected": -1.2761398553848267,
      "step": 5542
    },
    {
      "epoch": 0.7253455467408195,
      "grad_norm": 11.854698181152344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14688174426555634,
      "logits/rejected": -0.04593411087989807,
      "logps/chosen": -422.3421325683594,
      "logps/rejected": -485.43798828125,
      "loss": 0.8459,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9550796747207642,
      "rewards/margins": -0.054626256227493286,
      "rewards/rejected": -0.9004534482955933,
      "step": 5543
    },
    {
      "epoch": 0.7254764046781712,
      "grad_norm": 8.173789024353027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01717979833483696,
      "logits/rejected": -0.08859814703464508,
      "logps/chosen": -632.8009033203125,
      "logps/rejected": -625.2982177734375,
      "loss": 0.5563,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29117512702941895,
      "rewards/margins": 1.0047669410705566,
      "rewards/rejected": -1.295941948890686,
      "step": 5544
    },
    {
      "epoch": 0.725607262615523,
      "grad_norm": 6.431354999542236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03915371745824814,
      "logits/rejected": -0.060588493943214417,
      "logps/chosen": -433.69024658203125,
      "logps/rejected": -390.79010009765625,
      "loss": 0.6246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35616323351860046,
      "rewards/margins": 0.5927059054374695,
      "rewards/rejected": -0.9488691091537476,
      "step": 5545
    },
    {
      "epoch": 0.7257381205528748,
      "grad_norm": 7.923173904418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2919282913208008,
      "logits/rejected": 0.2794700860977173,
      "logps/chosen": -462.6376037597656,
      "logps/rejected": -394.3906555175781,
      "loss": 0.4244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5815321207046509,
      "rewards/margins": 0.91555255651474,
      "rewards/rejected": -0.3340204358100891,
      "step": 5546
    },
    {
      "epoch": 0.7258689784902266,
      "grad_norm": 7.556193828582764,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023019753396511078,
      "logits/rejected": -0.11895984411239624,
      "logps/chosen": -343.5499572753906,
      "logps/rejected": -504.3725280761719,
      "loss": 0.4902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04603138566017151,
      "rewards/margins": 1.3942902088165283,
      "rewards/rejected": -1.4403215646743774,
      "step": 5547
    },
    {
      "epoch": 0.7259998364275783,
      "grad_norm": 7.57975435256958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.053295135498046875,
      "logits/rejected": 0.0796346366405487,
      "logps/chosen": -481.3021240234375,
      "logps/rejected": -476.71337890625,
      "loss": 0.462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.476115345954895,
      "rewards/margins": 0.729317307472229,
      "rewards/rejected": -1.205432653427124,
      "step": 5548
    },
    {
      "epoch": 0.72613069436493,
      "grad_norm": 8.213522911071777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03948497027158737,
      "logits/rejected": -0.11197128891944885,
      "logps/chosen": -536.8814697265625,
      "logps/rejected": -566.6268310546875,
      "loss": 0.6062,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8256326913833618,
      "rewards/margins": 0.3810476064682007,
      "rewards/rejected": -1.206680417060852,
      "step": 5549
    },
    {
      "epoch": 0.7262615523022818,
      "grad_norm": 4.032438278198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41101235151290894,
      "logits/rejected": -0.3973349928855896,
      "logps/chosen": -438.8923645019531,
      "logps/rejected": -391.9601745605469,
      "loss": 0.2477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07412568479776382,
      "rewards/margins": 1.856071949005127,
      "rewards/rejected": -1.7819463014602661,
      "step": 5550
    },
    {
      "epoch": 0.7263924102396336,
      "grad_norm": 4.305718898773193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09222058951854706,
      "logits/rejected": -0.00528290867805481,
      "logps/chosen": -439.49169921875,
      "logps/rejected": -286.43756103515625,
      "loss": 0.322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4231763482093811,
      "rewards/margins": 1.617088794708252,
      "rewards/rejected": -2.0402653217315674,
      "step": 5551
    },
    {
      "epoch": 0.7265232681769853,
      "grad_norm": 8.945070266723633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12118402868509293,
      "logits/rejected": 0.023697558790445328,
      "logps/chosen": -401.7942199707031,
      "logps/rejected": -483.02740478515625,
      "loss": 0.5255,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.345414638519287,
      "rewards/margins": 1.2692172527313232,
      "rewards/rejected": -2.6146321296691895,
      "step": 5552
    },
    {
      "epoch": 0.7266541261143371,
      "grad_norm": 6.15989875793457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.308087557554245,
      "logits/rejected": 0.2765005826950073,
      "logps/chosen": -269.6277770996094,
      "logps/rejected": -301.1766052246094,
      "loss": 0.4124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2595653533935547,
      "rewards/margins": 0.888454794883728,
      "rewards/rejected": -1.1480202674865723,
      "step": 5553
    },
    {
      "epoch": 0.7267849840516889,
      "grad_norm": 6.871318817138672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1964418888092041,
      "logits/rejected": 0.2533723711967468,
      "logps/chosen": -425.8941345214844,
      "logps/rejected": -410.797607421875,
      "loss": 0.3015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7140934467315674,
      "rewards/margins": 1.414409875869751,
      "rewards/rejected": -2.1285033226013184,
      "step": 5554
    },
    {
      "epoch": 0.7269158419890407,
      "grad_norm": 6.180948257446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10517893731594086,
      "logits/rejected": -0.14502808451652527,
      "logps/chosen": -407.17303466796875,
      "logps/rejected": -399.51251220703125,
      "loss": 0.4761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35444721579551697,
      "rewards/margins": 0.7731277942657471,
      "rewards/rejected": -1.1275749206542969,
      "step": 5555
    },
    {
      "epoch": 0.7270466999263924,
      "grad_norm": 5.662472724914551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10954190790653229,
      "logits/rejected": -0.07527802884578705,
      "logps/chosen": -432.17669677734375,
      "logps/rejected": -490.3731994628906,
      "loss": 0.3139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.007344435900449753,
      "rewards/margins": 1.3447190523147583,
      "rewards/rejected": -1.352063536643982,
      "step": 5556
    },
    {
      "epoch": 0.7271775578637442,
      "grad_norm": 5.160375595092773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02217479608952999,
      "logits/rejected": 0.10269629955291748,
      "logps/chosen": -404.1995849609375,
      "logps/rejected": -455.9080810546875,
      "loss": 0.4849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008763894438743591,
      "rewards/margins": 0.7762451171875,
      "rewards/rejected": -0.7674812078475952,
      "step": 5557
    },
    {
      "epoch": 0.727308415801096,
      "grad_norm": 5.305185794830322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03398660942912102,
      "logits/rejected": 0.0033515840768814087,
      "logps/chosen": -335.11920166015625,
      "logps/rejected": -309.28765869140625,
      "loss": 0.4766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11089516431093216,
      "rewards/margins": 0.9637378454208374,
      "rewards/rejected": -0.8528427481651306,
      "step": 5558
    },
    {
      "epoch": 0.7274392737384477,
      "grad_norm": 10.191018104553223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06791778653860092,
      "logits/rejected": -0.1614009290933609,
      "logps/chosen": -576.2963256835938,
      "logps/rejected": -424.4110412597656,
      "loss": 0.9878,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2355581521987915,
      "rewards/margins": 0.7865631580352783,
      "rewards/rejected": -1.0221213102340698,
      "step": 5559
    },
    {
      "epoch": 0.7275701316757994,
      "grad_norm": 11.456401824951172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1926172524690628,
      "logits/rejected": 0.04524474963545799,
      "logps/chosen": -316.0159606933594,
      "logps/rejected": -319.3062744140625,
      "loss": 1.1173,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.048338532447815,
      "rewards/margins": -0.1608532965183258,
      "rewards/rejected": -0.8874852657318115,
      "step": 5560
    },
    {
      "epoch": 0.7277009896131512,
      "grad_norm": 5.800259113311768,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33539608120918274,
      "logits/rejected": 0.26233091950416565,
      "logps/chosen": -324.9656066894531,
      "logps/rejected": -402.9197082519531,
      "loss": 0.4651,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9111400246620178,
      "rewards/margins": 0.9269498586654663,
      "rewards/rejected": -1.838089942932129,
      "step": 5561
    },
    {
      "epoch": 0.727831847550503,
      "grad_norm": 4.153323650360107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1472722738981247,
      "logits/rejected": 0.12345095723867416,
      "logps/chosen": -559.0309448242188,
      "logps/rejected": -500.25421142578125,
      "loss": 0.2715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0322570726275444,
      "rewards/margins": 1.2891440391540527,
      "rewards/rejected": -1.2568868398666382,
      "step": 5562
    },
    {
      "epoch": 0.7279627054878548,
      "grad_norm": 6.2235918045043945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06480705738067627,
      "logits/rejected": -0.16215744614601135,
      "logps/chosen": -585.1642456054688,
      "logps/rejected": -563.89013671875,
      "loss": 0.3548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5771160125732422,
      "rewards/margins": 2.0286190509796143,
      "rewards/rejected": -2.6057353019714355,
      "step": 5563
    },
    {
      "epoch": 0.7280935634252065,
      "grad_norm": 4.699299335479736,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14434771239757538,
      "logits/rejected": -0.18216931819915771,
      "logps/chosen": -489.4161376953125,
      "logps/rejected": -386.3921813964844,
      "loss": 0.3164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11969110369682312,
      "rewards/margins": 1.390838623046875,
      "rewards/rejected": -1.2711474895477295,
      "step": 5564
    },
    {
      "epoch": 0.7282244213625583,
      "grad_norm": 7.557932376861572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03222755342721939,
      "logits/rejected": 0.07244186103343964,
      "logps/chosen": -466.713623046875,
      "logps/rejected": -414.9378967285156,
      "loss": 0.4915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5338628888130188,
      "rewards/margins": 0.5856742858886719,
      "rewards/rejected": -1.1195372343063354,
      "step": 5565
    },
    {
      "epoch": 0.7283552792999101,
      "grad_norm": 7.97011661529541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2601211369037628,
      "logits/rejected": -0.2598590552806854,
      "logps/chosen": -379.228271484375,
      "logps/rejected": -476.63580322265625,
      "loss": 0.5703,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2606226205825806,
      "rewards/margins": 0.8540340662002563,
      "rewards/rejected": -2.114656686782837,
      "step": 5566
    },
    {
      "epoch": 0.7284861372372617,
      "grad_norm": 7.703221797943115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16729912161827087,
      "logits/rejected": -0.020441390573978424,
      "logps/chosen": -579.60693359375,
      "logps/rejected": -446.4519348144531,
      "loss": 0.4,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.518650233745575,
      "rewards/margins": 1.6986404657363892,
      "rewards/rejected": -1.1799901723861694,
      "step": 5567
    },
    {
      "epoch": 0.7286169951746135,
      "grad_norm": 6.807309627532959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16895420849323273,
      "logits/rejected": -0.18311330676078796,
      "logps/chosen": -415.12164306640625,
      "logps/rejected": -444.6636962890625,
      "loss": 0.3092,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5966907739639282,
      "rewards/margins": 1.3289730548858643,
      "rewards/rejected": -2.925663948059082,
      "step": 5568
    },
    {
      "epoch": 0.7287478531119653,
      "grad_norm": 7.748900890350342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2291828989982605,
      "logits/rejected": 0.39648354053497314,
      "logps/chosen": -270.4715576171875,
      "logps/rejected": -303.3717041015625,
      "loss": 0.6047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5819042921066284,
      "rewards/margins": 0.6569720506668091,
      "rewards/rejected": -1.238876223564148,
      "step": 5569
    },
    {
      "epoch": 0.7288787110493171,
      "grad_norm": 7.385788917541504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0450759157538414,
      "logits/rejected": 0.16140803694725037,
      "logps/chosen": -392.5074157714844,
      "logps/rejected": -399.7292785644531,
      "loss": 0.5556,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0824581384658813,
      "rewards/margins": 0.5107414722442627,
      "rewards/rejected": -1.593199610710144,
      "step": 5570
    },
    {
      "epoch": 0.7290095689866688,
      "grad_norm": 10.509237289428711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13206374645233154,
      "logits/rejected": -0.11560166627168655,
      "logps/chosen": -393.5555725097656,
      "logps/rejected": -300.1257019042969,
      "loss": 0.6064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4324010908603668,
      "rewards/margins": 0.7391599416732788,
      "rewards/rejected": -1.1715610027313232,
      "step": 5571
    },
    {
      "epoch": 0.7291404269240206,
      "grad_norm": 20.33873748779297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1637265980243683,
      "logits/rejected": -0.16933362185955048,
      "logps/chosen": -447.3669128417969,
      "logps/rejected": -480.6816101074219,
      "loss": 0.5928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3091191053390503,
      "rewards/margins": 1.3111865520477295,
      "rewards/rejected": -2.6203055381774902,
      "step": 5572
    },
    {
      "epoch": 0.7292712848613724,
      "grad_norm": 8.629154205322266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17902684211730957,
      "logits/rejected": -0.2719164788722992,
      "logps/chosen": -392.59503173828125,
      "logps/rejected": -368.78741455078125,
      "loss": 0.9167,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8094912767410278,
      "rewards/margins": -0.18928438425064087,
      "rewards/rejected": -0.6202067732810974,
      "step": 5573
    },
    {
      "epoch": 0.7294021427987242,
      "grad_norm": 4.417634963989258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013466492295265198,
      "logits/rejected": -0.17308415472507477,
      "logps/chosen": -383.76409912109375,
      "logps/rejected": -474.7236328125,
      "loss": 0.2357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22374586760997772,
      "rewards/margins": 1.6936132907867432,
      "rewards/rejected": -1.9173592329025269,
      "step": 5574
    },
    {
      "epoch": 0.7295330007360759,
      "grad_norm": 8.365234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09489904344081879,
      "logits/rejected": 0.05804409831762314,
      "logps/chosen": -563.5083618164062,
      "logps/rejected": -533.6477661132812,
      "loss": 0.3596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18384386599063873,
      "rewards/margins": 1.3527761697769165,
      "rewards/rejected": -1.5366199016571045,
      "step": 5575
    },
    {
      "epoch": 0.7296638586734276,
      "grad_norm": 4.214722156524658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26830339431762695,
      "logits/rejected": -0.2114570140838623,
      "logps/chosen": -346.4187316894531,
      "logps/rejected": -398.5469055175781,
      "loss": 0.4143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8198042511940002,
      "rewards/margins": 1.297731876373291,
      "rewards/rejected": -2.1175360679626465,
      "step": 5576
    },
    {
      "epoch": 0.7297947166107794,
      "grad_norm": 11.123004913330078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018811509013175964,
      "logits/rejected": 0.04912635311484337,
      "logps/chosen": -540.994140625,
      "logps/rejected": -501.0311279296875,
      "loss": 0.7189,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.862973153591156,
      "rewards/margins": 0.255313515663147,
      "rewards/rejected": -1.1182867288589478,
      "step": 5577
    },
    {
      "epoch": 0.7299255745481312,
      "grad_norm": 6.8647003173828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026555778458714485,
      "logits/rejected": 0.002688322216272354,
      "logps/chosen": -416.8795471191406,
      "logps/rejected": -362.04791259765625,
      "loss": 0.5923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009380554780364037,
      "rewards/margins": 1.210340142250061,
      "rewards/rejected": -1.2009596824645996,
      "step": 5578
    },
    {
      "epoch": 0.7300564324854829,
      "grad_norm": 10.751498222351074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21389837563037872,
      "logits/rejected": 0.03497283533215523,
      "logps/chosen": -669.4512329101562,
      "logps/rejected": -493.8847961425781,
      "loss": 0.7685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20960235595703125,
      "rewards/margins": 0.5431572198867798,
      "rewards/rejected": -0.752759575843811,
      "step": 5579
    },
    {
      "epoch": 0.7301872904228347,
      "grad_norm": 5.963327407836914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08363964408636093,
      "logits/rejected": 0.19525766372680664,
      "logps/chosen": -388.6085205078125,
      "logps/rejected": -384.95989990234375,
      "loss": 0.3359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22945690155029297,
      "rewards/margins": 1.2389627695083618,
      "rewards/rejected": -1.0095058679580688,
      "step": 5580
    },
    {
      "epoch": 0.7303181483601865,
      "grad_norm": 8.887582778930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034069232642650604,
      "logits/rejected": -0.19914260506629944,
      "logps/chosen": -600.989990234375,
      "logps/rejected": -492.4159851074219,
      "loss": 0.5615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5039435625076294,
      "rewards/margins": 1.0057138204574585,
      "rewards/rejected": -2.509657382965088,
      "step": 5581
    },
    {
      "epoch": 0.7304490062975383,
      "grad_norm": 7.8522515296936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10093969851732254,
      "logits/rejected": -0.07260488718748093,
      "logps/chosen": -375.4498596191406,
      "logps/rejected": -572.3993530273438,
      "loss": 0.4709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6745418310165405,
      "rewards/margins": 1.1486319303512573,
      "rewards/rejected": -1.8231737613677979,
      "step": 5582
    },
    {
      "epoch": 0.73057986423489,
      "grad_norm": 9.706011772155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22033505141735077,
      "logits/rejected": -0.15910595655441284,
      "logps/chosen": -372.5337219238281,
      "logps/rejected": -464.7235412597656,
      "loss": 0.808,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4262926578521729,
      "rewards/margins": 0.19286613166332245,
      "rewards/rejected": -1.6191587448120117,
      "step": 5583
    },
    {
      "epoch": 0.7307107221722418,
      "grad_norm": 7.464484691619873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0980546697974205,
      "logits/rejected": 0.03289409354329109,
      "logps/chosen": -553.0632934570312,
      "logps/rejected": -432.44866943359375,
      "loss": 0.4403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6261917352676392,
      "rewards/margins": 1.1093966960906982,
      "rewards/rejected": -1.7355884313583374,
      "step": 5584
    },
    {
      "epoch": 0.7308415801095935,
      "grad_norm": 5.853247165679932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1150301843881607,
      "logits/rejected": 0.009413152933120728,
      "logps/chosen": -506.8005065917969,
      "logps/rejected": -527.2220458984375,
      "loss": 0.3955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9416139125823975,
      "rewards/margins": 1.2018461227416992,
      "rewards/rejected": -2.1434600353240967,
      "step": 5585
    },
    {
      "epoch": 0.7309724380469453,
      "grad_norm": 3.9769299030303955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03485395014286041,
      "logits/rejected": 0.14544175565242767,
      "logps/chosen": -483.9958190917969,
      "logps/rejected": -481.0690002441406,
      "loss": 0.2381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46931251883506775,
      "rewards/margins": 1.6213505268096924,
      "rewards/rejected": -1.1520379781723022,
      "step": 5586
    },
    {
      "epoch": 0.731103295984297,
      "grad_norm": 6.01963996887207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15245968103408813,
      "logits/rejected": 0.09458588808774948,
      "logps/chosen": -640.8668212890625,
      "logps/rejected": -464.75006103515625,
      "loss": 0.3875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.060680389404296875,
      "rewards/margins": 1.482146978378296,
      "rewards/rejected": -1.4214664697647095,
      "step": 5587
    },
    {
      "epoch": 0.7312341539216488,
      "grad_norm": 11.187726020812988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16639959812164307,
      "logits/rejected": -0.3204547166824341,
      "logps/chosen": -534.3477783203125,
      "logps/rejected": -409.4682312011719,
      "loss": 0.7633,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42398014664649963,
      "rewards/margins": 0.5681988000869751,
      "rewards/rejected": -0.9921790361404419,
      "step": 5588
    },
    {
      "epoch": 0.7313650118590006,
      "grad_norm": 6.38781213760376,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5147034525871277,
      "logits/rejected": 0.2381650060415268,
      "logps/chosen": -407.10284423828125,
      "logps/rejected": -365.3355712890625,
      "loss": 0.5332,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10497874021530151,
      "rewards/margins": 1.2131259441375732,
      "rewards/rejected": -1.1081472635269165,
      "step": 5589
    },
    {
      "epoch": 0.7314958697963523,
      "grad_norm": 9.696632385253906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2811790704727173,
      "logits/rejected": 0.29344314336776733,
      "logps/chosen": -255.60458374023438,
      "logps/rejected": -313.8320617675781,
      "loss": 0.4554,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30362895131111145,
      "rewards/margins": 1.6704208850860596,
      "rewards/rejected": -1.9740498065948486,
      "step": 5590
    },
    {
      "epoch": 0.7316267277337041,
      "grad_norm": 8.816880226135254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01432863250374794,
      "logits/rejected": -0.20118092000484467,
      "logps/chosen": -461.8470764160156,
      "logps/rejected": -405.1841125488281,
      "loss": 0.5904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8921137452125549,
      "rewards/margins": 0.6293476819992065,
      "rewards/rejected": -1.5214614868164062,
      "step": 5591
    },
    {
      "epoch": 0.7317575856710559,
      "grad_norm": 7.445065498352051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17084285616874695,
      "logits/rejected": -0.1385364681482315,
      "logps/chosen": -394.89874267578125,
      "logps/rejected": -498.8062438964844,
      "loss": 0.5056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43960249423980713,
      "rewards/margins": 1.3729585409164429,
      "rewards/rejected": -1.81256103515625,
      "step": 5592
    },
    {
      "epoch": 0.7318884436084077,
      "grad_norm": 10.682307243347168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2332111895084381,
      "logits/rejected": 0.23247507214546204,
      "logps/chosen": -468.15087890625,
      "logps/rejected": -529.75732421875,
      "loss": 0.6411,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9002611637115479,
      "rewards/margins": 0.2046840637922287,
      "rewards/rejected": -1.104945182800293,
      "step": 5593
    },
    {
      "epoch": 0.7320193015457593,
      "grad_norm": 10.986233711242676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29790905117988586,
      "logits/rejected": -0.5544236898422241,
      "logps/chosen": -352.169677734375,
      "logps/rejected": -478.6522216796875,
      "loss": 0.919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8914846777915955,
      "rewards/margins": 0.11551384627819061,
      "rewards/rejected": -1.0069985389709473,
      "step": 5594
    },
    {
      "epoch": 0.7321501594831111,
      "grad_norm": 6.167646408081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16962501406669617,
      "logits/rejected": -0.2894262969493866,
      "logps/chosen": -413.8858642578125,
      "logps/rejected": -594.4630126953125,
      "loss": 0.2692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.32319316267967224,
      "rewards/margins": 2.5822253227233887,
      "rewards/rejected": -2.905418634414673,
      "step": 5595
    },
    {
      "epoch": 0.7322810174204629,
      "grad_norm": 9.476821899414062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11945796757936478,
      "logits/rejected": 0.12330025434494019,
      "logps/chosen": -571.5908203125,
      "logps/rejected": -538.452880859375,
      "loss": 0.6248,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1837425231933594,
      "rewards/margins": 0.5559852123260498,
      "rewards/rejected": -1.7397277355194092,
      "step": 5596
    },
    {
      "epoch": 0.7324118753578147,
      "grad_norm": 6.692840099334717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017873456701636314,
      "logits/rejected": -0.0010225623846054077,
      "logps/chosen": -511.56732177734375,
      "logps/rejected": -396.8828430175781,
      "loss": 0.5081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6258913278579712,
      "rewards/margins": 0.5104996562004089,
      "rewards/rejected": -1.1363911628723145,
      "step": 5597
    },
    {
      "epoch": 0.7325427332951664,
      "grad_norm": 9.496286392211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11002976447343826,
      "logits/rejected": 0.14525993168354034,
      "logps/chosen": -451.0660705566406,
      "logps/rejected": -363.0911865234375,
      "loss": 0.5948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0147316455841064,
      "rewards/margins": 0.5970726013183594,
      "rewards/rejected": -1.6118041276931763,
      "step": 5598
    },
    {
      "epoch": 0.7326735912325182,
      "grad_norm": 10.14109992980957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07624337077140808,
      "logits/rejected": -0.02432149276137352,
      "logps/chosen": -417.7436218261719,
      "logps/rejected": -443.3527526855469,
      "loss": 0.6641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7928197979927063,
      "rewards/margins": 1.2344639301300049,
      "rewards/rejected": -2.0272836685180664,
      "step": 5599
    },
    {
      "epoch": 0.73280444916987,
      "grad_norm": 4.617404937744141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16173306107521057,
      "logits/rejected": -0.011869743466377258,
      "logps/chosen": -346.36859130859375,
      "logps/rejected": -384.19940185546875,
      "loss": 0.2812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17482168972492218,
      "rewards/margins": 2.1454877853393555,
      "rewards/rejected": -1.9706658124923706,
      "step": 5600
    },
    {
      "epoch": 0.7329353071072218,
      "grad_norm": 9.977113723754883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06496807932853699,
      "logits/rejected": -0.0738246962428093,
      "logps/chosen": -305.2939147949219,
      "logps/rejected": -331.433349609375,
      "loss": 1.1296,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9178599119186401,
      "rewards/margins": -0.3731590509414673,
      "rewards/rejected": -0.5447008013725281,
      "step": 5601
    },
    {
      "epoch": 0.7330661650445734,
      "grad_norm": 6.620865821838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052675653249025345,
      "logits/rejected": 0.012672850862145424,
      "logps/chosen": -558.8126220703125,
      "logps/rejected": -604.8226928710938,
      "loss": 0.394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21769294142723083,
      "rewards/margins": 1.6448252201080322,
      "rewards/rejected": -1.8625181913375854,
      "step": 5602
    },
    {
      "epoch": 0.7331970229819252,
      "grad_norm": 8.082866668701172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0668659657239914,
      "logits/rejected": 0.08405141532421112,
      "logps/chosen": -454.73773193359375,
      "logps/rejected": -433.56072998046875,
      "loss": 0.5532,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8082461357116699,
      "rewards/margins": 0.8918253183364868,
      "rewards/rejected": -1.7000715732574463,
      "step": 5603
    },
    {
      "epoch": 0.733327880919277,
      "grad_norm": 8.714876174926758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18742041289806366,
      "logits/rejected": -0.08005654066801071,
      "logps/chosen": -457.4732971191406,
      "logps/rejected": -459.44659423828125,
      "loss": 0.516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06642754375934601,
      "rewards/margins": 1.3732490539550781,
      "rewards/rejected": -1.4396764039993286,
      "step": 5604
    },
    {
      "epoch": 0.7334587388566288,
      "grad_norm": 4.99322509765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05166635662317276,
      "logits/rejected": 0.047452110797166824,
      "logps/chosen": -586.1450805664062,
      "logps/rejected": -617.1265869140625,
      "loss": 0.33,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23728792369365692,
      "rewards/margins": 1.5443849563598633,
      "rewards/rejected": -1.781672716140747,
      "step": 5605
    },
    {
      "epoch": 0.7335895967939805,
      "grad_norm": 18.25937843322754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21937811374664307,
      "logits/rejected": -0.32784223556518555,
      "logps/chosen": -909.3158569335938,
      "logps/rejected": -757.767578125,
      "loss": 1.1108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3066315650939941,
      "rewards/margins": 0.40152397751808167,
      "rewards/rejected": -1.7081555128097534,
      "step": 5606
    },
    {
      "epoch": 0.7337204547313323,
      "grad_norm": 7.03079891204834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13170218467712402,
      "logits/rejected": -0.15707537531852722,
      "logps/chosen": -397.1346435546875,
      "logps/rejected": -477.1661682128906,
      "loss": 0.5872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.362481713294983,
      "rewards/margins": 0.656549870967865,
      "rewards/rejected": -2.019031524658203,
      "step": 5607
    },
    {
      "epoch": 0.7338513126686841,
      "grad_norm": 8.554533004760742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14079813659191132,
      "logits/rejected": 0.034759942442178726,
      "logps/chosen": -664.975830078125,
      "logps/rejected": -610.1851196289062,
      "loss": 0.6749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7931289672851562,
      "rewards/margins": 0.2811082601547241,
      "rewards/rejected": -1.0742372274398804,
      "step": 5608
    },
    {
      "epoch": 0.7339821706060359,
      "grad_norm": 7.85087776184082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015749458223581314,
      "logits/rejected": -0.062094058841466904,
      "logps/chosen": -301.76361083984375,
      "logps/rejected": -377.783203125,
      "loss": 0.5987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10167120397090912,
      "rewards/margins": 0.7681518197059631,
      "rewards/rejected": -0.8698230981826782,
      "step": 5609
    },
    {
      "epoch": 0.7341130285433876,
      "grad_norm": 7.400085926055908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04366768151521683,
      "logits/rejected": 0.0005711913108825684,
      "logps/chosen": -347.41241455078125,
      "logps/rejected": -254.76324462890625,
      "loss": 0.8015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0514674186706543,
      "rewards/margins": 0.31404417753219604,
      "rewards/rejected": -1.3655115365982056,
      "step": 5610
    },
    {
      "epoch": 0.7342438864807393,
      "grad_norm": 6.894937515258789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0478459894657135,
      "logits/rejected": -0.33978182077407837,
      "logps/chosen": -503.8825988769531,
      "logps/rejected": -468.0935363769531,
      "loss": 0.3691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09622381627559662,
      "rewards/margins": 1.3509057760238647,
      "rewards/rejected": -1.254681944847107,
      "step": 5611
    },
    {
      "epoch": 0.7343747444180911,
      "grad_norm": 5.130123615264893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10727977007627487,
      "logits/rejected": -0.06270575523376465,
      "logps/chosen": -460.70672607421875,
      "logps/rejected": -562.3530883789062,
      "loss": 0.3684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5835697054862976,
      "rewards/margins": 1.054877519607544,
      "rewards/rejected": -1.6384471654891968,
      "step": 5612
    },
    {
      "epoch": 0.7345056023554428,
      "grad_norm": 11.580721855163574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08435748517513275,
      "logits/rejected": -0.055515266954898834,
      "logps/chosen": -348.0813903808594,
      "logps/rejected": -391.8431701660156,
      "loss": 0.8922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7618266344070435,
      "rewards/margins": 0.13875052332878113,
      "rewards/rejected": -0.900577187538147,
      "step": 5613
    },
    {
      "epoch": 0.7346364602927946,
      "grad_norm": 14.96280574798584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23792009055614471,
      "logits/rejected": -0.04771541804075241,
      "logps/chosen": -392.0634765625,
      "logps/rejected": -693.2015380859375,
      "loss": 1.026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2909903526306152,
      "rewards/margins": 0.9623791575431824,
      "rewards/rejected": -2.2533693313598633,
      "step": 5614
    },
    {
      "epoch": 0.7347673182301464,
      "grad_norm": 9.840778350830078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12199132144451141,
      "logits/rejected": -0.033380623906850815,
      "logps/chosen": -408.0293273925781,
      "logps/rejected": -404.1521911621094,
      "loss": 0.8786,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6495453119277954,
      "rewards/margins": -0.19613343477249146,
      "rewards/rejected": -0.4534118175506592,
      "step": 5615
    },
    {
      "epoch": 0.7348981761674982,
      "grad_norm": 7.188260555267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01798272132873535,
      "logits/rejected": -0.17536470293998718,
      "logps/chosen": -420.28411865234375,
      "logps/rejected": -549.0880126953125,
      "loss": 0.7429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5776763558387756,
      "rewards/margins": 0.8906036019325256,
      "rewards/rejected": -1.4682800769805908,
      "step": 5616
    },
    {
      "epoch": 0.7350290341048499,
      "grad_norm": 11.272590637207031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08159953355789185,
      "logits/rejected": -0.11322599649429321,
      "logps/chosen": -340.08294677734375,
      "logps/rejected": -431.3687744140625,
      "loss": 1.0925,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.5040597915649414,
      "rewards/margins": -0.23743528127670288,
      "rewards/rejected": -1.2666245698928833,
      "step": 5617
    },
    {
      "epoch": 0.7351598920422017,
      "grad_norm": 7.654011249542236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1542547047138214,
      "logits/rejected": 0.022095125168561935,
      "logps/chosen": -615.4178466796875,
      "logps/rejected": -483.4311828613281,
      "loss": 0.596,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5266422629356384,
      "rewards/margins": 0.7764670252799988,
      "rewards/rejected": -1.3031091690063477,
      "step": 5618
    },
    {
      "epoch": 0.7352907499795535,
      "grad_norm": 8.316241264343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016101613640785217,
      "logits/rejected": -0.05793764814734459,
      "logps/chosen": -599.5736694335938,
      "logps/rejected": -496.40155029296875,
      "loss": 0.5314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5822341442108154,
      "rewards/margins": 0.8337005972862244,
      "rewards/rejected": -1.4159348011016846,
      "step": 5619
    },
    {
      "epoch": 0.7354216079169053,
      "grad_norm": 8.006645202636719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022759757936000824,
      "logits/rejected": 0.058259837329387665,
      "logps/chosen": -384.3532409667969,
      "logps/rejected": -434.5975341796875,
      "loss": 0.5155,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8857054114341736,
      "rewards/margins": 0.753962516784668,
      "rewards/rejected": -1.6396678686141968,
      "step": 5620
    },
    {
      "epoch": 0.7355524658542569,
      "grad_norm": 6.1921706199646,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11426597833633423,
      "logits/rejected": -0.14655424654483795,
      "logps/chosen": -529.1661987304688,
      "logps/rejected": -454.0113830566406,
      "loss": 0.4964,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05400809645652771,
      "rewards/margins": 1.3125746250152588,
      "rewards/rejected": -1.3665826320648193,
      "step": 5621
    },
    {
      "epoch": 0.7356833237916087,
      "grad_norm": 9.952574729919434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.133520245552063,
      "logits/rejected": -0.1413128525018692,
      "logps/chosen": -351.3822021484375,
      "logps/rejected": -369.252685546875,
      "loss": 0.8044,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8920704126358032,
      "rewards/margins": 0.41819268465042114,
      "rewards/rejected": -1.3102630376815796,
      "step": 5622
    },
    {
      "epoch": 0.7358141817289605,
      "grad_norm": 5.028470993041992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07864205539226532,
      "logits/rejected": -0.02605351060628891,
      "logps/chosen": -641.8583984375,
      "logps/rejected": -577.6936645507812,
      "loss": 0.3309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5529114007949829,
      "rewards/margins": 2.055881977081299,
      "rewards/rejected": -1.5029704570770264,
      "step": 5623
    },
    {
      "epoch": 0.7359450396663123,
      "grad_norm": 9.814566612243652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24022284150123596,
      "logits/rejected": 0.12612000107765198,
      "logps/chosen": -400.10223388671875,
      "logps/rejected": -391.9240417480469,
      "loss": 0.5658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.501898467540741,
      "rewards/margins": 1.0043357610702515,
      "rewards/rejected": -1.5062341690063477,
      "step": 5624
    },
    {
      "epoch": 0.736075897603664,
      "grad_norm": 5.342180252075195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05070466175675392,
      "logits/rejected": 0.08020606637001038,
      "logps/chosen": -309.9464416503906,
      "logps/rejected": -400.4035949707031,
      "loss": 0.3866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11430873721837997,
      "rewards/margins": 1.036934733390808,
      "rewards/rejected": -1.1512434482574463,
      "step": 5625
    },
    {
      "epoch": 0.7362067555410158,
      "grad_norm": 7.133869171142578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05284454673528671,
      "logits/rejected": 0.0662650316953659,
      "logps/chosen": -400.82275390625,
      "logps/rejected": -470.8895263671875,
      "loss": 0.4684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4976329803466797,
      "rewards/margins": 1.1024067401885986,
      "rewards/rejected": -1.6000397205352783,
      "step": 5626
    },
    {
      "epoch": 0.7363376134783676,
      "grad_norm": 7.90981912612915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11299031972885132,
      "logits/rejected": -0.32604432106018066,
      "logps/chosen": -296.4056091308594,
      "logps/rejected": -279.65606689453125,
      "loss": 0.4858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6467483043670654,
      "rewards/margins": 0.7093587517738342,
      "rewards/rejected": -1.3561071157455444,
      "step": 5627
    },
    {
      "epoch": 0.7364684714157194,
      "grad_norm": 9.779265403747559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08539128303527832,
      "logits/rejected": 0.0459698922932148,
      "logps/chosen": -467.2215881347656,
      "logps/rejected": -452.38330078125,
      "loss": 0.6242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01772327721118927,
      "rewards/margins": 0.8774152994155884,
      "rewards/rejected": -0.8596920371055603,
      "step": 5628
    },
    {
      "epoch": 0.736599329353071,
      "grad_norm": 8.888920783996582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1477517932653427,
      "logits/rejected": 0.09671460092067719,
      "logps/chosen": -364.4869079589844,
      "logps/rejected": -360.6385803222656,
      "loss": 0.8953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.366315245628357,
      "rewards/margins": 0.0645132064819336,
      "rewards/rejected": -1.4308284521102905,
      "step": 5629
    },
    {
      "epoch": 0.7367301872904228,
      "grad_norm": 10.14643669128418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2113620489835739,
      "logits/rejected": -0.2791038155555725,
      "logps/chosen": -432.1859436035156,
      "logps/rejected": -396.0794677734375,
      "loss": 0.7841,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2375366687774658,
      "rewards/margins": 0.3377983272075653,
      "rewards/rejected": -1.575334906578064,
      "step": 5630
    },
    {
      "epoch": 0.7368610452277746,
      "grad_norm": 4.680689811706543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08575496822595596,
      "logits/rejected": 0.05950013920664787,
      "logps/chosen": -354.4505615234375,
      "logps/rejected": -304.14874267578125,
      "loss": 0.3066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14114496111869812,
      "rewards/margins": 1.3884285688400269,
      "rewards/rejected": -1.2472835779190063,
      "step": 5631
    },
    {
      "epoch": 0.7369919031651263,
      "grad_norm": 5.382416248321533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2675262987613678,
      "logits/rejected": 0.20425474643707275,
      "logps/chosen": -453.4053955078125,
      "logps/rejected": -491.2200927734375,
      "loss": 0.3839,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4664573669433594,
      "rewards/margins": 0.9515232443809509,
      "rewards/rejected": -1.417980670928955,
      "step": 5632
    },
    {
      "epoch": 0.7371227611024781,
      "grad_norm": 6.710596561431885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15961597859859467,
      "logits/rejected": 0.10153614729642868,
      "logps/chosen": -409.1416015625,
      "logps/rejected": -392.87664794921875,
      "loss": 0.4604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15976600348949432,
      "rewards/margins": 0.8749231696128845,
      "rewards/rejected": -0.7151570916175842,
      "step": 5633
    },
    {
      "epoch": 0.7372536190398299,
      "grad_norm": 6.650737285614014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006967456080019474,
      "logits/rejected": 0.053433775901794434,
      "logps/chosen": -283.00836181640625,
      "logps/rejected": -362.6161804199219,
      "loss": 0.4632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5022006630897522,
      "rewards/margins": 0.7187808156013489,
      "rewards/rejected": -1.220981478691101,
      "step": 5634
    },
    {
      "epoch": 0.7373844769771817,
      "grad_norm": 11.468437194824219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24644699692726135,
      "logits/rejected": 0.01652846857905388,
      "logps/chosen": -456.00042724609375,
      "logps/rejected": -584.830810546875,
      "loss": 0.887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1828923225402832,
      "rewards/margins": 0.636905312538147,
      "rewards/rejected": -1.8197975158691406,
      "step": 5635
    },
    {
      "epoch": 0.7375153349145334,
      "grad_norm": 7.212628364562988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022134415805339813,
      "logits/rejected": 0.14832185208797455,
      "logps/chosen": -761.376953125,
      "logps/rejected": -446.345947265625,
      "loss": 0.3336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6258730292320251,
      "rewards/margins": 1.0603125095367432,
      "rewards/rejected": -1.686185359954834,
      "step": 5636
    },
    {
      "epoch": 0.7376461928518852,
      "grad_norm": 4.813560962677002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10254333913326263,
      "logits/rejected": 0.14041802287101746,
      "logps/chosen": -337.63800048828125,
      "logps/rejected": -292.23602294921875,
      "loss": 0.3881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6718839406967163,
      "rewards/margins": 0.9769734144210815,
      "rewards/rejected": -1.6488573551177979,
      "step": 5637
    },
    {
      "epoch": 0.7377770507892369,
      "grad_norm": 8.188911437988281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05789410322904587,
      "logits/rejected": -0.118140809237957,
      "logps/chosen": -400.2813415527344,
      "logps/rejected": -377.1700134277344,
      "loss": 0.4888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04235747456550598,
      "rewards/margins": 1.8252317905426025,
      "rewards/rejected": -1.8675892353057861,
      "step": 5638
    },
    {
      "epoch": 0.7379079087265887,
      "grad_norm": 7.246467590332031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048351041972637177,
      "logits/rejected": 0.05056177079677582,
      "logps/chosen": -408.4263610839844,
      "logps/rejected": -383.6872863769531,
      "loss": 0.4886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6963125467300415,
      "rewards/margins": 0.7136507034301758,
      "rewards/rejected": -1.4099632501602173,
      "step": 5639
    },
    {
      "epoch": 0.7380387666639404,
      "grad_norm": 8.035449028015137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10758887231349945,
      "logits/rejected": 0.006701845675706863,
      "logps/chosen": -368.6286926269531,
      "logps/rejected": -408.26007080078125,
      "loss": 0.5897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24309006333351135,
      "rewards/margins": 1.2244534492492676,
      "rewards/rejected": -1.467543363571167,
      "step": 5640
    },
    {
      "epoch": 0.7381696246012922,
      "grad_norm": 7.417865753173828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09050551801919937,
      "logits/rejected": -0.2486344873905182,
      "logps/chosen": -342.1813659667969,
      "logps/rejected": -397.1926574707031,
      "loss": 0.5391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.787614107131958,
      "rewards/margins": 0.6989424228668213,
      "rewards/rejected": -1.4865565299987793,
      "step": 5641
    },
    {
      "epoch": 0.738300482538644,
      "grad_norm": 3.324542284011841,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09156240522861481,
      "logits/rejected": 0.15469756722450256,
      "logps/chosen": -427.17388916015625,
      "logps/rejected": -485.8104553222656,
      "loss": 0.2552,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08036483824253082,
      "rewards/margins": 1.8328121900558472,
      "rewards/rejected": -1.752447485923767,
      "step": 5642
    },
    {
      "epoch": 0.7384313404759958,
      "grad_norm": 5.408874988555908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15175476670265198,
      "logits/rejected": 0.13587608933448792,
      "logps/chosen": -338.3365478515625,
      "logps/rejected": -381.1759033203125,
      "loss": 0.4159,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2948036193847656,
      "rewards/margins": 0.8238371014595032,
      "rewards/rejected": -1.118640661239624,
      "step": 5643
    },
    {
      "epoch": 0.7385621984133475,
      "grad_norm": 9.430066108703613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10655421018600464,
      "logits/rejected": 0.17053215205669403,
      "logps/chosen": -290.2283935546875,
      "logps/rejected": -382.848876953125,
      "loss": 0.7392,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43245869874954224,
      "rewards/margins": 0.5927358865737915,
      "rewards/rejected": -1.025194525718689,
      "step": 5644
    },
    {
      "epoch": 0.7386930563506993,
      "grad_norm": 5.642970085144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27154114842414856,
      "logits/rejected": -0.49027007818222046,
      "logps/chosen": -557.0905151367188,
      "logps/rejected": -466.37677001953125,
      "loss": 0.4187,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0783853679895401,
      "rewards/margins": 1.13614821434021,
      "rewards/rejected": -1.2145335674285889,
      "step": 5645
    },
    {
      "epoch": 0.738823914288051,
      "grad_norm": 7.035892963409424,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36296337842941284,
      "logits/rejected": 0.30544140934944153,
      "logps/chosen": -525.4483032226562,
      "logps/rejected": -548.1968383789062,
      "loss": 0.3967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26582586765289307,
      "rewards/margins": 1.203415870666504,
      "rewards/rejected": -1.469241738319397,
      "step": 5646
    },
    {
      "epoch": 0.7389547722254028,
      "grad_norm": 10.675921440124512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06745448708534241,
      "logits/rejected": -0.1639844924211502,
      "logps/chosen": -416.47442626953125,
      "logps/rejected": -345.1501770019531,
      "loss": 0.7241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3982213735580444,
      "rewards/margins": 0.5830807089805603,
      "rewards/rejected": -1.98130202293396,
      "step": 5647
    },
    {
      "epoch": 0.7390856301627545,
      "grad_norm": 8.417943954467773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.365482360124588,
      "logits/rejected": 0.11112169176340103,
      "logps/chosen": -370.6221008300781,
      "logps/rejected": -451.3468322753906,
      "loss": 0.587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13923777639865875,
      "rewards/margins": 0.7710480093955994,
      "rewards/rejected": -0.6318102478981018,
      "step": 5648
    },
    {
      "epoch": 0.7392164881001063,
      "grad_norm": 5.399268627166748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032624050974845886,
      "logits/rejected": -0.11512860655784607,
      "logps/chosen": -451.2227783203125,
      "logps/rejected": -349.5442810058594,
      "loss": 0.3508,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.023808583617210388,
      "rewards/margins": 1.1343214511871338,
      "rewards/rejected": -1.1105128526687622,
      "step": 5649
    },
    {
      "epoch": 0.7393473460374581,
      "grad_norm": 7.563327789306641,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06729191541671753,
      "logits/rejected": 0.022572554647922516,
      "logps/chosen": -354.98193359375,
      "logps/rejected": -308.84393310546875,
      "loss": 0.4615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6785293221473694,
      "rewards/margins": 0.8041211366653442,
      "rewards/rejected": -1.482650637626648,
      "step": 5650
    },
    {
      "epoch": 0.7394782039748099,
      "grad_norm": 9.693781852722168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08297045528888702,
      "logits/rejected": -0.08204497396945953,
      "logps/chosen": -493.65045166015625,
      "logps/rejected": -413.97052001953125,
      "loss": 0.8725,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2169332504272461,
      "rewards/margins": 0.25244519114494324,
      "rewards/rejected": -0.46937844157218933,
      "step": 5651
    },
    {
      "epoch": 0.7396090619121616,
      "grad_norm": 6.109484672546387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14949284493923187,
      "logits/rejected": 0.12070260941982269,
      "logps/chosen": -425.1108093261719,
      "logps/rejected": -442.6053161621094,
      "loss": 0.4301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1219472885131836,
      "rewards/margins": 0.9254100918769836,
      "rewards/rejected": -1.0473573207855225,
      "step": 5652
    },
    {
      "epoch": 0.7397399198495134,
      "grad_norm": 7.311893939971924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19130340218544006,
      "logits/rejected": -0.20362377166748047,
      "logps/chosen": -693.06005859375,
      "logps/rejected": -664.0298461914062,
      "loss": 0.4639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7584825754165649,
      "rewards/margins": 0.8494917750358582,
      "rewards/rejected": -1.6079742908477783,
      "step": 5653
    },
    {
      "epoch": 0.7398707777868652,
      "grad_norm": 5.5704522132873535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26804235577583313,
      "logits/rejected": -0.10662902146577835,
      "logps/chosen": -476.4576416015625,
      "logps/rejected": -414.45062255859375,
      "loss": 0.3949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5505955219268799,
      "rewards/margins": 1.3035759925842285,
      "rewards/rejected": -1.8541713953018188,
      "step": 5654
    },
    {
      "epoch": 0.7400016357242168,
      "grad_norm": 9.186966896057129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17686723172664642,
      "logits/rejected": 0.13514702022075653,
      "logps/chosen": -348.0303649902344,
      "logps/rejected": -364.6077880859375,
      "loss": 0.478,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3276265859603882,
      "rewards/margins": 1.0203701257705688,
      "rewards/rejected": -1.347996711730957,
      "step": 5655
    },
    {
      "epoch": 0.7401324936615686,
      "grad_norm": 4.509510517120361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07553035020828247,
      "logits/rejected": -0.05804317817091942,
      "logps/chosen": -551.7921752929688,
      "logps/rejected": -377.8252868652344,
      "loss": 0.2484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3817816972732544,
      "rewards/margins": 1.8614932298660278,
      "rewards/rejected": -1.4797115325927734,
      "step": 5656
    },
    {
      "epoch": 0.7402633515989204,
      "grad_norm": 7.513937950134277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21290573477745056,
      "logits/rejected": 0.10427253693342209,
      "logps/chosen": -526.1168212890625,
      "logps/rejected": -403.6258239746094,
      "loss": 0.4979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47005367279052734,
      "rewards/margins": 1.2285076379776,
      "rewards/rejected": -1.698561191558838,
      "step": 5657
    },
    {
      "epoch": 0.7403942095362722,
      "grad_norm": 10.222712516784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.036989957094192505,
      "logits/rejected": -0.15990671515464783,
      "logps/chosen": -468.51727294921875,
      "logps/rejected": -411.0076599121094,
      "loss": 0.7669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0271142721176147,
      "rewards/margins": 0.38993239402770996,
      "rewards/rejected": -1.4170466661453247,
      "step": 5658
    },
    {
      "epoch": 0.7405250674736239,
      "grad_norm": 10.275352478027344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10096529126167297,
      "logits/rejected": 0.09051978588104248,
      "logps/chosen": -533.3353881835938,
      "logps/rejected": -549.0655517578125,
      "loss": 0.7329,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11065743863582611,
      "rewards/margins": 0.21347247064113617,
      "rewards/rejected": -0.10281504690647125,
      "step": 5659
    },
    {
      "epoch": 0.7406559254109757,
      "grad_norm": 4.948709011077881,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06764408946037292,
      "logits/rejected": -0.1473744809627533,
      "logps/chosen": -499.47625732421875,
      "logps/rejected": -459.15643310546875,
      "loss": 0.3292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03786200284957886,
      "rewards/margins": 1.4820501804351807,
      "rewards/rejected": -1.5199122428894043,
      "step": 5660
    },
    {
      "epoch": 0.7407867833483275,
      "grad_norm": 7.8973259925842285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016973860561847687,
      "logits/rejected": -0.15785406529903412,
      "logps/chosen": -654.4910278320312,
      "logps/rejected": -483.22833251953125,
      "loss": 0.5011,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.054654285311698914,
      "rewards/margins": 0.9000906348228455,
      "rewards/rejected": -0.9547449350357056,
      "step": 5661
    },
    {
      "epoch": 0.7409176412856793,
      "grad_norm": 10.934818267822266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017435580492019653,
      "logits/rejected": -0.05815492197871208,
      "logps/chosen": -428.0207214355469,
      "logps/rejected": -460.71246337890625,
      "loss": 0.9198,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6768465042114258,
      "rewards/margins": 0.21264603734016418,
      "rewards/rejected": -0.8894926309585571,
      "step": 5662
    },
    {
      "epoch": 0.741048499223031,
      "grad_norm": 7.629903793334961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11962980031967163,
      "logits/rejected": -0.3755987882614136,
      "logps/chosen": -359.3642578125,
      "logps/rejected": -344.857177734375,
      "loss": 0.5838,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9271355271339417,
      "rewards/margins": 0.4613654911518097,
      "rewards/rejected": -1.3885011672973633,
      "step": 5663
    },
    {
      "epoch": 0.7411793571603827,
      "grad_norm": 4.526996612548828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06319911777973175,
      "logits/rejected": -0.09728562831878662,
      "logps/chosen": -473.1355285644531,
      "logps/rejected": -532.5701904296875,
      "loss": 0.2158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19459420442581177,
      "rewards/margins": 1.7743678092956543,
      "rewards/rejected": -1.5797735452651978,
      "step": 5664
    },
    {
      "epoch": 0.7413102150977345,
      "grad_norm": 15.348264694213867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2691882848739624,
      "logits/rejected": 0.24403734505176544,
      "logps/chosen": -583.4112548828125,
      "logps/rejected": -480.1568908691406,
      "loss": 1.4176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3809356689453125,
      "rewards/margins": -0.5803558230400085,
      "rewards/rejected": -0.8005799055099487,
      "step": 5665
    },
    {
      "epoch": 0.7414410730350863,
      "grad_norm": 8.090422630310059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08356864005327225,
      "logits/rejected": -0.024676229804754257,
      "logps/chosen": -619.4447631835938,
      "logps/rejected": -574.5841064453125,
      "loss": 0.4576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5055265426635742,
      "rewards/margins": 1.6622167825698853,
      "rewards/rejected": -2.167743444442749,
      "step": 5666
    },
    {
      "epoch": 0.741571930972438,
      "grad_norm": 5.602727890014648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14314289391040802,
      "logits/rejected": -0.025768112391233444,
      "logps/chosen": -394.9670715332031,
      "logps/rejected": -378.0960998535156,
      "loss": 0.4063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32947486639022827,
      "rewards/margins": 1.0957573652267456,
      "rewards/rejected": -1.425232172012329,
      "step": 5667
    },
    {
      "epoch": 0.7417027889097898,
      "grad_norm": 6.850131988525391,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08080681413412094,
      "logits/rejected": -0.0442797914147377,
      "logps/chosen": -239.46279907226562,
      "logps/rejected": -340.3184509277344,
      "loss": 0.6482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.256500244140625,
      "rewards/margins": 0.6440432071685791,
      "rewards/rejected": -0.9005434513092041,
      "step": 5668
    },
    {
      "epoch": 0.7418336468471416,
      "grad_norm": 3.9396708011627197,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009069956839084625,
      "logits/rejected": -0.03804498165845871,
      "logps/chosen": -528.798095703125,
      "logps/rejected": -533.5501098632812,
      "loss": 0.2088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25972846150398254,
      "rewards/margins": 1.6058776378631592,
      "rewards/rejected": -1.8656060695648193,
      "step": 5669
    },
    {
      "epoch": 0.7419645047844934,
      "grad_norm": 6.833922386169434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06044524908065796,
      "logits/rejected": -0.27933189272880554,
      "logps/chosen": -506.5538330078125,
      "logps/rejected": -557.9925537109375,
      "loss": 0.3928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.42138099670410156,
      "rewards/margins": 0.9480488300323486,
      "rewards/rejected": -0.5266677737236023,
      "step": 5670
    },
    {
      "epoch": 0.7420953627218451,
      "grad_norm": 8.771475791931152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1749533712863922,
      "logits/rejected": 0.22066456079483032,
      "logps/chosen": -392.38995361328125,
      "logps/rejected": -470.5674743652344,
      "loss": 0.7805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2625184953212738,
      "rewards/margins": 0.3114965558052063,
      "rewards/rejected": -0.5740150213241577,
      "step": 5671
    },
    {
      "epoch": 0.7422262206591969,
      "grad_norm": 4.566961765289307,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1535259336233139,
      "logits/rejected": -0.37523046135902405,
      "logps/chosen": -371.1182556152344,
      "logps/rejected": -296.2350158691406,
      "loss": 0.2842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38986343145370483,
      "rewards/margins": 1.6421704292297363,
      "rewards/rejected": -1.2523069381713867,
      "step": 5672
    },
    {
      "epoch": 0.7423570785965486,
      "grad_norm": 6.192445755004883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2271287739276886,
      "logits/rejected": -0.17336225509643555,
      "logps/chosen": -334.2639465332031,
      "logps/rejected": -386.742431640625,
      "loss": 0.5798,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3909417390823364,
      "rewards/margins": 0.6464102864265442,
      "rewards/rejected": -2.0373518466949463,
      "step": 5673
    },
    {
      "epoch": 0.7424879365339004,
      "grad_norm": 6.0686492919921875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09464111924171448,
      "logits/rejected": -0.031182795763015747,
      "logps/chosen": -441.0816650390625,
      "logps/rejected": -411.6151428222656,
      "loss": 0.4783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5542711019515991,
      "rewards/margins": 1.0533983707427979,
      "rewards/rejected": -0.49912720918655396,
      "step": 5674
    },
    {
      "epoch": 0.7426187944712521,
      "grad_norm": 5.138594627380371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06360287964344025,
      "logits/rejected": 0.19407638907432556,
      "logps/chosen": -418.279052734375,
      "logps/rejected": -435.6947021484375,
      "loss": 0.336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2658922076225281,
      "rewards/margins": 1.1330535411834717,
      "rewards/rejected": -0.8671613931655884,
      "step": 5675
    },
    {
      "epoch": 0.7427496524086039,
      "grad_norm": 12.169814109802246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3705183267593384,
      "logits/rejected": -0.40356433391571045,
      "logps/chosen": -422.5909118652344,
      "logps/rejected": -361.64178466796875,
      "loss": 0.8233,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1152689456939697,
      "rewards/margins": 0.38791459798812866,
      "rewards/rejected": -1.5031836032867432,
      "step": 5676
    },
    {
      "epoch": 0.7428805103459557,
      "grad_norm": 4.013404846191406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09081801772117615,
      "logits/rejected": -0.012668736279010773,
      "logps/chosen": -263.395263671875,
      "logps/rejected": -318.111328125,
      "loss": 0.2981,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6872579455375671,
      "rewards/margins": 1.5898945331573486,
      "rewards/rejected": -2.2771525382995605,
      "step": 5677
    },
    {
      "epoch": 0.7430113682833074,
      "grad_norm": 10.631437301635742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08243072777986526,
      "logits/rejected": 0.021466609090566635,
      "logps/chosen": -384.0011291503906,
      "logps/rejected": -443.33184814453125,
      "loss": 0.5837,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33615076541900635,
      "rewards/margins": 0.9253689646720886,
      "rewards/rejected": -1.2615196704864502,
      "step": 5678
    },
    {
      "epoch": 0.7431422262206592,
      "grad_norm": 7.933259963989258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21331235766410828,
      "logits/rejected": -0.25106269121170044,
      "logps/chosen": -415.2022705078125,
      "logps/rejected": -479.2518310546875,
      "loss": 0.5173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2669669985771179,
      "rewards/margins": 1.166090726852417,
      "rewards/rejected": -1.4330577850341797,
      "step": 5679
    },
    {
      "epoch": 0.743273084158011,
      "grad_norm": 5.453070640563965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0008857548236846924,
      "logits/rejected": -0.1840813308954239,
      "logps/chosen": -453.7391052246094,
      "logps/rejected": -406.10260009765625,
      "loss": 0.3928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.516218900680542,
      "rewards/margins": 1.429173469543457,
      "rewards/rejected": -1.9453926086425781,
      "step": 5680
    },
    {
      "epoch": 0.7434039420953628,
      "grad_norm": 10.675015449523926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14406763017177582,
      "logits/rejected": -0.010889008641242981,
      "logps/chosen": -337.4229431152344,
      "logps/rejected": -406.82568359375,
      "loss": 0.899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38173502683639526,
      "rewards/margins": 0.23421308398246765,
      "rewards/rejected": -0.6159481406211853,
      "step": 5681
    },
    {
      "epoch": 0.7435348000327144,
      "grad_norm": 7.138400554656982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054355423897504807,
      "logits/rejected": -0.025472963228821754,
      "logps/chosen": -342.76190185546875,
      "logps/rejected": -421.4198303222656,
      "loss": 0.5314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3994041383266449,
      "rewards/margins": 0.7419158816337585,
      "rewards/rejected": -1.1413201093673706,
      "step": 5682
    },
    {
      "epoch": 0.7436656579700662,
      "grad_norm": 4.658628940582275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013490848243236542,
      "logits/rejected": -0.18891367316246033,
      "logps/chosen": -353.06134033203125,
      "logps/rejected": -342.90692138671875,
      "loss": 0.3307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07321280241012573,
      "rewards/margins": 1.2750176191329956,
      "rewards/rejected": -1.3482304811477661,
      "step": 5683
    },
    {
      "epoch": 0.743796515907418,
      "grad_norm": 6.228635787963867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3011705279350281,
      "logits/rejected": -0.2907169461250305,
      "logps/chosen": -310.3171081542969,
      "logps/rejected": -429.4457702636719,
      "loss": 0.508,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0076985359191895,
      "rewards/margins": 1.4360811710357666,
      "rewards/rejected": -2.443779706954956,
      "step": 5684
    },
    {
      "epoch": 0.7439273738447698,
      "grad_norm": 5.293989658355713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09886129945516586,
      "logits/rejected": 0.07792676240205765,
      "logps/chosen": -587.16943359375,
      "logps/rejected": -410.5845031738281,
      "loss": 0.2797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09090021252632141,
      "rewards/margins": 1.6900532245635986,
      "rewards/rejected": -1.5991530418395996,
      "step": 5685
    },
    {
      "epoch": 0.7440582317821215,
      "grad_norm": 6.274284362792969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1813012957572937,
      "logits/rejected": 0.1678132712841034,
      "logps/chosen": -492.3191833496094,
      "logps/rejected": -518.910400390625,
      "loss": 0.4162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6208375692367554,
      "rewards/margins": 1.1446876525878906,
      "rewards/rejected": -0.5238500833511353,
      "step": 5686
    },
    {
      "epoch": 0.7441890897194733,
      "grad_norm": 5.092549800872803,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39010369777679443,
      "logits/rejected": 0.2740609049797058,
      "logps/chosen": -391.29449462890625,
      "logps/rejected": -392.9920654296875,
      "loss": 0.3231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5650487542152405,
      "rewards/margins": 1.738160490989685,
      "rewards/rejected": -2.3032093048095703,
      "step": 5687
    },
    {
      "epoch": 0.7443199476568251,
      "grad_norm": 10.868379592895508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15371575951576233,
      "logits/rejected": -0.03016672655940056,
      "logps/chosen": -401.8536071777344,
      "logps/rejected": -328.5325927734375,
      "loss": 0.6113,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6497633457183838,
      "rewards/margins": 0.6059377789497375,
      "rewards/rejected": -1.2557010650634766,
      "step": 5688
    },
    {
      "epoch": 0.7444508055941769,
      "grad_norm": 11.284588813781738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06752561032772064,
      "logits/rejected": -0.1707526594400406,
      "logps/chosen": -384.3905944824219,
      "logps/rejected": -302.129150390625,
      "loss": 0.5023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1696231365203857,
      "rewards/margins": 1.1167950630187988,
      "rewards/rejected": -2.2864184379577637,
      "step": 5689
    },
    {
      "epoch": 0.7445816635315285,
      "grad_norm": 8.788337707519531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1816408932209015,
      "logits/rejected": 0.13433986902236938,
      "logps/chosen": -586.6141357421875,
      "logps/rejected": -368.9002990722656,
      "loss": 0.4899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.739096999168396,
      "rewards/margins": 1.12497878074646,
      "rewards/rejected": -1.8640758991241455,
      "step": 5690
    },
    {
      "epoch": 0.7447125214688803,
      "grad_norm": 10.618505477905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0696130245923996,
      "logits/rejected": -0.18640105426311493,
      "logps/chosen": -499.7138671875,
      "logps/rejected": -522.61474609375,
      "loss": 0.6262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2637392282485962,
      "rewards/margins": 0.4752207100391388,
      "rewards/rejected": -1.7389599084854126,
      "step": 5691
    },
    {
      "epoch": 0.7448433794062321,
      "grad_norm": 8.637043952941895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32685673236846924,
      "logits/rejected": -0.14778132736682892,
      "logps/chosen": -543.4027709960938,
      "logps/rejected": -434.1889953613281,
      "loss": 0.5349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6575170755386353,
      "rewards/margins": 0.6960775852203369,
      "rewards/rejected": -1.3535945415496826,
      "step": 5692
    },
    {
      "epoch": 0.7449742373435839,
      "grad_norm": 4.263092994689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2553083896636963,
      "logits/rejected": -0.17746339738368988,
      "logps/chosen": -494.1236267089844,
      "logps/rejected": -343.1016845703125,
      "loss": 0.2877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.029331207275390625,
      "rewards/margins": 1.4071438312530518,
      "rewards/rejected": -1.4364750385284424,
      "step": 5693
    },
    {
      "epoch": 0.7451050952809356,
      "grad_norm": 7.9443583488464355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13357849419116974,
      "logits/rejected": -0.13944897055625916,
      "logps/chosen": -375.944091796875,
      "logps/rejected": -439.7626647949219,
      "loss": 0.4954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16155147552490234,
      "rewards/margins": 1.0376818180084229,
      "rewards/rejected": -0.8761302828788757,
      "step": 5694
    },
    {
      "epoch": 0.7452359532182874,
      "grad_norm": 2.6393871307373047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02333122119307518,
      "logits/rejected": 0.1587183028459549,
      "logps/chosen": -714.7714233398438,
      "logps/rejected": -581.6648559570312,
      "loss": 0.1174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4920617938041687,
      "rewards/margins": 2.3303229808807373,
      "rewards/rejected": -1.838261365890503,
      "step": 5695
    },
    {
      "epoch": 0.7453668111556392,
      "grad_norm": 9.66004467010498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08519934117794037,
      "logits/rejected": 0.08081696927547455,
      "logps/chosen": -456.4171142578125,
      "logps/rejected": -461.20098876953125,
      "loss": 0.729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6571205854415894,
      "rewards/margins": 0.22147886455059052,
      "rewards/rejected": -0.8785993456840515,
      "step": 5696
    },
    {
      "epoch": 0.7454976690929909,
      "grad_norm": 7.7071638107299805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10827610641717911,
      "logits/rejected": -0.1280306875705719,
      "logps/chosen": -384.1518859863281,
      "logps/rejected": -362.43109130859375,
      "loss": 0.6967,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9618602991104126,
      "rewards/margins": 0.28311577439308167,
      "rewards/rejected": -1.2449760437011719,
      "step": 5697
    },
    {
      "epoch": 0.7456285270303427,
      "grad_norm": 12.526261329650879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0617394894361496,
      "logits/rejected": -0.1591419279575348,
      "logps/chosen": -598.624267578125,
      "logps/rejected": -539.78515625,
      "loss": 0.7734,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7087616324424744,
      "rewards/margins": 0.7601673603057861,
      "rewards/rejected": -1.4689290523529053,
      "step": 5698
    },
    {
      "epoch": 0.7457593849676944,
      "grad_norm": 5.135851860046387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09088188409805298,
      "logits/rejected": 0.16798314452171326,
      "logps/chosen": -466.9378967285156,
      "logps/rejected": -457.8882751464844,
      "loss": 0.2938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08546219021081924,
      "rewards/margins": 1.4284321069717407,
      "rewards/rejected": -1.5138942003250122,
      "step": 5699
    },
    {
      "epoch": 0.7458902429050462,
      "grad_norm": 11.007756233215332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007813513278961182,
      "logits/rejected": -0.08044914901256561,
      "logps/chosen": -417.9483337402344,
      "logps/rejected": -465.8500061035156,
      "loss": 0.9991,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0127390623092651,
      "rewards/margins": 0.1681250035762787,
      "rewards/rejected": -1.1808639764785767,
      "step": 5700
    },
    {
      "epoch": 0.7460211008423979,
      "grad_norm": 3.6685776710510254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14668744802474976,
      "logits/rejected": -0.046501681208610535,
      "logps/chosen": -401.95269775390625,
      "logps/rejected": -712.901123046875,
      "loss": 0.1936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.025796130299568176,
      "rewards/margins": 1.9612295627593994,
      "rewards/rejected": -1.9354337453842163,
      "step": 5701
    },
    {
      "epoch": 0.7461519587797497,
      "grad_norm": 7.509021282196045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043623097240924835,
      "logits/rejected": 0.0056540705263614655,
      "logps/chosen": -487.560791015625,
      "logps/rejected": -431.5650634765625,
      "loss": 0.4513,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22715133428573608,
      "rewards/margins": 1.1528764963150024,
      "rewards/rejected": -0.9257252216339111,
      "step": 5702
    },
    {
      "epoch": 0.7462828167171015,
      "grad_norm": 9.398374557495117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09775607287883759,
      "logits/rejected": 0.11109570413827896,
      "logps/chosen": -351.7080993652344,
      "logps/rejected": -379.025634765625,
      "loss": 0.874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28787487745285034,
      "rewards/margins": 0.2376946657896042,
      "rewards/rejected": -0.5255695581436157,
      "step": 5703
    },
    {
      "epoch": 0.7464136746544533,
      "grad_norm": 7.85916805267334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35276955366134644,
      "logits/rejected": 0.026260390877723694,
      "logps/chosen": -555.1334838867188,
      "logps/rejected": -505.1950378417969,
      "loss": 0.5612,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48800966143608093,
      "rewards/margins": 0.781167209148407,
      "rewards/rejected": -1.2691768407821655,
      "step": 5704
    },
    {
      "epoch": 0.746544532591805,
      "grad_norm": 6.849079608917236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024952152743935585,
      "logits/rejected": -0.01802171766757965,
      "logps/chosen": -504.97393798828125,
      "logps/rejected": -495.6957092285156,
      "loss": 0.6055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09668733179569244,
      "rewards/margins": 0.37662506103515625,
      "rewards/rejected": -0.279937744140625,
      "step": 5705
    },
    {
      "epoch": 0.7466753905291568,
      "grad_norm": 8.313797950744629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3382694125175476,
      "logits/rejected": -0.27666234970092773,
      "logps/chosen": -389.9279479980469,
      "logps/rejected": -416.676513671875,
      "loss": 0.6118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7818282842636108,
      "rewards/margins": 0.49609583616256714,
      "rewards/rejected": -1.2779241800308228,
      "step": 5706
    },
    {
      "epoch": 0.7468062484665086,
      "grad_norm": 4.740318298339844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02922140061855316,
      "logits/rejected": 0.0029724277555942535,
      "logps/chosen": -549.7296752929688,
      "logps/rejected": -521.636962890625,
      "loss": 0.3258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17340391874313354,
      "rewards/margins": 1.4847363233566284,
      "rewards/rejected": -1.6581401824951172,
      "step": 5707
    },
    {
      "epoch": 0.7469371064038604,
      "grad_norm": 8.439804077148438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19458752870559692,
      "logits/rejected": -0.06392431259155273,
      "logps/chosen": -596.2839965820312,
      "logps/rejected": -521.7318115234375,
      "loss": 0.7393,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6793066263198853,
      "rewards/margins": 0.15459270775318146,
      "rewards/rejected": -0.8338993191719055,
      "step": 5708
    },
    {
      "epoch": 0.747067964341212,
      "grad_norm": 9.633747100830078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1928374171257019,
      "logits/rejected": -0.3655475974082947,
      "logps/chosen": -425.6896057128906,
      "logps/rejected": -406.72149658203125,
      "loss": 0.5594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6673990488052368,
      "rewards/margins": 1.4878177642822266,
      "rewards/rejected": -2.155216693878174,
      "step": 5709
    },
    {
      "epoch": 0.7471988222785638,
      "grad_norm": 4.194070816040039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12213149666786194,
      "logits/rejected": 0.11127267777919769,
      "logps/chosen": -394.32806396484375,
      "logps/rejected": -406.4100646972656,
      "loss": 0.2495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18433327972888947,
      "rewards/margins": 1.782143473625183,
      "rewards/rejected": -1.9664767980575562,
      "step": 5710
    },
    {
      "epoch": 0.7473296802159156,
      "grad_norm": 8.69947624206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16936612129211426,
      "logits/rejected": 0.1530693769454956,
      "logps/chosen": -387.88409423828125,
      "logps/rejected": -337.4129638671875,
      "loss": 0.7505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3366687595844269,
      "rewards/margins": 0.846219539642334,
      "rewards/rejected": -1.182888388633728,
      "step": 5711
    },
    {
      "epoch": 0.7474605381532674,
      "grad_norm": 5.667385578155518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003060676157474518,
      "logits/rejected": -0.07993754744529724,
      "logps/chosen": -398.15863037109375,
      "logps/rejected": -449.1820983886719,
      "loss": 0.3717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3987677991390228,
      "rewards/margins": 1.521645426750183,
      "rewards/rejected": -1.9204132556915283,
      "step": 5712
    },
    {
      "epoch": 0.7475913960906191,
      "grad_norm": 7.039985656738281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1508868932723999,
      "logits/rejected": -0.09304100275039673,
      "logps/chosen": -393.75506591796875,
      "logps/rejected": -603.4915771484375,
      "loss": 0.4761,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7698338031768799,
      "rewards/margins": 0.9046624898910522,
      "rewards/rejected": -1.6744962930679321,
      "step": 5713
    },
    {
      "epoch": 0.7477222540279709,
      "grad_norm": 12.337606430053711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2135576754808426,
      "logits/rejected": -0.22532187402248383,
      "logps/chosen": -458.13482666015625,
      "logps/rejected": -473.2959289550781,
      "loss": 0.8702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11852341890335083,
      "rewards/margins": 0.5620031356811523,
      "rewards/rejected": -0.680526614189148,
      "step": 5714
    },
    {
      "epoch": 0.7478531119653227,
      "grad_norm": 6.946038722991943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04425273835659027,
      "logits/rejected": -0.08917216211557388,
      "logps/chosen": -269.2205810546875,
      "logps/rejected": -416.668701171875,
      "loss": 0.4344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.655626654624939,
      "rewards/margins": 1.3082756996154785,
      "rewards/rejected": -1.9639023542404175,
      "step": 5715
    },
    {
      "epoch": 0.7479839699026745,
      "grad_norm": 3.904567241668701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17516568303108215,
      "logits/rejected": -0.021494094282388687,
      "logps/chosen": -450.4918518066406,
      "logps/rejected": -356.382080078125,
      "loss": 0.2226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2918188273906708,
      "rewards/margins": 2.67714262008667,
      "rewards/rejected": -2.3853235244750977,
      "step": 5716
    },
    {
      "epoch": 0.7481148278400261,
      "grad_norm": 6.77471399307251,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040168724954128265,
      "logits/rejected": 0.33580416440963745,
      "logps/chosen": -494.6812744140625,
      "logps/rejected": -464.1128845214844,
      "loss": 0.4579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07340432703495026,
      "rewards/margins": 0.7801368832588196,
      "rewards/rejected": -0.7067325115203857,
      "step": 5717
    },
    {
      "epoch": 0.7482456857773779,
      "grad_norm": 8.151241302490234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11236544698476791,
      "logits/rejected": -0.08358296751976013,
      "logps/chosen": -412.3029479980469,
      "logps/rejected": -291.01953125,
      "loss": 0.7402,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5404289364814758,
      "rewards/margins": 0.5339056253433228,
      "rewards/rejected": -1.0743345022201538,
      "step": 5718
    },
    {
      "epoch": 0.7483765437147297,
      "grad_norm": 9.446633338928223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21413029730319977,
      "logits/rejected": 0.1974586844444275,
      "logps/chosen": -450.7734069824219,
      "logps/rejected": -415.8749694824219,
      "loss": 0.6048,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5026199817657471,
      "rewards/margins": 0.8938779234886169,
      "rewards/rejected": -1.3964979648590088,
      "step": 5719
    },
    {
      "epoch": 0.7485074016520814,
      "grad_norm": 9.587106704711914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16149969398975372,
      "logits/rejected": -0.3116067051887512,
      "logps/chosen": -555.7241821289062,
      "logps/rejected": -495.85400390625,
      "loss": 0.5707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2352641820907593,
      "rewards/margins": 0.6355482339859009,
      "rewards/rejected": -1.8708122968673706,
      "step": 5720
    },
    {
      "epoch": 0.7486382595894332,
      "grad_norm": 7.626356601715088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12306497246026993,
      "logits/rejected": -0.14666476845741272,
      "logps/chosen": -477.7380676269531,
      "logps/rejected": -531.0756225585938,
      "loss": 0.4086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05720914900302887,
      "rewards/margins": 1.621807336807251,
      "rewards/rejected": -1.5645980834960938,
      "step": 5721
    },
    {
      "epoch": 0.748769117526785,
      "grad_norm": 7.614408493041992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18841174244880676,
      "logits/rejected": 0.027596861124038696,
      "logps/chosen": -602.4938354492188,
      "logps/rejected": -534.9574584960938,
      "loss": 0.3328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18489879369735718,
      "rewards/margins": 1.844525933265686,
      "rewards/rejected": -1.6596269607543945,
      "step": 5722
    },
    {
      "epoch": 0.7488999754641368,
      "grad_norm": 4.789916515350342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09968884289264679,
      "logits/rejected": -0.10119708627462387,
      "logps/chosen": -743.3568115234375,
      "logps/rejected": -610.4459228515625,
      "loss": 0.3114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.892649233341217,
      "rewards/margins": 1.925191879272461,
      "rewards/rejected": -1.0325425863265991,
      "step": 5723
    },
    {
      "epoch": 0.7490308334014885,
      "grad_norm": 8.637999534606934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07319539785385132,
      "logits/rejected": 0.12049989402294159,
      "logps/chosen": -431.86236572265625,
      "logps/rejected": -500.8971862792969,
      "loss": 0.6047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8532134294509888,
      "rewards/margins": 0.5822916030883789,
      "rewards/rejected": -1.4355049133300781,
      "step": 5724
    },
    {
      "epoch": 0.7491616913388403,
      "grad_norm": 4.738901615142822,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002842456102371216,
      "logits/rejected": -0.044953130185604095,
      "logps/chosen": -433.6259460449219,
      "logps/rejected": -430.7606506347656,
      "loss": 0.3355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3467714488506317,
      "rewards/margins": 1.5277470350265503,
      "rewards/rejected": -1.1809755563735962,
      "step": 5725
    },
    {
      "epoch": 0.749292549276192,
      "grad_norm": 7.810240745544434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30405259132385254,
      "logits/rejected": -0.33858394622802734,
      "logps/chosen": -359.056396484375,
      "logps/rejected": -358.76031494140625,
      "loss": 0.5458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0527973175048828,
      "rewards/margins": 0.5324712991714478,
      "rewards/rejected": -1.585268497467041,
      "step": 5726
    },
    {
      "epoch": 0.7494234072135438,
      "grad_norm": 6.391119003295898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30102092027664185,
      "logits/rejected": -0.35612753033638,
      "logps/chosen": -347.9035339355469,
      "logps/rejected": -354.37701416015625,
      "loss": 0.341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32114648818969727,
      "rewards/margins": 1.6242538690567017,
      "rewards/rejected": -1.945400357246399,
      "step": 5727
    },
    {
      "epoch": 0.7495542651508955,
      "grad_norm": 7.0640177726745605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22969642281532288,
      "logits/rejected": 0.27554038166999817,
      "logps/chosen": -473.9547119140625,
      "logps/rejected": -412.78033447265625,
      "loss": 0.4143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34198760986328125,
      "rewards/margins": 1.361172080039978,
      "rewards/rejected": -1.7031598091125488,
      "step": 5728
    },
    {
      "epoch": 0.7496851230882473,
      "grad_norm": 7.780094623565674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.091189906001091,
      "logits/rejected": 0.05161139369010925,
      "logps/chosen": -361.9504089355469,
      "logps/rejected": -506.52020263671875,
      "loss": 0.5368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7135547995567322,
      "rewards/margins": 1.4345744848251343,
      "rewards/rejected": -0.7210197448730469,
      "step": 5729
    },
    {
      "epoch": 0.7498159810255991,
      "grad_norm": 9.459336280822754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04955125227570534,
      "logits/rejected": -0.07796822488307953,
      "logps/chosen": -523.0765991210938,
      "logps/rejected": -500.45892333984375,
      "loss": 0.7531,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.818449079990387,
      "rewards/margins": -0.015936680138111115,
      "rewards/rejected": -0.8025123476982117,
      "step": 5730
    },
    {
      "epoch": 0.7499468389629509,
      "grad_norm": 7.6052985191345215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.233225017786026,
      "logits/rejected": -0.07848617434501648,
      "logps/chosen": -358.05194091796875,
      "logps/rejected": -513.2875366210938,
      "loss": 0.5836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49913865327835083,
      "rewards/margins": 1.200656771659851,
      "rewards/rejected": -1.6997953653335571,
      "step": 5731
    },
    {
      "epoch": 0.7500776969003026,
      "grad_norm": 8.46575927734375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15447451174259186,
      "logits/rejected": 0.11753693968057632,
      "logps/chosen": -478.7384338378906,
      "logps/rejected": -390.02923583984375,
      "loss": 0.7578,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6571239233016968,
      "rewards/margins": 0.6569029092788696,
      "rewards/rejected": -1.3140268325805664,
      "step": 5732
    },
    {
      "epoch": 0.7502085548376544,
      "grad_norm": 7.309221267700195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08221429586410522,
      "logits/rejected": -0.21854424476623535,
      "logps/chosen": -509.0878601074219,
      "logps/rejected": -505.1403503417969,
      "loss": 0.4365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13516616821289062,
      "rewards/margins": 1.6227967739105225,
      "rewards/rejected": -1.757962942123413,
      "step": 5733
    },
    {
      "epoch": 0.7503394127750062,
      "grad_norm": 9.072992324829102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2310546636581421,
      "logits/rejected": -0.29734793305397034,
      "logps/chosen": -354.37493896484375,
      "logps/rejected": -426.38836669921875,
      "loss": 0.5027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26209181547164917,
      "rewards/margins": 0.9745625853538513,
      "rewards/rejected": -1.2366544008255005,
      "step": 5734
    },
    {
      "epoch": 0.750470270712358,
      "grad_norm": 8.220182418823242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24447238445281982,
      "logits/rejected": -0.2817656397819519,
      "logps/chosen": -399.48260498046875,
      "logps/rejected": -462.1166076660156,
      "loss": 0.5067,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8120958805084229,
      "rewards/margins": 0.9144356846809387,
      "rewards/rejected": -1.7265316247940063,
      "step": 5735
    },
    {
      "epoch": 0.7506011286497096,
      "grad_norm": 4.4207024574279785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18401740491390228,
      "logits/rejected": 0.19994063675403595,
      "logps/chosen": -381.8777160644531,
      "logps/rejected": -364.54473876953125,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06543979793787003,
      "rewards/margins": 1.192809820175171,
      "rewards/rejected": -1.1273701190948486,
      "step": 5736
    },
    {
      "epoch": 0.7507319865870614,
      "grad_norm": 10.926183700561523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20872029662132263,
      "logits/rejected": -0.12369169294834137,
      "logps/chosen": -433.1600036621094,
      "logps/rejected": -554.5164184570312,
      "loss": 0.8473,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6990805864334106,
      "rewards/margins": 0.1347658336162567,
      "rewards/rejected": -0.8338463306427002,
      "step": 5737
    },
    {
      "epoch": 0.7508628445244132,
      "grad_norm": 6.749474048614502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04585712030529976,
      "logits/rejected": -0.5178959369659424,
      "logps/chosen": -430.8770446777344,
      "logps/rejected": -765.1281127929688,
      "loss": 0.435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.014095708727836609,
      "rewards/margins": 1.9329452514648438,
      "rewards/rejected": -1.9470409154891968,
      "step": 5738
    },
    {
      "epoch": 0.750993702461765,
      "grad_norm": 6.664591312408447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02861124277114868,
      "logits/rejected": -0.07233655452728271,
      "logps/chosen": -290.77593994140625,
      "logps/rejected": -306.040283203125,
      "loss": 0.3828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21717262268066406,
      "rewards/margins": 0.9260570406913757,
      "rewards/rejected": -1.1432297229766846,
      "step": 5739
    },
    {
      "epoch": 0.7511245603991167,
      "grad_norm": 4.6588640213012695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02741173282265663,
      "logits/rejected": -0.26899465918540955,
      "logps/chosen": -455.2530212402344,
      "logps/rejected": -423.5550537109375,
      "loss": 0.2818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4124303460121155,
      "rewards/margins": 1.614345908164978,
      "rewards/rejected": -2.026776075363159,
      "step": 5740
    },
    {
      "epoch": 0.7512554183364685,
      "grad_norm": 5.323477745056152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052358537912368774,
      "logits/rejected": -0.16210973262786865,
      "logps/chosen": -318.8455505371094,
      "logps/rejected": -324.7200012207031,
      "loss": 0.3556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2936912775039673,
      "rewards/margins": 1.4632047414779663,
      "rewards/rejected": -1.7568960189819336,
      "step": 5741
    },
    {
      "epoch": 0.7513862762738203,
      "grad_norm": 8.888858795166016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03830154985189438,
      "logits/rejected": -0.13779491186141968,
      "logps/chosen": -686.5715942382812,
      "logps/rejected": -642.3681640625,
      "loss": 0.5617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.19500045478343964,
      "rewards/margins": 0.8247499465942383,
      "rewards/rejected": -0.6297495365142822,
      "step": 5742
    },
    {
      "epoch": 0.7515171342111719,
      "grad_norm": 12.733030319213867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02830762416124344,
      "logits/rejected": -0.1076498031616211,
      "logps/chosen": -455.99603271484375,
      "logps/rejected": -373.1719665527344,
      "loss": 0.9832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8817555904388428,
      "rewards/margins": 0.08677521347999573,
      "rewards/rejected": -1.9685307741165161,
      "step": 5743
    },
    {
      "epoch": 0.7516479921485237,
      "grad_norm": 9.142394065856934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21281740069389343,
      "logits/rejected": 0.009442221373319626,
      "logps/chosen": -556.7399291992188,
      "logps/rejected": -423.53314208984375,
      "loss": 0.4177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10714036226272583,
      "rewards/margins": 1.6080507040023804,
      "rewards/rejected": -1.5009102821350098,
      "step": 5744
    },
    {
      "epoch": 0.7517788500858755,
      "grad_norm": 5.301761627197266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03803762421011925,
      "logits/rejected": -0.1114407628774643,
      "logps/chosen": -305.8897399902344,
      "logps/rejected": -379.63592529296875,
      "loss": 0.3835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00424155592918396,
      "rewards/margins": 1.2128398418426514,
      "rewards/rejected": -1.2170815467834473,
      "step": 5745
    },
    {
      "epoch": 0.7519097080232273,
      "grad_norm": 4.672756195068359,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25533100962638855,
      "logits/rejected": 0.13738782703876495,
      "logps/chosen": -442.8601989746094,
      "logps/rejected": -404.7007141113281,
      "loss": 0.3348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29499757289886475,
      "rewards/margins": 1.2957746982574463,
      "rewards/rejected": -1.590772271156311,
      "step": 5746
    },
    {
      "epoch": 0.752040565960579,
      "grad_norm": 5.430498123168945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14335595071315765,
      "logits/rejected": 0.026950795203447342,
      "logps/chosen": -538.3550415039062,
      "logps/rejected": -380.9683532714844,
      "loss": 0.332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28172725439071655,
      "rewards/margins": 1.0623903274536133,
      "rewards/rejected": -0.7806630730628967,
      "step": 5747
    },
    {
      "epoch": 0.7521714238979308,
      "grad_norm": 7.2811431884765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06230485066771507,
      "logits/rejected": -0.22064794600009918,
      "logps/chosen": -295.5414733886719,
      "logps/rejected": -720.1918334960938,
      "loss": 0.5626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2814769744873047,
      "rewards/margins": 1.406643271446228,
      "rewards/rejected": -1.6881202459335327,
      "step": 5748
    },
    {
      "epoch": 0.7523022818352826,
      "grad_norm": 7.355423450469971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026228057220578194,
      "logits/rejected": -0.03405061364173889,
      "logps/chosen": -442.0168762207031,
      "logps/rejected": -469.08642578125,
      "loss": 0.5169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34331169724464417,
      "rewards/margins": 1.0505154132843018,
      "rewards/rejected": -1.393827199935913,
      "step": 5749
    },
    {
      "epoch": 0.7524331397726344,
      "grad_norm": 6.419402122497559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15852859616279602,
      "logits/rejected": 0.027528151869773865,
      "logps/chosen": -530.660888671875,
      "logps/rejected": -430.7006530761719,
      "loss": 0.5047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.46654394268989563,
      "rewards/margins": 0.8567923903465271,
      "rewards/rejected": -0.39024829864501953,
      "step": 5750
    },
    {
      "epoch": 0.752563997709986,
      "grad_norm": 4.41649055480957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19921770691871643,
      "logits/rejected": -0.17040270566940308,
      "logps/chosen": -550.0924072265625,
      "logps/rejected": -418.6372985839844,
      "loss": 0.2723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6153163313865662,
      "rewards/margins": 1.7193210124969482,
      "rewards/rejected": -2.33463716506958,
      "step": 5751
    },
    {
      "epoch": 0.7526948556473378,
      "grad_norm": 6.372461795806885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26914751529693604,
      "logits/rejected": 0.1726865917444229,
      "logps/chosen": -372.53070068359375,
      "logps/rejected": -501.075927734375,
      "loss": 0.3476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08937856554985046,
      "rewards/margins": 1.4227309226989746,
      "rewards/rejected": -1.3333524465560913,
      "step": 5752
    },
    {
      "epoch": 0.7528257135846896,
      "grad_norm": 6.95881462097168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023673448711633682,
      "logits/rejected": 0.007986065000295639,
      "logps/chosen": -379.27618408203125,
      "logps/rejected": -365.5165710449219,
      "loss": 0.4459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.28270936012268066,
      "rewards/margins": 1.0133075714111328,
      "rewards/rejected": -0.7305981516838074,
      "step": 5753
    },
    {
      "epoch": 0.7529565715220414,
      "grad_norm": 8.796534538269043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0018278956413269043,
      "logits/rejected": 0.17800959944725037,
      "logps/chosen": -507.82586669921875,
      "logps/rejected": -574.4349975585938,
      "loss": 0.5882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05556394159793854,
      "rewards/margins": 0.8024492263793945,
      "rewards/rejected": -0.8580131530761719,
      "step": 5754
    },
    {
      "epoch": 0.7530874294593931,
      "grad_norm": 5.906169414520264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07291696965694427,
      "logits/rejected": 0.00609607994556427,
      "logps/chosen": -429.9720153808594,
      "logps/rejected": -377.12091064453125,
      "loss": 0.3546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6947698593139648,
      "rewards/margins": 1.565920114517212,
      "rewards/rejected": -0.8711501955986023,
      "step": 5755
    },
    {
      "epoch": 0.7532182873967449,
      "grad_norm": 8.971708297729492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13482508063316345,
      "logits/rejected": -0.11508383601903915,
      "logps/chosen": -411.4412841796875,
      "logps/rejected": -365.88494873046875,
      "loss": 0.6099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14161092042922974,
      "rewards/margins": 1.1637407541275024,
      "rewards/rejected": -1.0221298933029175,
      "step": 5756
    },
    {
      "epoch": 0.7533491453340967,
      "grad_norm": 9.238468170166016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18932360410690308,
      "logits/rejected": -0.10864584147930145,
      "logps/chosen": -477.2702331542969,
      "logps/rejected": -390.9527587890625,
      "loss": 0.7321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.49536576867103577,
      "rewards/margins": 0.3976837396621704,
      "rewards/rejected": 0.09768198430538177,
      "step": 5757
    },
    {
      "epoch": 0.7534800032714485,
      "grad_norm": 8.888158798217773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03290775790810585,
      "logits/rejected": 0.07048846781253815,
      "logps/chosen": -358.4474182128906,
      "logps/rejected": -304.496826171875,
      "loss": 0.837,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5613843202590942,
      "rewards/margins": -0.06642609089612961,
      "rewards/rejected": -0.4949582815170288,
      "step": 5758
    },
    {
      "epoch": 0.7536108612088002,
      "grad_norm": 6.868612766265869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028423644602298737,
      "logits/rejected": 0.04795721918344498,
      "logps/chosen": -527.0824584960938,
      "logps/rejected": -661.4710083007812,
      "loss": 0.3539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.35000938177108765,
      "rewards/margins": 1.9881072044372559,
      "rewards/rejected": -1.6380977630615234,
      "step": 5759
    },
    {
      "epoch": 0.753741719146152,
      "grad_norm": 8.604323387145996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05256364494562149,
      "logits/rejected": 0.08801741898059845,
      "logps/chosen": -364.162841796875,
      "logps/rejected": -341.6604919433594,
      "loss": 0.7603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44052445888519287,
      "rewards/margins": 0.31915879249572754,
      "rewards/rejected": -0.7596832513809204,
      "step": 5760
    },
    {
      "epoch": 0.7538725770835037,
      "grad_norm": 9.622336387634277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2771473824977875,
      "logits/rejected": -0.231362983584404,
      "logps/chosen": -457.4667663574219,
      "logps/rejected": -364.1373596191406,
      "loss": 0.7661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4067239761352539,
      "rewards/margins": 0.15889930725097656,
      "rewards/rejected": -0.5656233429908752,
      "step": 5761
    },
    {
      "epoch": 0.7540034350208555,
      "grad_norm": 5.27814245223999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28415948152542114,
      "logits/rejected": 0.1751881092786789,
      "logps/chosen": -413.7958984375,
      "logps/rejected": -418.1251525878906,
      "loss": 0.2303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09853154420852661,
      "rewards/margins": 2.135054588317871,
      "rewards/rejected": -2.233585834503174,
      "step": 5762
    },
    {
      "epoch": 0.7541342929582072,
      "grad_norm": 7.789826393127441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05074276775121689,
      "logits/rejected": -0.06059608608484268,
      "logps/chosen": -436.9889831542969,
      "logps/rejected": -540.8744506835938,
      "loss": 0.4234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3455867767333984,
      "rewards/margins": 0.8472048044204712,
      "rewards/rejected": -2.19279146194458,
      "step": 5763
    },
    {
      "epoch": 0.754265150895559,
      "grad_norm": 4.504568099975586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2906217575073242,
      "logits/rejected": -0.35822904109954834,
      "logps/chosen": -418.5528564453125,
      "logps/rejected": -402.8180236816406,
      "loss": 0.2622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006259717047214508,
      "rewards/margins": 1.3533447980880737,
      "rewards/rejected": -1.3596044778823853,
      "step": 5764
    },
    {
      "epoch": 0.7543960088329108,
      "grad_norm": 6.627862453460693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1961800754070282,
      "logits/rejected": -0.019447781145572662,
      "logps/chosen": -553.7356567382812,
      "logps/rejected": -597.3919677734375,
      "loss": 0.4258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.041136547923088074,
      "rewards/margins": 1.111863613128662,
      "rewards/rejected": -1.0707271099090576,
      "step": 5765
    },
    {
      "epoch": 0.7545268667702625,
      "grad_norm": 7.833509922027588,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06794356554746628,
      "logits/rejected": -0.07012754678726196,
      "logps/chosen": -580.780029296875,
      "logps/rejected": -427.3174133300781,
      "loss": 0.3552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1803060621023178,
      "rewards/margins": 1.4374083280563354,
      "rewards/rejected": -1.257102370262146,
      "step": 5766
    },
    {
      "epoch": 0.7546577247076143,
      "grad_norm": 6.032729625701904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02564997598528862,
      "logits/rejected": -0.2059851586818695,
      "logps/chosen": -499.3765563964844,
      "logps/rejected": -313.5647888183594,
      "loss": 0.4568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2557292878627777,
      "rewards/margins": 0.6599748730659485,
      "rewards/rejected": -0.9157041311264038,
      "step": 5767
    },
    {
      "epoch": 0.7547885826449661,
      "grad_norm": 4.1963372230529785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2283320277929306,
      "logits/rejected": -0.39900362491607666,
      "logps/chosen": -483.28582763671875,
      "logps/rejected": -629.7549438476562,
      "loss": 0.312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5230392217636108,
      "rewards/margins": 1.3990782499313354,
      "rewards/rejected": -1.9221174716949463,
      "step": 5768
    },
    {
      "epoch": 0.7549194405823179,
      "grad_norm": 11.016230583190918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22901588678359985,
      "logits/rejected": -0.10248216241598129,
      "logps/chosen": -551.9657592773438,
      "logps/rejected": -612.571044921875,
      "loss": 0.7967,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33845198154449463,
      "rewards/margins": 0.9669480323791504,
      "rewards/rejected": -1.3054001331329346,
      "step": 5769
    },
    {
      "epoch": 0.7550502985196695,
      "grad_norm": 5.479738712310791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024788597598671913,
      "logits/rejected": -0.052232809364795685,
      "logps/chosen": -232.16847229003906,
      "logps/rejected": -399.478759765625,
      "loss": 0.3908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35584110021591187,
      "rewards/margins": 1.0744848251342773,
      "rewards/rejected": -1.430325984954834,
      "step": 5770
    },
    {
      "epoch": 0.7551811564570213,
      "grad_norm": 5.930016994476318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003825860098004341,
      "logits/rejected": -0.10405226796865463,
      "logps/chosen": -421.067626953125,
      "logps/rejected": -495.8250732421875,
      "loss": 0.4035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1766912341117859,
      "rewards/margins": 0.9875112175941467,
      "rewards/rejected": -0.8108200430870056,
      "step": 5771
    },
    {
      "epoch": 0.7553120143943731,
      "grad_norm": 6.202478408813477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01880236715078354,
      "logits/rejected": -0.2859054207801819,
      "logps/chosen": -421.3329772949219,
      "logps/rejected": -442.8917236328125,
      "loss": 0.4996,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5825872421264648,
      "rewards/margins": 1.3711590766906738,
      "rewards/rejected": -1.9537461996078491,
      "step": 5772
    },
    {
      "epoch": 0.7554428723317249,
      "grad_norm": 4.0914998054504395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.039833374321460724,
      "logits/rejected": 0.029110513627529144,
      "logps/chosen": -429.46728515625,
      "logps/rejected": -429.92401123046875,
      "loss": 0.3358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25011056661605835,
      "rewards/margins": 1.081258773803711,
      "rewards/rejected": -0.8311483860015869,
      "step": 5773
    },
    {
      "epoch": 0.7555737302690766,
      "grad_norm": 6.365686893463135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1983061581850052,
      "logits/rejected": -0.03623780608177185,
      "logps/chosen": -475.5223693847656,
      "logps/rejected": -462.93524169921875,
      "loss": 0.3533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20885680615901947,
      "rewards/margins": 1.1369097232818604,
      "rewards/rejected": -1.345766544342041,
      "step": 5774
    },
    {
      "epoch": 0.7557045882064284,
      "grad_norm": 9.65778923034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07265426218509674,
      "logits/rejected": 0.2887299060821533,
      "logps/chosen": -426.6917724609375,
      "logps/rejected": -411.5333251953125,
      "loss": 0.7782,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5608020424842834,
      "rewards/margins": -0.018927570432424545,
      "rewards/rejected": -0.5418745279312134,
      "step": 5775
    },
    {
      "epoch": 0.7558354461437802,
      "grad_norm": 4.03483247756958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008095186203718185,
      "logits/rejected": -0.18607878684997559,
      "logps/chosen": -439.73724365234375,
      "logps/rejected": -342.865234375,
      "loss": 0.2271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3576725125312805,
      "rewards/margins": 1.7132304906845093,
      "rewards/rejected": -1.3555580377578735,
      "step": 5776
    },
    {
      "epoch": 0.755966304081132,
      "grad_norm": 3.0838441848754883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.140358567237854,
      "logits/rejected": -0.001458222046494484,
      "logps/chosen": -395.3744812011719,
      "logps/rejected": -399.2427062988281,
      "loss": 0.1947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4089410901069641,
      "rewards/margins": 1.866348385810852,
      "rewards/rejected": -1.4574072360992432,
      "step": 5777
    },
    {
      "epoch": 0.7560971620184836,
      "grad_norm": 8.591164588928223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29504916071891785,
      "logits/rejected": 0.19261808693408966,
      "logps/chosen": -367.6900634765625,
      "logps/rejected": -399.5040588378906,
      "loss": 0.634,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03950998932123184,
      "rewards/margins": 0.6346824765205383,
      "rewards/rejected": -0.5951724648475647,
      "step": 5778
    },
    {
      "epoch": 0.7562280199558354,
      "grad_norm": 6.823890209197998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.042741354554891586,
      "logits/rejected": -0.13656122982501984,
      "logps/chosen": -366.7463073730469,
      "logps/rejected": -394.258056640625,
      "loss": 0.4295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4890771806240082,
      "rewards/margins": 1.069594383239746,
      "rewards/rejected": -1.5586715936660767,
      "step": 5779
    },
    {
      "epoch": 0.7563588778931872,
      "grad_norm": 8.4092435836792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08321774005889893,
      "logits/rejected": -0.05243796110153198,
      "logps/chosen": -382.34259033203125,
      "logps/rejected": -395.315673828125,
      "loss": 0.6968,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8427032828330994,
      "rewards/margins": 0.2336111068725586,
      "rewards/rejected": -1.0763144493103027,
      "step": 5780
    },
    {
      "epoch": 0.756489735830539,
      "grad_norm": 5.812960147857666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20418938994407654,
      "logits/rejected": -0.10422383248806,
      "logps/chosen": -408.713134765625,
      "logps/rejected": -388.241455078125,
      "loss": 0.3657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2139507234096527,
      "rewards/margins": 1.3653062582015991,
      "rewards/rejected": -1.5792571306228638,
      "step": 5781
    },
    {
      "epoch": 0.7566205937678907,
      "grad_norm": 6.554728031158447,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09937085211277008,
      "logits/rejected": -0.09052260220050812,
      "logps/chosen": -339.11224365234375,
      "logps/rejected": -517.7963256835938,
      "loss": 0.5054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6159460544586182,
      "rewards/margins": 0.6314038634300232,
      "rewards/rejected": -1.2473498582839966,
      "step": 5782
    },
    {
      "epoch": 0.7567514517052425,
      "grad_norm": 6.6965436935424805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09827257692813873,
      "logits/rejected": 0.1538141667842865,
      "logps/chosen": -407.53277587890625,
      "logps/rejected": -407.0950927734375,
      "loss": 0.5677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6209417581558228,
      "rewards/margins": 1.002010464668274,
      "rewards/rejected": -0.38106879591941833,
      "step": 5783
    },
    {
      "epoch": 0.7568823096425943,
      "grad_norm": 6.121842861175537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17412270605564117,
      "logits/rejected": 0.03144078329205513,
      "logps/chosen": -406.36114501953125,
      "logps/rejected": -367.15643310546875,
      "loss": 0.4722,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21488399803638458,
      "rewards/margins": 1.248464822769165,
      "rewards/rejected": -1.0335807800292969,
      "step": 5784
    },
    {
      "epoch": 0.757013167579946,
      "grad_norm": 6.832921028137207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16487683355808258,
      "logits/rejected": 0.17991146445274353,
      "logps/chosen": -462.87652587890625,
      "logps/rejected": -530.3567504882812,
      "loss": 0.5278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42056313157081604,
      "rewards/margins": 0.8054149746894836,
      "rewards/rejected": -1.2259780168533325,
      "step": 5785
    },
    {
      "epoch": 0.7571440255172978,
      "grad_norm": 7.760268211364746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11911320686340332,
      "logits/rejected": 0.23408189415931702,
      "logps/chosen": -448.59979248046875,
      "logps/rejected": -481.06756591796875,
      "loss": 0.4777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3184797167778015,
      "rewards/margins": 1.6877467632293701,
      "rewards/rejected": -1.3692668676376343,
      "step": 5786
    },
    {
      "epoch": 0.7572748834546495,
      "grad_norm": 7.8764848709106445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10156052559614182,
      "logits/rejected": 0.0836234763264656,
      "logps/chosen": -456.2085266113281,
      "logps/rejected": -405.314208984375,
      "loss": 0.591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11274334788322449,
      "rewards/margins": 0.39630088210105896,
      "rewards/rejected": -0.5090442895889282,
      "step": 5787
    },
    {
      "epoch": 0.7574057413920013,
      "grad_norm": 5.639061450958252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1782604455947876,
      "logits/rejected": -0.15640534460544586,
      "logps/chosen": -538.46044921875,
      "logps/rejected": -442.5755310058594,
      "loss": 0.2877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21014900505542755,
      "rewards/margins": 1.5009276866912842,
      "rewards/rejected": -1.290778636932373,
      "step": 5788
    },
    {
      "epoch": 0.757536599329353,
      "grad_norm": 9.655634880065918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022708799690008163,
      "logits/rejected": -0.03385908901691437,
      "logps/chosen": -348.8451843261719,
      "logps/rejected": -378.95904541015625,
      "loss": 0.8696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5065591931343079,
      "rewards/margins": -0.18098606169223785,
      "rewards/rejected": -0.3255731463432312,
      "step": 5789
    },
    {
      "epoch": 0.7576674572667048,
      "grad_norm": 11.384932518005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17844760417938232,
      "logits/rejected": -0.18144568800926208,
      "logps/chosen": -487.65631103515625,
      "logps/rejected": -642.27001953125,
      "loss": 0.7529,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7871115803718567,
      "rewards/margins": 0.3120035231113434,
      "rewards/rejected": -1.099115014076233,
      "step": 5790
    },
    {
      "epoch": 0.7577983152040566,
      "grad_norm": 5.231003761291504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005349382758140564,
      "logits/rejected": -0.3464434742927551,
      "logps/chosen": -359.9356384277344,
      "logps/rejected": -442.3494567871094,
      "loss": 0.3691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18074572086334229,
      "rewards/margins": 1.1901249885559082,
      "rewards/rejected": -1.370870590209961,
      "step": 5791
    },
    {
      "epoch": 0.7579291731414084,
      "grad_norm": 7.093574047088623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008910153061151505,
      "logits/rejected": -0.06126474216580391,
      "logps/chosen": -384.6859436035156,
      "logps/rejected": -365.0141906738281,
      "loss": 0.5348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1742582619190216,
      "rewards/margins": 0.935121476650238,
      "rewards/rejected": -1.109379768371582,
      "step": 5792
    },
    {
      "epoch": 0.7580600310787601,
      "grad_norm": 10.675085067749023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18795952200889587,
      "logits/rejected": -0.06166205182671547,
      "logps/chosen": -498.9388427734375,
      "logps/rejected": -411.4832763671875,
      "loss": 0.8604,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8631080389022827,
      "rewards/margins": -0.25929126143455505,
      "rewards/rejected": -0.6038168668746948,
      "step": 5793
    },
    {
      "epoch": 0.7581908890161119,
      "grad_norm": 7.188299179077148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05555946007370949,
      "logits/rejected": -0.20086921751499176,
      "logps/chosen": -368.1121826171875,
      "logps/rejected": -483.52838134765625,
      "loss": 0.4998,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6088495254516602,
      "rewards/margins": 1.2709566354751587,
      "rewards/rejected": -1.8798060417175293,
      "step": 5794
    },
    {
      "epoch": 0.7583217469534637,
      "grad_norm": 8.986377716064453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1425171196460724,
      "logits/rejected": -0.276145339012146,
      "logps/chosen": -583.0294799804688,
      "logps/rejected": -426.7930908203125,
      "loss": 0.524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.056649789214134216,
      "rewards/margins": 0.866607666015625,
      "rewards/rejected": -0.9232574701309204,
      "step": 5795
    },
    {
      "epoch": 0.7584526048908155,
      "grad_norm": 9.770389556884766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060715947300195694,
      "logits/rejected": 0.18262523412704468,
      "logps/chosen": -405.3582458496094,
      "logps/rejected": -373.9089050292969,
      "loss": 0.7262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9385896921157837,
      "rewards/margins": 0.28816157579421997,
      "rewards/rejected": -1.2267512083053589,
      "step": 5796
    },
    {
      "epoch": 0.7585834628281671,
      "grad_norm": 5.147214412689209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03236936777830124,
      "logits/rejected": -0.17330612242221832,
      "logps/chosen": -293.2724304199219,
      "logps/rejected": -297.6016540527344,
      "loss": 0.3658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03693781793117523,
      "rewards/margins": 1.0042513608932495,
      "rewards/rejected": -1.041189193725586,
      "step": 5797
    },
    {
      "epoch": 0.7587143207655189,
      "grad_norm": 8.749753952026367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11682789772748947,
      "logits/rejected": -0.06580564379692078,
      "logps/chosen": -264.875732421875,
      "logps/rejected": -254.732421875,
      "loss": 0.7658,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26886841654777527,
      "rewards/margins": 0.24352285265922546,
      "rewards/rejected": -0.5123913288116455,
      "step": 5798
    },
    {
      "epoch": 0.7588451787028707,
      "grad_norm": 8.678179740905762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12281011790037155,
      "logits/rejected": -0.2271430790424347,
      "logps/chosen": -650.7686767578125,
      "logps/rejected": -686.3290405273438,
      "loss": 0.6324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7914234399795532,
      "rewards/margins": 0.6378364562988281,
      "rewards/rejected": -1.4292598962783813,
      "step": 5799
    },
    {
      "epoch": 0.7589760366402225,
      "grad_norm": 11.304586410522461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06908546388149261,
      "logits/rejected": -0.01526651531457901,
      "logps/chosen": -522.0131225585938,
      "logps/rejected": -457.5606384277344,
      "loss": 0.9644,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34584543108940125,
      "rewards/margins": 0.2507857382297516,
      "rewards/rejected": -0.5966312289237976,
      "step": 5800
    },
    {
      "epoch": 0.7591068945775742,
      "grad_norm": 4.913383960723877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23931531608104706,
      "logits/rejected": 0.11252132803201675,
      "logps/chosen": -656.780517578125,
      "logps/rejected": -534.523193359375,
      "loss": 0.3216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5391944050788879,
      "rewards/margins": 1.3156453371047974,
      "rewards/rejected": -1.85483980178833,
      "step": 5801
    },
    {
      "epoch": 0.759237752514926,
      "grad_norm": 11.014827728271484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26785796880722046,
      "logits/rejected": 0.08576006442308426,
      "logps/chosen": -448.5511474609375,
      "logps/rejected": -386.989501953125,
      "loss": 0.705,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4049754738807678,
      "rewards/margins": 0.6600183248519897,
      "rewards/rejected": -1.0649938583374023,
      "step": 5802
    },
    {
      "epoch": 0.7593686104522778,
      "grad_norm": 8.635973930358887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0822473093867302,
      "logits/rejected": -0.11956356465816498,
      "logps/chosen": -457.59649658203125,
      "logps/rejected": -498.13360595703125,
      "loss": 0.5294,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.40509912371635437,
      "rewards/margins": 0.5272769927978516,
      "rewards/rejected": -0.12217789888381958,
      "step": 5803
    },
    {
      "epoch": 0.7594994683896296,
      "grad_norm": 8.604178428649902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1736595183610916,
      "logits/rejected": -0.34431618452072144,
      "logps/chosen": -680.4591064453125,
      "logps/rejected": -635.9111938476562,
      "loss": 0.7837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6577789187431335,
      "rewards/margins": 0.778550386428833,
      "rewards/rejected": -1.4363293647766113,
      "step": 5804
    },
    {
      "epoch": 0.7596303263269812,
      "grad_norm": 13.067591667175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018249651417136192,
      "logits/rejected": -0.0843038409948349,
      "logps/chosen": -578.4223022460938,
      "logps/rejected": -617.108642578125,
      "loss": 1.1221,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.0551652908325195,
      "rewards/margins": -0.4291231036186218,
      "rewards/rejected": -0.6260421872138977,
      "step": 5805
    },
    {
      "epoch": 0.759761184264333,
      "grad_norm": 8.716755867004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17962080240249634,
      "logits/rejected": 0.045683059841394424,
      "logps/chosen": -352.0532531738281,
      "logps/rejected": -392.73297119140625,
      "loss": 0.6296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19814777374267578,
      "rewards/margins": 1.1941546201705933,
      "rewards/rejected": -0.9960068464279175,
      "step": 5806
    },
    {
      "epoch": 0.7598920422016848,
      "grad_norm": 7.165950775146484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.062222015112638474,
      "logits/rejected": 0.17875118553638458,
      "logps/chosen": -600.4188842773438,
      "logps/rejected": -480.45245361328125,
      "loss": 0.3675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5222227573394775,
      "rewards/margins": 1.052903175354004,
      "rewards/rejected": -0.5306804180145264,
      "step": 5807
    },
    {
      "epoch": 0.7600229001390365,
      "grad_norm": 7.69137716293335,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18760913610458374,
      "logits/rejected": 0.20607119798660278,
      "logps/chosen": -455.6359558105469,
      "logps/rejected": -433.059814453125,
      "loss": 0.5495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11751596629619598,
      "rewards/margins": 0.40720099210739136,
      "rewards/rejected": -0.5247169733047485,
      "step": 5808
    },
    {
      "epoch": 0.7601537580763883,
      "grad_norm": 9.201894760131836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0003200611099600792,
      "logits/rejected": -0.12830331921577454,
      "logps/chosen": -578.2623291015625,
      "logps/rejected": -393.7813415527344,
      "loss": 0.4044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3437652587890625,
      "rewards/margins": 1.1230285167694092,
      "rewards/rejected": -1.4667938947677612,
      "step": 5809
    },
    {
      "epoch": 0.7602846160137401,
      "grad_norm": 10.367018699645996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04897987097501755,
      "logits/rejected": 0.12052474915981293,
      "logps/chosen": -383.6226806640625,
      "logps/rejected": -289.90386962890625,
      "loss": 0.7704,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6063747406005859,
      "rewards/margins": 0.30537423491477966,
      "rewards/rejected": -0.9117487668991089,
      "step": 5810
    },
    {
      "epoch": 0.7604154739510919,
      "grad_norm": 5.703116416931152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32439467310905457,
      "logits/rejected": 0.22039008140563965,
      "logps/chosen": -585.6516723632812,
      "logps/rejected": -498.2713928222656,
      "loss": 0.3903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011683855205774307,
      "rewards/margins": 1.0568115711212158,
      "rewards/rejected": -1.0451276302337646,
      "step": 5811
    },
    {
      "epoch": 0.7605463318884436,
      "grad_norm": 5.1157145500183105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11544911563396454,
      "logits/rejected": -0.026509780436754227,
      "logps/chosen": -672.912109375,
      "logps/rejected": -677.40234375,
      "loss": 0.2264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09258690476417542,
      "rewards/margins": 1.9081951379776,
      "rewards/rejected": -1.8156081438064575,
      "step": 5812
    },
    {
      "epoch": 0.7606771898257954,
      "grad_norm": 4.510157108306885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1471925526857376,
      "logits/rejected": 0.049899160861968994,
      "logps/chosen": -519.1613159179688,
      "logps/rejected": -400.1387634277344,
      "loss": 0.3051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06988336145877838,
      "rewards/margins": 1.7981617450714111,
      "rewards/rejected": -1.728278398513794,
      "step": 5813
    },
    {
      "epoch": 0.7608080477631471,
      "grad_norm": 7.743432521820068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26848843693733215,
      "logits/rejected": 0.2036534696817398,
      "logps/chosen": -540.55078125,
      "logps/rejected": -383.9385986328125,
      "loss": 0.4047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3269638419151306,
      "rewards/margins": 1.168613314628601,
      "rewards/rejected": -1.495577096939087,
      "step": 5814
    },
    {
      "epoch": 0.7609389057004989,
      "grad_norm": 9.1199369430542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06887209415435791,
      "logits/rejected": -0.07975058257579803,
      "logps/chosen": -370.46343994140625,
      "logps/rejected": -419.50518798828125,
      "loss": 0.7226,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5345615148544312,
      "rewards/margins": 0.3272806406021118,
      "rewards/rejected": -0.861842155456543,
      "step": 5815
    },
    {
      "epoch": 0.7610697636378506,
      "grad_norm": 4.304696083068848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.039797231554985046,
      "logits/rejected": 0.0410320907831192,
      "logps/chosen": -503.18017578125,
      "logps/rejected": -525.8019409179688,
      "loss": 0.2303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19617712497711182,
      "rewards/margins": 1.807809829711914,
      "rewards/rejected": -1.6116328239440918,
      "step": 5816
    },
    {
      "epoch": 0.7612006215752024,
      "grad_norm": 5.932527542114258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32662197947502136,
      "logits/rejected": -0.3412788510322571,
      "logps/chosen": -450.3731689453125,
      "logps/rejected": -465.47772216796875,
      "loss": 0.356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2018575668334961,
      "rewards/margins": 1.3361740112304688,
      "rewards/rejected": -1.1343164443969727,
      "step": 5817
    },
    {
      "epoch": 0.7613314795125542,
      "grad_norm": 6.8844075202941895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32196149230003357,
      "logits/rejected": 0.2463877946138382,
      "logps/chosen": -513.24267578125,
      "logps/rejected": -474.6703796386719,
      "loss": 0.6109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1342800259590149,
      "rewards/margins": 0.6658749580383301,
      "rewards/rejected": -0.8001549243927002,
      "step": 5818
    },
    {
      "epoch": 0.761462337449906,
      "grad_norm": 7.259115695953369,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022805191576480865,
      "logits/rejected": -0.23764817416667938,
      "logps/chosen": -297.7110900878906,
      "logps/rejected": -339.76513671875,
      "loss": 0.5457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4990594983100891,
      "rewards/margins": 1.0187158584594727,
      "rewards/rejected": -1.5177754163742065,
      "step": 5819
    },
    {
      "epoch": 0.7615931953872577,
      "grad_norm": 8.536449432373047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015691988170146942,
      "logits/rejected": 0.25518643856048584,
      "logps/chosen": -522.1387329101562,
      "logps/rejected": -526.9354858398438,
      "loss": 0.5151,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15005400776863098,
      "rewards/margins": 0.5722548961639404,
      "rewards/rejected": -0.722308874130249,
      "step": 5820
    },
    {
      "epoch": 0.7617240533246095,
      "grad_norm": 6.999927997589111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13975214958190918,
      "logits/rejected": -0.07457814365625381,
      "logps/chosen": -335.28924560546875,
      "logps/rejected": -477.96044921875,
      "loss": 0.3229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5509139895439148,
      "rewards/margins": 1.466749668121338,
      "rewards/rejected": -0.9158355593681335,
      "step": 5821
    },
    {
      "epoch": 0.7618549112619613,
      "grad_norm": 6.017236232757568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22203239798545837,
      "logits/rejected": 0.19617410004138947,
      "logps/chosen": -505.83551025390625,
      "logps/rejected": -508.30230712890625,
      "loss": 0.3661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28201621770858765,
      "rewards/margins": 1.6307121515274048,
      "rewards/rejected": -1.348695993423462,
      "step": 5822
    },
    {
      "epoch": 0.761985769199313,
      "grad_norm": 8.078493118286133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21073849499225616,
      "logits/rejected": 0.12619341909885406,
      "logps/chosen": -408.572021484375,
      "logps/rejected": -343.8166198730469,
      "loss": 0.5664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6709650158882141,
      "rewards/margins": 0.7961581349372864,
      "rewards/rejected": -1.4671231508255005,
      "step": 5823
    },
    {
      "epoch": 0.7621166271366647,
      "grad_norm": 8.59107494354248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14921529591083527,
      "logits/rejected": -0.0214059017598629,
      "logps/chosen": -573.4421997070312,
      "logps/rejected": -456.8511962890625,
      "loss": 0.5552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2934820055961609,
      "rewards/margins": 0.6597543954849243,
      "rewards/rejected": -0.3662723898887634,
      "step": 5824
    },
    {
      "epoch": 0.7622474850740165,
      "grad_norm": 11.863114356994629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05340207368135452,
      "logits/rejected": 0.16948658227920532,
      "logps/chosen": -519.3427734375,
      "logps/rejected": -352.14971923828125,
      "loss": 0.6554,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5456056594848633,
      "rewards/margins": 0.8341314196586609,
      "rewards/rejected": -1.379737138748169,
      "step": 5825
    },
    {
      "epoch": 0.7623783430113683,
      "grad_norm": 5.636501312255859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06088809669017792,
      "logits/rejected": -0.09358206391334534,
      "logps/chosen": -446.6733093261719,
      "logps/rejected": -349.8184814453125,
      "loss": 0.3563,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26825371384620667,
      "rewards/margins": 1.311959981918335,
      "rewards/rejected": -1.5802136659622192,
      "step": 5826
    },
    {
      "epoch": 0.7625092009487201,
      "grad_norm": 8.747674942016602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19531619548797607,
      "logits/rejected": 0.29694664478302,
      "logps/chosen": -460.2245788574219,
      "logps/rejected": -490.9332275390625,
      "loss": 0.6419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016913026571273804,
      "rewards/margins": 1.5211670398712158,
      "rewards/rejected": -1.5042542219161987,
      "step": 5827
    },
    {
      "epoch": 0.7626400588860718,
      "grad_norm": 9.36079216003418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19672292470932007,
      "logits/rejected": 0.16840332746505737,
      "logps/chosen": -268.1798095703125,
      "logps/rejected": -411.4004211425781,
      "loss": 0.9786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024653911590576172,
      "rewards/margins": 0.556804358959198,
      "rewards/rejected": -0.5321504473686218,
      "step": 5828
    },
    {
      "epoch": 0.7627709168234236,
      "grad_norm": 10.101543426513672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03485669568181038,
      "logits/rejected": -0.20872345566749573,
      "logps/chosen": -364.86517333984375,
      "logps/rejected": -395.2937927246094,
      "loss": 0.6696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11140385270118713,
      "rewards/margins": 0.4382186532020569,
      "rewards/rejected": -0.5496225357055664,
      "step": 5829
    },
    {
      "epoch": 0.7629017747607754,
      "grad_norm": 7.280142307281494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18160894513130188,
      "logits/rejected": 0.22054332494735718,
      "logps/chosen": -456.61419677734375,
      "logps/rejected": -423.7194519042969,
      "loss": 0.5261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.029262900352478,
      "rewards/margins": 0.5892497301101685,
      "rewards/rejected": -1.618512749671936,
      "step": 5830
    },
    {
      "epoch": 0.763032632698127,
      "grad_norm": 6.669367790222168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06172335892915726,
      "logits/rejected": 0.05586286634206772,
      "logps/chosen": -510.4305725097656,
      "logps/rejected": -496.2605895996094,
      "loss": 0.4457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3836781978607178,
      "rewards/margins": 0.8642040491104126,
      "rewards/rejected": -0.48052579164505005,
      "step": 5831
    },
    {
      "epoch": 0.7631634906354788,
      "grad_norm": 11.055562973022461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04878369718790054,
      "logits/rejected": -0.11416538804769516,
      "logps/chosen": -559.2679443359375,
      "logps/rejected": -598.091552734375,
      "loss": 0.7704,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8244632482528687,
      "rewards/margins": -0.0391082838177681,
      "rewards/rejected": -0.7853549718856812,
      "step": 5832
    },
    {
      "epoch": 0.7632943485728306,
      "grad_norm": 5.641524314880371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2088308334350586,
      "logits/rejected": 0.028212692588567734,
      "logps/chosen": -481.2806701660156,
      "logps/rejected": -463.2628173828125,
      "loss": 0.2729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.377593994140625,
      "rewards/margins": 1.7211607694625854,
      "rewards/rejected": -1.343566656112671,
      "step": 5833
    },
    {
      "epoch": 0.7634252065101824,
      "grad_norm": 7.84528923034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07257644832134247,
      "logits/rejected": 0.06067565828561783,
      "logps/chosen": -684.2932739257812,
      "logps/rejected": -600.0335693359375,
      "loss": 0.5059,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.664550244808197,
      "rewards/margins": 0.7682799100875854,
      "rewards/rejected": -0.10372960567474365,
      "step": 5834
    },
    {
      "epoch": 0.7635560644475341,
      "grad_norm": 6.042508602142334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22879847884178162,
      "logits/rejected": 0.15472225844860077,
      "logps/chosen": -426.9366149902344,
      "logps/rejected": -421.0009460449219,
      "loss": 0.3668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6267938613891602,
      "rewards/margins": 1.3659672737121582,
      "rewards/rejected": -0.7391735315322876,
      "step": 5835
    },
    {
      "epoch": 0.7636869223848859,
      "grad_norm": 8.538860321044922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11268570274114609,
      "logits/rejected": 0.11265827715396881,
      "logps/chosen": -494.091064453125,
      "logps/rejected": -449.4698181152344,
      "loss": 0.6179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4277084469795227,
      "rewards/margins": 0.5661563873291016,
      "rewards/rejected": -0.993864893913269,
      "step": 5836
    },
    {
      "epoch": 0.7638177803222377,
      "grad_norm": 5.072136878967285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1621912717819214,
      "logits/rejected": 0.15238580107688904,
      "logps/chosen": -591.978759765625,
      "logps/rejected": -512.2032470703125,
      "loss": 0.2469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5855892300605774,
      "rewards/margins": 1.7909976243972778,
      "rewards/rejected": -1.2054085731506348,
      "step": 5837
    },
    {
      "epoch": 0.7639486382595895,
      "grad_norm": 15.173566818237305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0498899482190609,
      "logits/rejected": -0.18193477392196655,
      "logps/chosen": -433.7991943359375,
      "logps/rejected": -499.8507385253906,
      "loss": 0.6425,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04794217646121979,
      "rewards/margins": 0.6397262215614319,
      "rewards/rejected": -0.6876684427261353,
      "step": 5838
    },
    {
      "epoch": 0.7640794961969412,
      "grad_norm": 7.47676944732666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.057810574769973755,
      "logits/rejected": 0.1440679132938385,
      "logps/chosen": -500.85601806640625,
      "logps/rejected": -456.04473876953125,
      "loss": 0.5881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5566511154174805,
      "rewards/margins": 0.6533640623092651,
      "rewards/rejected": -0.0967128798365593,
      "step": 5839
    },
    {
      "epoch": 0.764210354134293,
      "grad_norm": 10.595293998718262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3120500147342682,
      "logits/rejected": -0.02205769531428814,
      "logps/chosen": -740.9241943359375,
      "logps/rejected": -643.9818115234375,
      "loss": 0.7078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.48409557342529297,
      "rewards/margins": 0.588604211807251,
      "rewards/rejected": -0.10450860857963562,
      "step": 5840
    },
    {
      "epoch": 0.7643412120716447,
      "grad_norm": 8.54960823059082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09647145122289658,
      "logits/rejected": -0.018940351903438568,
      "logps/chosen": -450.13482666015625,
      "logps/rejected": -467.52484130859375,
      "loss": 0.5067,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18278178572654724,
      "rewards/margins": 0.9889205694198608,
      "rewards/rejected": -0.8061386942863464,
      "step": 5841
    },
    {
      "epoch": 0.7644720700089965,
      "grad_norm": 6.682869911193848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12024736404418945,
      "logits/rejected": -0.044188492000103,
      "logps/chosen": -370.7948913574219,
      "logps/rejected": -716.853271484375,
      "loss": 0.3376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3729580044746399,
      "rewards/margins": 1.8076684474945068,
      "rewards/rejected": -2.180626392364502,
      "step": 5842
    },
    {
      "epoch": 0.7646029279463482,
      "grad_norm": 6.827489376068115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30915820598602295,
      "logits/rejected": 0.017408065497875214,
      "logps/chosen": -422.0014953613281,
      "logps/rejected": -332.4253234863281,
      "loss": 0.4598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2614690661430359,
      "rewards/margins": 0.7906891107559204,
      "rewards/rejected": -1.052158236503601,
      "step": 5843
    },
    {
      "epoch": 0.7647337858837,
      "grad_norm": 10.069519996643066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1838473677635193,
      "logits/rejected": 0.14420661330223083,
      "logps/chosen": -506.546142578125,
      "logps/rejected": -386.2685546875,
      "loss": 1.0021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04825270175933838,
      "rewards/margins": 0.664592444896698,
      "rewards/rejected": -0.6163396835327148,
      "step": 5844
    },
    {
      "epoch": 0.7648646438210518,
      "grad_norm": 8.880674362182617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01841963827610016,
      "logits/rejected": 0.12304522842168808,
      "logps/chosen": -313.7505798339844,
      "logps/rejected": -423.8049011230469,
      "loss": 0.7632,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.42709892988204956,
      "rewards/margins": 0.4261070191860199,
      "rewards/rejected": -0.8532057404518127,
      "step": 5845
    },
    {
      "epoch": 0.7649955017584036,
      "grad_norm": 5.108994483947754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10844999551773071,
      "logits/rejected": -0.045821771025657654,
      "logps/chosen": -463.1371154785156,
      "logps/rejected": -422.77716064453125,
      "loss": 0.3582,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3016716241836548,
      "rewards/margins": 1.3188552856445312,
      "rewards/rejected": -1.620526909828186,
      "step": 5846
    },
    {
      "epoch": 0.7651263596957553,
      "grad_norm": 6.535488128662109,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22358477115631104,
      "logits/rejected": 0.18197289109230042,
      "logps/chosen": -690.270751953125,
      "logps/rejected": -513.2464599609375,
      "loss": 0.4253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6318604946136475,
      "rewards/margins": 1.3962153196334839,
      "rewards/rejected": -2.028075695037842,
      "step": 5847
    },
    {
      "epoch": 0.7652572176331071,
      "grad_norm": 8.885212898254395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12596344947814941,
      "logits/rejected": -0.22355982661247253,
      "logps/chosen": -351.52008056640625,
      "logps/rejected": -506.9285888671875,
      "loss": 0.7804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029422566294670105,
      "rewards/margins": 0.3200432062149048,
      "rewards/rejected": -0.2906206250190735,
      "step": 5848
    },
    {
      "epoch": 0.7653880755704588,
      "grad_norm": 6.145871639251709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013796709477901459,
      "logits/rejected": -0.02228519320487976,
      "logps/chosen": -513.1260375976562,
      "logps/rejected": -732.769287109375,
      "loss": 0.4528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33368775248527527,
      "rewards/margins": 1.3341728448867798,
      "rewards/rejected": -1.0004850625991821,
      "step": 5849
    },
    {
      "epoch": 0.7655189335078105,
      "grad_norm": 3.721257448196411,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031195806339383125,
      "logits/rejected": -0.07981662452220917,
      "logps/chosen": -376.74578857421875,
      "logps/rejected": -445.9112854003906,
      "loss": 0.2218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2729802131652832,
      "rewards/margins": 2.100254535675049,
      "rewards/rejected": -1.8272743225097656,
      "step": 5850
    },
    {
      "epoch": 0.7656497914451623,
      "grad_norm": 4.382075309753418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11623695492744446,
      "logits/rejected": 0.2523724138736725,
      "logps/chosen": -432.56988525390625,
      "logps/rejected": -435.2197570800781,
      "loss": 0.3095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2979879379272461,
      "rewards/margins": 1.2947523593902588,
      "rewards/rejected": -0.9967644214630127,
      "step": 5851
    },
    {
      "epoch": 0.7657806493825141,
      "grad_norm": 6.838754177093506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.115486279129982,
      "logits/rejected": -0.2507590651512146,
      "logps/chosen": -434.931396484375,
      "logps/rejected": -486.87725830078125,
      "loss": 0.5629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36195889115333557,
      "rewards/margins": 0.4654911160469055,
      "rewards/rejected": -0.8274500370025635,
      "step": 5852
    },
    {
      "epoch": 0.7659115073198659,
      "grad_norm": 7.965945243835449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2871899902820587,
      "logits/rejected": -0.04452882707118988,
      "logps/chosen": -544.9195556640625,
      "logps/rejected": -408.75897216796875,
      "loss": 0.5719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30470559000968933,
      "rewards/margins": 0.7079020738601685,
      "rewards/rejected": -1.0126076936721802,
      "step": 5853
    },
    {
      "epoch": 0.7660423652572176,
      "grad_norm": 6.638122081756592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2868516743183136,
      "logits/rejected": 0.27459588646888733,
      "logps/chosen": -637.1317749023438,
      "logps/rejected": -444.10174560546875,
      "loss": 0.4241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.022789739072322845,
      "rewards/margins": 1.751295804977417,
      "rewards/rejected": -1.774085521697998,
      "step": 5854
    },
    {
      "epoch": 0.7661732231945694,
      "grad_norm": 8.29478645324707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10199035704135895,
      "logits/rejected": -0.056597501039505005,
      "logps/chosen": -610.1546020507812,
      "logps/rejected": -497.4086608886719,
      "loss": 0.5664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08549632132053375,
      "rewards/margins": 0.7337579727172852,
      "rewards/rejected": -0.6482616066932678,
      "step": 5855
    },
    {
      "epoch": 0.7663040811319212,
      "grad_norm": 7.493418216705322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010854480788111687,
      "logits/rejected": -0.06620422750711441,
      "logps/chosen": -433.5256042480469,
      "logps/rejected": -525.4999389648438,
      "loss": 0.5091,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27508237957954407,
      "rewards/margins": 0.8194659948348999,
      "rewards/rejected": -1.0945484638214111,
      "step": 5856
    },
    {
      "epoch": 0.766434939069273,
      "grad_norm": 7.728621482849121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1758810579776764,
      "logits/rejected": -0.02551378682255745,
      "logps/chosen": -443.02996826171875,
      "logps/rejected": -343.6323547363281,
      "loss": 0.5928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3184547424316406,
      "rewards/margins": 0.4611898958683014,
      "rewards/rejected": -0.7796446084976196,
      "step": 5857
    },
    {
      "epoch": 0.7665657970066246,
      "grad_norm": 3.658413887023926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3261528015136719,
      "logits/rejected": -0.20341850817203522,
      "logps/chosen": -332.9574890136719,
      "logps/rejected": -290.5022888183594,
      "loss": 0.3056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34251803159713745,
      "rewards/margins": 1.7838547229766846,
      "rewards/rejected": -2.126372814178467,
      "step": 5858
    },
    {
      "epoch": 0.7666966549439764,
      "grad_norm": 9.97310733795166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37492620944976807,
      "logits/rejected": 0.3041157126426697,
      "logps/chosen": -359.83489990234375,
      "logps/rejected": -354.9679870605469,
      "loss": 0.9423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32918816804885864,
      "rewards/margins": 0.6473141312599182,
      "rewards/rejected": -0.9765022397041321,
      "step": 5859
    },
    {
      "epoch": 0.7668275128813282,
      "grad_norm": 6.394036769866943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12779581546783447,
      "logits/rejected": -0.0387580469250679,
      "logps/chosen": -311.46185302734375,
      "logps/rejected": -397.0827331542969,
      "loss": 0.2661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5704395771026611,
      "rewards/margins": 2.063096523284912,
      "rewards/rejected": -1.49265718460083,
      "step": 5860
    },
    {
      "epoch": 0.76695837081868,
      "grad_norm": 10.969637870788574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2348650097846985,
      "logits/rejected": -0.007927056401968002,
      "logps/chosen": -390.8082275390625,
      "logps/rejected": -333.4591064453125,
      "loss": 0.6692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28697454929351807,
      "rewards/margins": 0.4747368097305298,
      "rewards/rejected": -0.7617113590240479,
      "step": 5861
    },
    {
      "epoch": 0.7670892287560317,
      "grad_norm": 11.372784614562988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2566773295402527,
      "logits/rejected": 0.18937338888645172,
      "logps/chosen": -781.4449462890625,
      "logps/rejected": -650.083251953125,
      "loss": 0.734,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9094123840332031,
      "rewards/margins": 0.3131659924983978,
      "rewards/rejected": -1.222578525543213,
      "step": 5862
    },
    {
      "epoch": 0.7672200866933835,
      "grad_norm": 8.568524360656738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11415594816207886,
      "logits/rejected": -0.1587861180305481,
      "logps/chosen": -547.37109375,
      "logps/rejected": -477.4442443847656,
      "loss": 0.4589,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23664318025112152,
      "rewards/margins": 1.3838443756103516,
      "rewards/rejected": -1.1472011804580688,
      "step": 5863
    },
    {
      "epoch": 0.7673509446307353,
      "grad_norm": 8.998852729797363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06673730164766312,
      "logits/rejected": 0.10464400798082352,
      "logps/chosen": -299.6939697265625,
      "logps/rejected": -419.0509948730469,
      "loss": 0.6087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5882729291915894,
      "rewards/margins": 0.36541205644607544,
      "rewards/rejected": -0.9536850452423096,
      "step": 5864
    },
    {
      "epoch": 0.7674818025680871,
      "grad_norm": 10.323089599609375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3009454011917114,
      "logits/rejected": -0.18784400820732117,
      "logps/chosen": -371.95867919921875,
      "logps/rejected": -310.98956298828125,
      "loss": 1.0495,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1872292757034302,
      "rewards/margins": -0.4097675383090973,
      "rewards/rejected": -0.7774618864059448,
      "step": 5865
    },
    {
      "epoch": 0.7676126605054387,
      "grad_norm": 6.031630516052246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.050347547978162766,
      "logits/rejected": -0.28798744082450867,
      "logps/chosen": -577.598876953125,
      "logps/rejected": -319.2730712890625,
      "loss": 0.4516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.102294683456421,
      "rewards/margins": 1.5063118934631348,
      "rewards/rejected": -0.40401726961135864,
      "step": 5866
    },
    {
      "epoch": 0.7677435184427905,
      "grad_norm": 6.592118740081787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10825179517269135,
      "logits/rejected": 0.06516590714454651,
      "logps/chosen": -461.206787109375,
      "logps/rejected": -566.7193603515625,
      "loss": 0.3824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4068252742290497,
      "rewards/margins": 1.3472371101379395,
      "rewards/rejected": -0.9404119849205017,
      "step": 5867
    },
    {
      "epoch": 0.7678743763801423,
      "grad_norm": 8.662359237670898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003981314599514008,
      "logits/rejected": -0.1609993577003479,
      "logps/chosen": -362.37982177734375,
      "logps/rejected": -697.779052734375,
      "loss": 0.8765,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03406846523284912,
      "rewards/margins": 0.35385382175445557,
      "rewards/rejected": -0.31978532671928406,
      "step": 5868
    },
    {
      "epoch": 0.7680052343174941,
      "grad_norm": 4.646768093109131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26318562030792236,
      "logits/rejected": 0.12440924346446991,
      "logps/chosen": -532.783203125,
      "logps/rejected": -520.540283203125,
      "loss": 0.2223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5555015206336975,
      "rewards/margins": 1.634334921836853,
      "rewards/rejected": -1.0788334608078003,
      "step": 5869
    },
    {
      "epoch": 0.7681360922548458,
      "grad_norm": 11.43661117553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1762857586145401,
      "logits/rejected": 0.1190778836607933,
      "logps/chosen": -487.69207763671875,
      "logps/rejected": -630.8206176757812,
      "loss": 0.8637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11600038409233093,
      "rewards/margins": 0.26515620946884155,
      "rewards/rejected": -0.14915582537651062,
      "step": 5870
    },
    {
      "epoch": 0.7682669501921976,
      "grad_norm": 8.929688453674316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12127936631441116,
      "logits/rejected": 0.16291086375713348,
      "logps/chosen": -337.73724365234375,
      "logps/rejected": -627.8602294921875,
      "loss": 0.49,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4439539313316345,
      "rewards/margins": 1.253800392150879,
      "rewards/rejected": -0.8098465800285339,
      "step": 5871
    },
    {
      "epoch": 0.7683978081295494,
      "grad_norm": 11.292638778686523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10704423487186432,
      "logits/rejected": -0.2306024134159088,
      "logps/chosen": -587.6082763671875,
      "logps/rejected": -454.1474304199219,
      "loss": 0.9359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3298547863960266,
      "rewards/margins": 1.1505495309829712,
      "rewards/rejected": -1.4804043769836426,
      "step": 5872
    },
    {
      "epoch": 0.7685286660669011,
      "grad_norm": 9.24310302734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10593372583389282,
      "logits/rejected": -0.11467470973730087,
      "logps/chosen": -542.5391235351562,
      "logps/rejected": -423.7974853515625,
      "loss": 0.7418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5712913870811462,
      "rewards/margins": 0.4870971441268921,
      "rewards/rejected": -1.058388590812683,
      "step": 5873
    },
    {
      "epoch": 0.7686595240042529,
      "grad_norm": 10.538026809692383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.34991925954818726,
      "logits/rejected": 0.556221067905426,
      "logps/chosen": -453.02569580078125,
      "logps/rejected": -520.479248046875,
      "loss": 0.7031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23783773183822632,
      "rewards/margins": 0.6257758140563965,
      "rewards/rejected": -0.863613486289978,
      "step": 5874
    },
    {
      "epoch": 0.7687903819416047,
      "grad_norm": 4.40731954574585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25378599762916565,
      "logits/rejected": 0.262570321559906,
      "logps/chosen": -595.493408203125,
      "logps/rejected": -506.9267272949219,
      "loss": 0.3139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3113517761230469,
      "rewards/margins": 1.4786605834960938,
      "rewards/rejected": -1.7900123596191406,
      "step": 5875
    },
    {
      "epoch": 0.7689212398789564,
      "grad_norm": 6.931191921234131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03944864124059677,
      "logits/rejected": -0.294812947511673,
      "logps/chosen": -313.51043701171875,
      "logps/rejected": -365.7237548828125,
      "loss": 0.7783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8966705799102783,
      "rewards/margins": -0.008045971393585205,
      "rewards/rejected": -0.8886245489120483,
      "step": 5876
    },
    {
      "epoch": 0.7690520978163081,
      "grad_norm": 4.322996139526367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.053477197885513306,
      "logits/rejected": 0.07538822293281555,
      "logps/chosen": -490.43878173828125,
      "logps/rejected": -293.18060302734375,
      "loss": 0.1986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0044418275356292725,
      "rewards/margins": 2.296367883682251,
      "rewards/rejected": -2.2919259071350098,
      "step": 5877
    },
    {
      "epoch": 0.7691829557536599,
      "grad_norm": 6.368945121765137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27731937170028687,
      "logits/rejected": 0.17986293137073517,
      "logps/chosen": -376.3281555175781,
      "logps/rejected": -490.22161865234375,
      "loss": 0.4176,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2596527338027954,
      "rewards/margins": 1.553442358970642,
      "rewards/rejected": -1.813094973564148,
      "step": 5878
    },
    {
      "epoch": 0.7693138136910117,
      "grad_norm": 9.881989479064941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23672494292259216,
      "logits/rejected": 0.28163284063339233,
      "logps/chosen": -346.8933410644531,
      "logps/rejected": -341.66387939453125,
      "loss": 0.8023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32966849207878113,
      "rewards/margins": 0.4727542996406555,
      "rewards/rejected": -0.8024227619171143,
      "step": 5879
    },
    {
      "epoch": 0.7694446716283635,
      "grad_norm": 7.738917827606201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14056746661663055,
      "logits/rejected": 0.12990021705627441,
      "logps/chosen": -432.6490478515625,
      "logps/rejected": -402.876953125,
      "loss": 0.5463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.676447868347168,
      "rewards/margins": 1.1962379217147827,
      "rewards/rejected": -0.5197901129722595,
      "step": 5880
    },
    {
      "epoch": 0.7695755295657152,
      "grad_norm": 12.342253684997559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0278165340423584,
      "logits/rejected": -0.08782820403575897,
      "logps/chosen": -466.9580078125,
      "logps/rejected": -399.52899169921875,
      "loss": 0.9062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5752546787261963,
      "rewards/margins": 0.5345675349235535,
      "rewards/rejected": -1.109822154045105,
      "step": 5881
    },
    {
      "epoch": 0.769706387503067,
      "grad_norm": 5.910516262054443,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15119203925132751,
      "logits/rejected": -0.2530469298362732,
      "logps/chosen": -341.1510314941406,
      "logps/rejected": -434.6658935546875,
      "loss": 0.3734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4161851406097412,
      "rewards/margins": 1.1701931953430176,
      "rewards/rejected": -1.5863784551620483,
      "step": 5882
    },
    {
      "epoch": 0.7698372454404188,
      "grad_norm": 5.111568450927734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2357928305864334,
      "logits/rejected": -0.3554861545562744,
      "logps/chosen": -360.21435546875,
      "logps/rejected": -325.3865661621094,
      "loss": 0.5139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4535682797431946,
      "rewards/margins": 0.5811005234718323,
      "rewards/rejected": -1.0346686840057373,
      "step": 5883
    },
    {
      "epoch": 0.7699681033777706,
      "grad_norm": 8.932190895080566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03154216706752777,
      "logits/rejected": 0.05953313410282135,
      "logps/chosen": -463.8999938964844,
      "logps/rejected": -414.64508056640625,
      "loss": 0.6503,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5987403988838196,
      "rewards/margins": 0.3888780176639557,
      "rewards/rejected": -0.9876184463500977,
      "step": 5884
    },
    {
      "epoch": 0.7700989613151222,
      "grad_norm": 6.196435451507568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07527890801429749,
      "logits/rejected": 0.10086910426616669,
      "logps/chosen": -457.0179138183594,
      "logps/rejected": -364.0666198730469,
      "loss": 0.3557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5973242521286011,
      "rewards/margins": 1.5796318054199219,
      "rewards/rejected": -0.9823076725006104,
      "step": 5885
    },
    {
      "epoch": 0.770229819252474,
      "grad_norm": 7.363080024719238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06324471533298492,
      "logits/rejected": -0.1814001500606537,
      "logps/chosen": -427.80926513671875,
      "logps/rejected": -322.691162109375,
      "loss": 0.5363,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.30285534262657166,
      "rewards/margins": 0.42263054847717285,
      "rewards/rejected": -0.1197751834988594,
      "step": 5886
    },
    {
      "epoch": 0.7703606771898258,
      "grad_norm": 8.72097396850586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23600320518016815,
      "logits/rejected": 0.023581407964229584,
      "logps/chosen": -479.30279541015625,
      "logps/rejected": -475.89764404296875,
      "loss": 0.6322,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06279946863651276,
      "rewards/margins": 0.6449424624443054,
      "rewards/rejected": -0.707741916179657,
      "step": 5887
    },
    {
      "epoch": 0.7704915351271776,
      "grad_norm": 9.998465538024902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14627933502197266,
      "logits/rejected": 0.08860720694065094,
      "logps/chosen": -449.71221923828125,
      "logps/rejected": -487.3872375488281,
      "loss": 0.4996,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4321691691875458,
      "rewards/margins": 1.7362452745437622,
      "rewards/rejected": -2.1684141159057617,
      "step": 5888
    },
    {
      "epoch": 0.7706223930645293,
      "grad_norm": 5.01924991607666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08969298005104065,
      "logits/rejected": -0.2782885432243347,
      "logps/chosen": -555.5432739257812,
      "logps/rejected": -484.72271728515625,
      "loss": 0.3176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07244184613227844,
      "rewards/margins": 1.5652252435684204,
      "rewards/rejected": -1.637667179107666,
      "step": 5889
    },
    {
      "epoch": 0.7707532510018811,
      "grad_norm": 7.79886531829834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041884057223796844,
      "logits/rejected": -0.0277602206915617,
      "logps/chosen": -262.1449890136719,
      "logps/rejected": -332.73480224609375,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5672323107719421,
      "rewards/margins": 0.6827154755592346,
      "rewards/rejected": -1.2499477863311768,
      "step": 5890
    },
    {
      "epoch": 0.7708841089392329,
      "grad_norm": 7.050924301147461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11381428688764572,
      "logits/rejected": -0.1346278190612793,
      "logps/chosen": -392.01025390625,
      "logps/rejected": -331.4938049316406,
      "loss": 0.5273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19858099520206451,
      "rewards/margins": 0.6832865476608276,
      "rewards/rejected": -0.48470553755760193,
      "step": 5891
    },
    {
      "epoch": 0.7710149668765847,
      "grad_norm": 7.886702060699463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13561087846755981,
      "logits/rejected": 0.27069538831710815,
      "logps/chosen": -470.34539794921875,
      "logps/rejected": -541.3377685546875,
      "loss": 0.6472,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24791547656059265,
      "rewards/margins": 0.5854524374008179,
      "rewards/rejected": -0.33753702044487,
      "step": 5892
    },
    {
      "epoch": 0.7711458248139363,
      "grad_norm": 8.472580909729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044072579592466354,
      "logits/rejected": -0.11253298819065094,
      "logps/chosen": -533.447509765625,
      "logps/rejected": -441.84210205078125,
      "loss": 0.6277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.34636956453323364,
      "rewards/margins": 0.6067338585853577,
      "rewards/rejected": -0.2603643536567688,
      "step": 5893
    },
    {
      "epoch": 0.7712766827512881,
      "grad_norm": 8.196139335632324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08818773180246353,
      "logits/rejected": 0.17830629646778107,
      "logps/chosen": -315.66412353515625,
      "logps/rejected": -375.7384948730469,
      "loss": 0.788,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.290270060300827,
      "rewards/margins": 0.3280780613422394,
      "rewards/rejected": -0.6183481216430664,
      "step": 5894
    },
    {
      "epoch": 0.7714075406886399,
      "grad_norm": 4.658621311187744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4254327416419983,
      "logits/rejected": 0.24875743687152863,
      "logps/chosen": -498.3610534667969,
      "logps/rejected": -701.1848754882812,
      "loss": 0.3756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29359567165374756,
      "rewards/margins": 1.0957492589950562,
      "rewards/rejected": -0.8021535873413086,
      "step": 5895
    },
    {
      "epoch": 0.7715383986259916,
      "grad_norm": 8.379199981689453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027702542021870613,
      "logits/rejected": -0.16287079453468323,
      "logps/chosen": -685.7938842773438,
      "logps/rejected": -508.3222961425781,
      "loss": 0.7184,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.270587682723999,
      "rewards/margins": 0.8838415145874023,
      "rewards/rejected": -1.154429316520691,
      "step": 5896
    },
    {
      "epoch": 0.7716692565633434,
      "grad_norm": 8.282095909118652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11750168353319168,
      "logits/rejected": 0.037605807185173035,
      "logps/chosen": -698.062255859375,
      "logps/rejected": -506.0029296875,
      "loss": 0.5231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4857519268989563,
      "rewards/margins": 1.3559818267822266,
      "rewards/rejected": -1.8417338132858276,
      "step": 5897
    },
    {
      "epoch": 0.7718001145006952,
      "grad_norm": 10.2091703414917,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2581931948661804,
      "logits/rejected": 0.18424484133720398,
      "logps/chosen": -388.2171630859375,
      "logps/rejected": -465.2092590332031,
      "loss": 0.69,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.695936381816864,
      "rewards/margins": 0.560826301574707,
      "rewards/rejected": -1.2567627429962158,
      "step": 5898
    },
    {
      "epoch": 0.771930972438047,
      "grad_norm": 7.1923980712890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3497973084449768,
      "logits/rejected": -0.22432896494865417,
      "logps/chosen": -288.163818359375,
      "logps/rejected": -378.359619140625,
      "loss": 0.4869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7310962677001953,
      "rewards/margins": 0.8063897490501404,
      "rewards/rejected": -1.5374860763549805,
      "step": 5899
    },
    {
      "epoch": 0.7720618303753987,
      "grad_norm": 4.30462121963501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1395566165447235,
      "logits/rejected": -0.1714055836200714,
      "logps/chosen": -293.6165466308594,
      "logps/rejected": -273.45611572265625,
      "loss": 0.2564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38802987337112427,
      "rewards/margins": 1.9537214040756226,
      "rewards/rejected": -1.5656914710998535,
      "step": 5900
    },
    {
      "epoch": 0.7721926883127505,
      "grad_norm": 5.158053874969482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005093187093734741,
      "logits/rejected": 0.10041214525699615,
      "logps/chosen": -507.7955322265625,
      "logps/rejected": -389.7005615234375,
      "loss": 0.2529,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28410908579826355,
      "rewards/margins": 1.5660046339035034,
      "rewards/rejected": -1.2818955183029175,
      "step": 5901
    },
    {
      "epoch": 0.7723235462501022,
      "grad_norm": 9.85999584197998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02449238859117031,
      "logits/rejected": 0.15551231801509857,
      "logps/chosen": -463.1490478515625,
      "logps/rejected": -405.20745849609375,
      "loss": 0.7497,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.412051796913147,
      "rewards/margins": 0.6247377395629883,
      "rewards/rejected": -1.0367895364761353,
      "step": 5902
    },
    {
      "epoch": 0.772454404187454,
      "grad_norm": 7.389463424682617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39060813188552856,
      "logits/rejected": 0.18679580092430115,
      "logps/chosen": -385.38226318359375,
      "logps/rejected": -384.8947448730469,
      "loss": 0.673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.42145824432373047,
      "rewards/margins": 0.3983064293861389,
      "rewards/rejected": -0.8197647929191589,
      "step": 5903
    },
    {
      "epoch": 0.7725852621248057,
      "grad_norm": 6.700804233551025,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20546504855155945,
      "logits/rejected": -0.11403342336416245,
      "logps/chosen": -745.0923461914062,
      "logps/rejected": -525.0394287109375,
      "loss": 0.6274,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6303905844688416,
      "rewards/margins": 0.7885727882385254,
      "rewards/rejected": -1.4189633131027222,
      "step": 5904
    },
    {
      "epoch": 0.7727161200621575,
      "grad_norm": 5.100491046905518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0686919242143631,
      "logits/rejected": -0.32998165488243103,
      "logps/chosen": -417.53179931640625,
      "logps/rejected": -440.7749328613281,
      "loss": 0.3394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3972037136554718,
      "rewards/margins": 1.8142763376235962,
      "rewards/rejected": -1.4170727729797363,
      "step": 5905
    },
    {
      "epoch": 0.7728469779995093,
      "grad_norm": 6.875439167022705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10653328895568848,
      "logits/rejected": 0.14493021368980408,
      "logps/chosen": -456.1978759765625,
      "logps/rejected": -489.1677551269531,
      "loss": 0.5061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4769701063632965,
      "rewards/margins": 0.5771228671073914,
      "rewards/rejected": -0.10015278309583664,
      "step": 5906
    },
    {
      "epoch": 0.7729778359368611,
      "grad_norm": 9.289097785949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19732287526130676,
      "logits/rejected": -0.02899092808365822,
      "logps/chosen": -451.53448486328125,
      "logps/rejected": -424.0442199707031,
      "loss": 0.7036,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6441164016723633,
      "rewards/margins": 0.5984189510345459,
      "rewards/rejected": -1.2425353527069092,
      "step": 5907
    },
    {
      "epoch": 0.7731086938742128,
      "grad_norm": 9.8530855178833,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17442801594734192,
      "logits/rejected": 0.03454969823360443,
      "logps/chosen": -537.1976928710938,
      "logps/rejected": -326.49835205078125,
      "loss": 0.6512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6044402718544006,
      "rewards/margins": 0.7930170297622681,
      "rewards/rejected": -1.3974573612213135,
      "step": 5908
    },
    {
      "epoch": 0.7732395518115646,
      "grad_norm": 6.229611396789551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18209072947502136,
      "logits/rejected": -0.22210347652435303,
      "logps/chosen": -307.3138122558594,
      "logps/rejected": -436.1673583984375,
      "loss": 0.3653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33022040128707886,
      "rewards/margins": 1.6394987106323242,
      "rewards/rejected": -1.3092782497406006,
      "step": 5909
    },
    {
      "epoch": 0.7733704097489164,
      "grad_norm": 3.8654096126556396,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29175692796707153,
      "logits/rejected": 0.027928289026021957,
      "logps/chosen": -270.4131164550781,
      "logps/rejected": -323.5900573730469,
      "loss": 0.3278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.26823773980140686,
      "rewards/margins": 1.2632967233657837,
      "rewards/rejected": -1.5315344333648682,
      "step": 5910
    },
    {
      "epoch": 0.7735012676862681,
      "grad_norm": 10.826094627380371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10471367835998535,
      "logits/rejected": -0.06397447735071182,
      "logps/chosen": -535.7448120117188,
      "logps/rejected": -520.5882568359375,
      "loss": 0.82,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14721356332302094,
      "rewards/margins": 0.79286128282547,
      "rewards/rejected": -0.9400749802589417,
      "step": 5911
    },
    {
      "epoch": 0.7736321256236198,
      "grad_norm": 8.616650581359863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.422532320022583,
      "logits/rejected": 0.30905425548553467,
      "logps/chosen": -459.026123046875,
      "logps/rejected": -453.61346435546875,
      "loss": 0.5344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2543136477470398,
      "rewards/margins": 0.920096755027771,
      "rewards/rejected": -1.1744105815887451,
      "step": 5912
    },
    {
      "epoch": 0.7737629835609716,
      "grad_norm": 5.41607666015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23702266812324524,
      "logits/rejected": -0.15701904892921448,
      "logps/chosen": -538.8352661132812,
      "logps/rejected": -444.4433898925781,
      "loss": 0.4088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1577032059431076,
      "rewards/margins": 1.0117735862731934,
      "rewards/rejected": -0.8540704250335693,
      "step": 5913
    },
    {
      "epoch": 0.7738938414983234,
      "grad_norm": 9.812050819396973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055045805871486664,
      "logits/rejected": -0.15207132697105408,
      "logps/chosen": -419.0667419433594,
      "logps/rejected": -443.74224853515625,
      "loss": 0.6602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2819438874721527,
      "rewards/margins": 0.6872711181640625,
      "rewards/rejected": -0.40532729029655457,
      "step": 5914
    },
    {
      "epoch": 0.7740246994356751,
      "grad_norm": 4.502719879150391,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08493182063102722,
      "logits/rejected": -0.029294010251760483,
      "logps/chosen": -322.73187255859375,
      "logps/rejected": -464.2762451171875,
      "loss": 0.2652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2828001081943512,
      "rewards/margins": 1.9415042400360107,
      "rewards/rejected": -2.224304437637329,
      "step": 5915
    },
    {
      "epoch": 0.7741555573730269,
      "grad_norm": 8.587335586547852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24520762264728546,
      "logits/rejected": -0.21652400493621826,
      "logps/chosen": -737.597412109375,
      "logps/rejected": -448.44720458984375,
      "loss": 0.5181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6570279598236084,
      "rewards/margins": 1.1427347660064697,
      "rewards/rejected": -1.7997628450393677,
      "step": 5916
    },
    {
      "epoch": 0.7742864153103787,
      "grad_norm": 9.151054382324219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1473667323589325,
      "logits/rejected": 0.1057141125202179,
      "logps/chosen": -467.2330322265625,
      "logps/rejected": -488.50152587890625,
      "loss": 0.7783,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.31513407826423645,
      "rewards/margins": 0.2980615794658661,
      "rewards/rejected": -0.6131956577301025,
      "step": 5917
    },
    {
      "epoch": 0.7744172732477305,
      "grad_norm": 11.911637306213379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009049907326698303,
      "logits/rejected": 0.1207253709435463,
      "logps/chosen": -400.1353454589844,
      "logps/rejected": -442.43280029296875,
      "loss": 1.0608,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0536623001098633,
      "rewards/margins": -0.15100803971290588,
      "rewards/rejected": -0.9026542901992798,
      "step": 5918
    },
    {
      "epoch": 0.7745481311850821,
      "grad_norm": 13.145670890808105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13156703114509583,
      "logits/rejected": -0.06873369216918945,
      "logps/chosen": -551.2042236328125,
      "logps/rejected": -745.356201171875,
      "loss": 1.1378,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.4368959665298462,
      "rewards/margins": -0.6044785976409912,
      "rewards/rejected": 0.1675826907157898,
      "step": 5919
    },
    {
      "epoch": 0.7746789891224339,
      "grad_norm": 4.6347856521606445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07753684371709824,
      "logits/rejected": 0.04782218486070633,
      "logps/chosen": -401.69189453125,
      "logps/rejected": -493.70947265625,
      "loss": 0.3371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25688210129737854,
      "rewards/margins": 1.482203483581543,
      "rewards/rejected": -1.7390856742858887,
      "step": 5920
    },
    {
      "epoch": 0.7748098470597857,
      "grad_norm": 4.712509632110596,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10174113512039185,
      "logits/rejected": 0.018657250329852104,
      "logps/chosen": -485.9207763671875,
      "logps/rejected": -584.0966186523438,
      "loss": 0.2088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5702720284461975,
      "rewards/margins": 2.038912296295166,
      "rewards/rejected": -1.4686402082443237,
      "step": 5921
    },
    {
      "epoch": 0.7749407049971375,
      "grad_norm": 5.613793849945068,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29139643907546997,
      "logits/rejected": 0.04221705347299576,
      "logps/chosen": -332.4811096191406,
      "logps/rejected": -421.9667053222656,
      "loss": 0.3186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15414562821388245,
      "rewards/margins": 1.2084109783172607,
      "rewards/rejected": -1.3625566959381104,
      "step": 5922
    },
    {
      "epoch": 0.7750715629344892,
      "grad_norm": 6.94621467590332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06539370864629745,
      "logits/rejected": 0.10558187961578369,
      "logps/chosen": -395.57037353515625,
      "logps/rejected": -492.8945617675781,
      "loss": 0.564,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15160197019577026,
      "rewards/margins": 1.2506499290466309,
      "rewards/rejected": -1.0990478992462158,
      "step": 5923
    },
    {
      "epoch": 0.775202420871841,
      "grad_norm": 5.831725120544434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0509716272354126,
      "logits/rejected": -0.06726932525634766,
      "logps/chosen": -642.3245849609375,
      "logps/rejected": -568.2752685546875,
      "loss": 0.3816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6541683673858643,
      "rewards/margins": 0.9794318675994873,
      "rewards/rejected": -0.3252636194229126,
      "step": 5924
    },
    {
      "epoch": 0.7753332788091928,
      "grad_norm": 7.663511753082275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020004255697131157,
      "logits/rejected": 0.09927770495414734,
      "logps/chosen": -431.9183044433594,
      "logps/rejected": -475.56109619140625,
      "loss": 0.4057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0782947763800621,
      "rewards/margins": 1.2252541780471802,
      "rewards/rejected": -1.303548812866211,
      "step": 5925
    },
    {
      "epoch": 0.7754641367465446,
      "grad_norm": 6.150012016296387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22233770787715912,
      "logits/rejected": -0.021368421614170074,
      "logps/chosen": -675.600830078125,
      "logps/rejected": -406.674560546875,
      "loss": 0.5572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7238567471504211,
      "rewards/margins": 0.8424858450889587,
      "rewards/rejected": -1.5663425922393799,
      "step": 5926
    },
    {
      "epoch": 0.7755949946838963,
      "grad_norm": 7.873988628387451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0029586777091026306,
      "logits/rejected": -0.11718317866325378,
      "logps/chosen": -327.31103515625,
      "logps/rejected": -411.98095703125,
      "loss": 0.5927,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5050733685493469,
      "rewards/margins": 1.386806845664978,
      "rewards/rejected": -0.8817333579063416,
      "step": 5927
    },
    {
      "epoch": 0.775725852621248,
      "grad_norm": 7.008601665496826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0015243403613567352,
      "logits/rejected": 0.06481548398733139,
      "logps/chosen": -402.361572265625,
      "logps/rejected": -544.9938354492188,
      "loss": 0.434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4787013530731201,
      "rewards/margins": 1.5349615812301636,
      "rewards/rejected": -1.056260108947754,
      "step": 5928
    },
    {
      "epoch": 0.7758567105585998,
      "grad_norm": 6.401993274688721,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22924748063087463,
      "logits/rejected": -0.31146183609962463,
      "logps/chosen": -534.1784057617188,
      "logps/rejected": -608.3077392578125,
      "loss": 0.3143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5006262063980103,
      "rewards/margins": 1.525521993637085,
      "rewards/rejected": -1.0248957872390747,
      "step": 5929
    },
    {
      "epoch": 0.7759875684959516,
      "grad_norm": 8.246850967407227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10847257822751999,
      "logits/rejected": 0.1469338983297348,
      "logps/chosen": -318.12347412109375,
      "logps/rejected": -442.907958984375,
      "loss": 0.6239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5574769973754883,
      "rewards/margins": 1.058476448059082,
      "rewards/rejected": -1.6159534454345703,
      "step": 5930
    },
    {
      "epoch": 0.7761184264333033,
      "grad_norm": 9.51508617401123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07117562741041183,
      "logits/rejected": 0.16655248403549194,
      "logps/chosen": -344.0256652832031,
      "logps/rejected": -388.872802734375,
      "loss": 0.7717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.674921989440918,
      "rewards/margins": 0.017916515469551086,
      "rewards/rejected": -0.6928384900093079,
      "step": 5931
    },
    {
      "epoch": 0.7762492843706551,
      "grad_norm": 5.75546407699585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20277702808380127,
      "logits/rejected": -0.13120457530021667,
      "logps/chosen": -561.040771484375,
      "logps/rejected": -351.24420166015625,
      "loss": 0.2901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07538510859012604,
      "rewards/margins": 1.8432964086532593,
      "rewards/rejected": -1.9186816215515137,
      "step": 5932
    },
    {
      "epoch": 0.7763801423080069,
      "grad_norm": 8.427934646606445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1680675446987152,
      "logits/rejected": -0.18077844381332397,
      "logps/chosen": -492.7156982421875,
      "logps/rejected": -400.2492980957031,
      "loss": 0.5236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8916367292404175,
      "rewards/margins": 0.8795656561851501,
      "rewards/rejected": -1.7712024450302124,
      "step": 5933
    },
    {
      "epoch": 0.7765110002453587,
      "grad_norm": 16.713851928710938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09031577408313751,
      "logits/rejected": 0.039678581058979034,
      "logps/chosen": -392.3910827636719,
      "logps/rejected": -336.03704833984375,
      "loss": 1.2618,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0114672183990479,
      "rewards/margins": -0.25488436222076416,
      "rewards/rejected": -0.7565828561782837,
      "step": 5934
    },
    {
      "epoch": 0.7766418581827104,
      "grad_norm": 12.285483360290527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02621237374842167,
      "logits/rejected": -0.006365573033690453,
      "logps/chosen": -341.9059753417969,
      "logps/rejected": -439.8274841308594,
      "loss": 1.0171,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.47107091546058655,
      "rewards/margins": -0.3179687559604645,
      "rewards/rejected": -0.15310212969779968,
      "step": 5935
    },
    {
      "epoch": 0.7767727161200622,
      "grad_norm": 4.429439067840576,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07143402844667435,
      "logits/rejected": 0.019924050197005272,
      "logps/chosen": -458.4093017578125,
      "logps/rejected": -433.6943359375,
      "loss": 0.2394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09506894648075104,
      "rewards/margins": 2.066848039627075,
      "rewards/rejected": -1.9717791080474854,
      "step": 5936
    },
    {
      "epoch": 0.776903574057414,
      "grad_norm": 6.489747524261475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21504516899585724,
      "logits/rejected": 0.05760210379958153,
      "logps/chosen": -397.8173522949219,
      "logps/rejected": -505.7325439453125,
      "loss": 0.4387,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3139910101890564,
      "rewards/margins": 1.2240099906921387,
      "rewards/rejected": -1.5380010604858398,
      "step": 5937
    },
    {
      "epoch": 0.7770344319947656,
      "grad_norm": 9.921504974365234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07845468819141388,
      "logits/rejected": -0.20717328786849976,
      "logps/chosen": -459.2306823730469,
      "logps/rejected": -457.1626892089844,
      "loss": 0.5809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3182670474052429,
      "rewards/margins": 0.8167152404785156,
      "rewards/rejected": -0.4984481930732727,
      "step": 5938
    },
    {
      "epoch": 0.7771652899321174,
      "grad_norm": 9.537075996398926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13732436299324036,
      "logits/rejected": -0.09397890418767929,
      "logps/chosen": -326.6656799316406,
      "logps/rejected": -258.5588684082031,
      "loss": 0.7723,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6249126195907593,
      "rewards/margins": 0.1036214828491211,
      "rewards/rejected": -0.7285341024398804,
      "step": 5939
    },
    {
      "epoch": 0.7772961478694692,
      "grad_norm": 10.620506286621094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0836474820971489,
      "logits/rejected": 0.012528222054243088,
      "logps/chosen": -469.6127014160156,
      "logps/rejected": -540.852783203125,
      "loss": 0.6795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.7162183523178101,
      "rewards/margins": 1.6705677509307861,
      "rewards/rejected": -0.9543493390083313,
      "step": 5940
    },
    {
      "epoch": 0.777427005806821,
      "grad_norm": 9.415430068969727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16181793808937073,
      "logits/rejected": -0.08038549870252609,
      "logps/chosen": -628.21484375,
      "logps/rejected": -475.2744140625,
      "loss": 0.4786,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4387887716293335,
      "rewards/margins": 0.6422378420829773,
      "rewards/rejected": -2.081026554107666,
      "step": 5941
    },
    {
      "epoch": 0.7775578637441727,
      "grad_norm": 6.705024242401123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06484667956829071,
      "logits/rejected": -0.009925901889801025,
      "logps/chosen": -556.2652587890625,
      "logps/rejected": -408.93975830078125,
      "loss": 0.4438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1391693353652954,
      "rewards/margins": 1.1832542419433594,
      "rewards/rejected": -1.3224235773086548,
      "step": 5942
    },
    {
      "epoch": 0.7776887216815245,
      "grad_norm": 10.248414039611816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2295132428407669,
      "logits/rejected": -0.11816279590129852,
      "logps/chosen": -369.95513916015625,
      "logps/rejected": -398.3702087402344,
      "loss": 0.8114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6433925628662109,
      "rewards/margins": 0.6210935711860657,
      "rewards/rejected": -1.2644861936569214,
      "step": 5943
    },
    {
      "epoch": 0.7778195796188763,
      "grad_norm": 6.768768310546875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13162201642990112,
      "logits/rejected": -0.058057066053152084,
      "logps/chosen": -449.43951416015625,
      "logps/rejected": -468.01104736328125,
      "loss": 0.3392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04568694531917572,
      "rewards/margins": 1.275477409362793,
      "rewards/rejected": -1.229790449142456,
      "step": 5944
    },
    {
      "epoch": 0.7779504375562281,
      "grad_norm": 4.5389723777771,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19067063927650452,
      "logits/rejected": -0.3348050117492676,
      "logps/chosen": -409.51409912109375,
      "logps/rejected": -381.8385925292969,
      "loss": 0.2361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3248416781425476,
      "rewards/margins": 1.8739075660705566,
      "rewards/rejected": -2.198749303817749,
      "step": 5945
    },
    {
      "epoch": 0.7780812954935797,
      "grad_norm": 4.1273088455200195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18687835335731506,
      "logits/rejected": -0.00986134260892868,
      "logps/chosen": -424.24566650390625,
      "logps/rejected": -415.79931640625,
      "loss": 0.2638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40198177099227905,
      "rewards/margins": 1.3206095695495605,
      "rewards/rejected": -1.7225911617279053,
      "step": 5946
    },
    {
      "epoch": 0.7782121534309315,
      "grad_norm": 4.630759239196777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12081734836101532,
      "logits/rejected": -0.11736970394849777,
      "logps/chosen": -519.3975830078125,
      "logps/rejected": -472.8604431152344,
      "loss": 0.2021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4903419613838196,
      "rewards/margins": 1.9511241912841797,
      "rewards/rejected": -1.4607820510864258,
      "step": 5947
    },
    {
      "epoch": 0.7783430113682833,
      "grad_norm": 6.471736907958984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19584758579730988,
      "logits/rejected": 0.029244419187307358,
      "logps/chosen": -438.4227600097656,
      "logps/rejected": -394.82763671875,
      "loss": 0.3839,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43597888946533203,
      "rewards/margins": 1.1268365383148193,
      "rewards/rejected": -1.5628154277801514,
      "step": 5948
    },
    {
      "epoch": 0.7784738693056351,
      "grad_norm": 6.358242511749268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06816231459379196,
      "logits/rejected": -0.09545783698558807,
      "logps/chosen": -377.52093505859375,
      "logps/rejected": -357.7943115234375,
      "loss": 0.3984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5532997846603394,
      "rewards/margins": 1.402482032775879,
      "rewards/rejected": -1.9557818174362183,
      "step": 5949
    },
    {
      "epoch": 0.7786047272429868,
      "grad_norm": 5.466110706329346,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13205380737781525,
      "logits/rejected": 0.05247911065816879,
      "logps/chosen": -339.0077819824219,
      "logps/rejected": -382.8793029785156,
      "loss": 0.3128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4302000403404236,
      "rewards/margins": 1.2380694150924683,
      "rewards/rejected": -1.6682696342468262,
      "step": 5950
    },
    {
      "epoch": 0.7787355851803386,
      "grad_norm": 13.40444278717041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11889902502298355,
      "logits/rejected": -0.04816768690943718,
      "logps/chosen": -524.2489013671875,
      "logps/rejected": -402.0301818847656,
      "loss": 0.8755,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12610602378845215,
      "rewards/margins": 0.4867860972881317,
      "rewards/rejected": -0.3606800436973572,
      "step": 5951
    },
    {
      "epoch": 0.7788664431176904,
      "grad_norm": 12.15026569366455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.221138596534729,
      "logits/rejected": -0.2453494369983673,
      "logps/chosen": -351.6084899902344,
      "logps/rejected": -399.3135681152344,
      "loss": 1.0317,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.1786552667617798,
      "rewards/margins": -0.1520133912563324,
      "rewards/rejected": -1.026641845703125,
      "step": 5952
    },
    {
      "epoch": 0.7789973010550422,
      "grad_norm": 4.904496192932129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31125521659851074,
      "logits/rejected": -0.5052749514579773,
      "logps/chosen": -723.692138671875,
      "logps/rejected": -541.4359741210938,
      "loss": 0.3354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15755757689476013,
      "rewards/margins": 1.6771074533462524,
      "rewards/rejected": -1.51954984664917,
      "step": 5953
    },
    {
      "epoch": 0.7791281589923938,
      "grad_norm": 6.704067230224609,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08485943078994751,
      "logits/rejected": 0.11179880797863007,
      "logps/chosen": -512.2312622070312,
      "logps/rejected": -498.4400939941406,
      "loss": 0.4932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12338637560606003,
      "rewards/margins": 0.7023817300796509,
      "rewards/rejected": -0.5789953470230103,
      "step": 5954
    },
    {
      "epoch": 0.7792590169297456,
      "grad_norm": 7.277405261993408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33665600419044495,
      "logits/rejected": -0.2293969988822937,
      "logps/chosen": -439.7881774902344,
      "logps/rejected": -603.78759765625,
      "loss": 0.4521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2543983459472656,
      "rewards/margins": 0.8060734272003174,
      "rewards/rejected": -2.060471773147583,
      "step": 5955
    },
    {
      "epoch": 0.7793898748670974,
      "grad_norm": 8.080942153930664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12371020019054413,
      "logits/rejected": -0.010336987674236298,
      "logps/chosen": -392.4436950683594,
      "logps/rejected": -319.1913146972656,
      "loss": 0.363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.044456660747528076,
      "rewards/margins": 1.555973768234253,
      "rewards/rejected": -1.6004304885864258,
      "step": 5956
    },
    {
      "epoch": 0.7795207328044492,
      "grad_norm": 9.218059539794922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13383211195468903,
      "logits/rejected": -0.05734415352344513,
      "logps/chosen": -381.36962890625,
      "logps/rejected": -341.29571533203125,
      "loss": 0.8687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.06181468814611435,
      "rewards/margins": 0.017670821398496628,
      "rewards/rejected": 0.04414387419819832,
      "step": 5957
    },
    {
      "epoch": 0.7796515907418009,
      "grad_norm": 6.118723392486572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21016612648963928,
      "logits/rejected": -0.20759394764900208,
      "logps/chosen": -414.1236877441406,
      "logps/rejected": -349.59844970703125,
      "loss": 0.3167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3031372129917145,
      "rewards/margins": 1.9287141561508179,
      "rewards/rejected": -1.6255769729614258,
      "step": 5958
    },
    {
      "epoch": 0.7797824486791527,
      "grad_norm": 5.422650337219238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1907663196325302,
      "logits/rejected": -0.22487588226795197,
      "logps/chosen": -517.1070556640625,
      "logps/rejected": -435.86224365234375,
      "loss": 0.3539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08334408700466156,
      "rewards/margins": 1.3376219272613525,
      "rewards/rejected": -1.2542778253555298,
      "step": 5959
    },
    {
      "epoch": 0.7799133066165045,
      "grad_norm": 9.402667999267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11802981793880463,
      "logits/rejected": -0.23533466458320618,
      "logps/chosen": -452.76275634765625,
      "logps/rejected": -379.6470642089844,
      "loss": 0.7803,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19988176226615906,
      "rewards/margins": 0.5152740478515625,
      "rewards/rejected": -0.715155839920044,
      "step": 5960
    },
    {
      "epoch": 0.7800441645538562,
      "grad_norm": 9.530425071716309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05864518880844116,
      "logits/rejected": -0.13425186276435852,
      "logps/chosen": -438.1100158691406,
      "logps/rejected": -610.100830078125,
      "loss": 0.6542,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4912700653076172,
      "rewards/margins": 1.2406257390975952,
      "rewards/rejected": -1.731895923614502,
      "step": 5961
    },
    {
      "epoch": 0.780175022491208,
      "grad_norm": 8.156005859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21642524003982544,
      "logits/rejected": 0.043640054762363434,
      "logps/chosen": -355.0028991699219,
      "logps/rejected": -432.4164733886719,
      "loss": 0.6723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4974885880947113,
      "rewards/margins": 0.3284797668457031,
      "rewards/rejected": -0.8259683847427368,
      "step": 5962
    },
    {
      "epoch": 0.7803058804285598,
      "grad_norm": 6.460000038146973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014428924769163132,
      "logits/rejected": 0.030059918761253357,
      "logps/chosen": -424.9645690917969,
      "logps/rejected": -431.9461364746094,
      "loss": 0.4194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10240821540355682,
      "rewards/margins": 1.091156244277954,
      "rewards/rejected": -1.1935644149780273,
      "step": 5963
    },
    {
      "epoch": 0.7804367383659115,
      "grad_norm": 7.942035675048828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09434685856103897,
      "logits/rejected": -0.16217303276062012,
      "logps/chosen": -430.4309997558594,
      "logps/rejected": -439.48016357421875,
      "loss": 0.5324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7747265100479126,
      "rewards/margins": 0.9488145709037781,
      "rewards/rejected": -1.723541021347046,
      "step": 5964
    },
    {
      "epoch": 0.7805675963032632,
      "grad_norm": 6.731245994567871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013707466423511505,
      "logits/rejected": -0.15502938628196716,
      "logps/chosen": -408.93170166015625,
      "logps/rejected": -388.4351806640625,
      "loss": 0.5773,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19222086668014526,
      "rewards/margins": 1.135207176208496,
      "rewards/rejected": -1.3274281024932861,
      "step": 5965
    },
    {
      "epoch": 0.780698454240615,
      "grad_norm": 8.847981452941895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012480994686484337,
      "logits/rejected": 0.12811382114887238,
      "logps/chosen": -310.4322204589844,
      "logps/rejected": -234.6226806640625,
      "loss": 0.5115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6071916222572327,
      "rewards/margins": 0.6550456285476685,
      "rewards/rejected": -1.2622371912002563,
      "step": 5966
    },
    {
      "epoch": 0.7808293121779668,
      "grad_norm": 10.554914474487305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05125575512647629,
      "logits/rejected": -0.17872004210948944,
      "logps/chosen": -367.22381591796875,
      "logps/rejected": -438.85333251953125,
      "loss": 0.5645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1997298002243042,
      "rewards/margins": 0.6533886194229126,
      "rewards/rejected": -1.8531184196472168,
      "step": 5967
    },
    {
      "epoch": 0.7809601701153186,
      "grad_norm": 12.417383193969727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14197930693626404,
      "logits/rejected": -0.34613892436027527,
      "logps/chosen": -650.6953735351562,
      "logps/rejected": -513.2028198242188,
      "loss": 0.5826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7997598648071289,
      "rewards/margins": 0.8036201596260071,
      "rewards/rejected": -1.6033802032470703,
      "step": 5968
    },
    {
      "epoch": 0.7810910280526703,
      "grad_norm": 9.313591003417969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08402806520462036,
      "logits/rejected": -0.12720844149589539,
      "logps/chosen": -453.57012939453125,
      "logps/rejected": -498.4496154785156,
      "loss": 0.707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6911603212356567,
      "rewards/margins": 0.4895784854888916,
      "rewards/rejected": -1.180738925933838,
      "step": 5969
    },
    {
      "epoch": 0.7812218859900221,
      "grad_norm": 8.49020004272461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0033977534621953964,
      "logits/rejected": -0.0027579814195632935,
      "logps/chosen": -499.5643005371094,
      "logps/rejected": -532.3960571289062,
      "loss": 0.696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4836057722568512,
      "rewards/margins": 0.8526414632797241,
      "rewards/rejected": -1.336247205734253,
      "step": 5970
    },
    {
      "epoch": 0.7813527439273739,
      "grad_norm": 8.658153533935547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02363283932209015,
      "logits/rejected": 0.1169823706150055,
      "logps/chosen": -490.24072265625,
      "logps/rejected": -430.1573181152344,
      "loss": 0.6283,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.47957873344421387,
      "rewards/margins": 0.6954581141471863,
      "rewards/rejected": -0.2158793807029724,
      "step": 5971
    },
    {
      "epoch": 0.7814836018647257,
      "grad_norm": 6.483968257904053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017822660505771637,
      "logits/rejected": -0.12810513377189636,
      "logps/chosen": -425.3431701660156,
      "logps/rejected": -317.3739929199219,
      "loss": 0.5476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6158994436264038,
      "rewards/margins": 0.5712182521820068,
      "rewards/rejected": 0.04468120262026787,
      "step": 5972
    },
    {
      "epoch": 0.7816144598020773,
      "grad_norm": 10.27779769897461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16674689948558807,
      "logits/rejected": 0.27477699518203735,
      "logps/chosen": -530.4767456054688,
      "logps/rejected": -546.4173583984375,
      "loss": 0.6293,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2083505541086197,
      "rewards/margins": 0.4287569522857666,
      "rewards/rejected": -0.6371074914932251,
      "step": 5973
    },
    {
      "epoch": 0.7817453177394291,
      "grad_norm": 8.494976043701172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20027990639209747,
      "logits/rejected": -0.049238380044698715,
      "logps/chosen": -374.07403564453125,
      "logps/rejected": -424.17156982421875,
      "loss": 0.81,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6557748913764954,
      "rewards/margins": 0.2424720674753189,
      "rewards/rejected": -0.8982469439506531,
      "step": 5974
    },
    {
      "epoch": 0.7818761756767809,
      "grad_norm": 6.76663875579834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015781208872795105,
      "logits/rejected": 0.009478189051151276,
      "logps/chosen": -363.0934753417969,
      "logps/rejected": -386.4289245605469,
      "loss": 0.4074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.008672922849655151,
      "rewards/margins": 1.7359299659729004,
      "rewards/rejected": -1.7446027994155884,
      "step": 5975
    },
    {
      "epoch": 0.7820070336141327,
      "grad_norm": 9.318986892700195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17686432600021362,
      "logits/rejected": -0.06486284732818604,
      "logps/chosen": -670.4903564453125,
      "logps/rejected": -514.4154052734375,
      "loss": 0.6759,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38530102372169495,
      "rewards/margins": 0.7750396132469177,
      "rewards/rejected": -1.1603407859802246,
      "step": 5976
    },
    {
      "epoch": 0.7821378915514844,
      "grad_norm": 5.876216411590576,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07967165112495422,
      "logits/rejected": -0.25259628891944885,
      "logps/chosen": -505.66033935546875,
      "logps/rejected": -631.814697265625,
      "loss": 0.3299,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4287167191505432,
      "rewards/margins": 1.49082350730896,
      "rewards/rejected": -1.9195404052734375,
      "step": 5977
    },
    {
      "epoch": 0.7822687494888362,
      "grad_norm": 15.269794464111328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020427659153938293,
      "logits/rejected": -0.08313433825969696,
      "logps/chosen": -542.0072021484375,
      "logps/rejected": -532.3028564453125,
      "loss": 0.5235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37480470538139343,
      "rewards/margins": 0.9051746129989624,
      "rewards/rejected": -1.2799792289733887,
      "step": 5978
    },
    {
      "epoch": 0.782399607426188,
      "grad_norm": 6.847890853881836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040944501757621765,
      "logits/rejected": 0.023456498980522156,
      "logps/chosen": -605.7244873046875,
      "logps/rejected": -532.7147827148438,
      "loss": 0.4407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18016335368156433,
      "rewards/margins": 0.9472184181213379,
      "rewards/rejected": -0.7670550346374512,
      "step": 5979
    },
    {
      "epoch": 0.7825304653635398,
      "grad_norm": 7.71327543258667,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04038834944367409,
      "logits/rejected": -0.1991494596004486,
      "logps/chosen": -415.083740234375,
      "logps/rejected": -559.807861328125,
      "loss": 0.464,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2103278636932373,
      "rewards/margins": 1.8042019605636597,
      "rewards/rejected": -3.0145297050476074,
      "step": 5980
    },
    {
      "epoch": 0.7826613233008914,
      "grad_norm": 15.287969589233398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08182274550199509,
      "logits/rejected": -0.010150745511054993,
      "logps/chosen": -416.57916259765625,
      "logps/rejected": -579.3611450195312,
      "loss": 1.184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46454811096191406,
      "rewards/margins": -0.2864975035190582,
      "rewards/rejected": -0.17805060744285583,
      "step": 5981
    },
    {
      "epoch": 0.7827921812382432,
      "grad_norm": 6.589979648590088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16660484671592712,
      "logits/rejected": -0.07532329112291336,
      "logps/chosen": -368.7186584472656,
      "logps/rejected": -311.9532165527344,
      "loss": 0.6566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1511862725019455,
      "rewards/margins": 0.4354075491428375,
      "rewards/rejected": -0.28422126173973083,
      "step": 5982
    },
    {
      "epoch": 0.782923039175595,
      "grad_norm": 8.78070068359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33089444041252136,
      "logits/rejected": 0.026389330625534058,
      "logps/chosen": -411.0937194824219,
      "logps/rejected": -423.57470703125,
      "loss": 0.5338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.615290105342865,
      "rewards/margins": 0.753244936466217,
      "rewards/rejected": -1.3685351610183716,
      "step": 5983
    },
    {
      "epoch": 0.7830538971129467,
      "grad_norm": 6.284348964691162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19743642210960388,
      "logits/rejected": -0.198552206158638,
      "logps/chosen": -397.5337219238281,
      "logps/rejected": -368.77838134765625,
      "loss": 0.4654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027320288121700287,
      "rewards/margins": 0.7844231128692627,
      "rewards/rejected": -0.8117433190345764,
      "step": 5984
    },
    {
      "epoch": 0.7831847550502985,
      "grad_norm": 3.705329179763794,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20826731622219086,
      "logits/rejected": 0.2702515423297882,
      "logps/chosen": -434.407958984375,
      "logps/rejected": -385.13934326171875,
      "loss": 0.1942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3428137004375458,
      "rewards/margins": 1.6675128936767578,
      "rewards/rejected": -1.3246991634368896,
      "step": 5985
    },
    {
      "epoch": 0.7833156129876503,
      "grad_norm": 8.205191612243652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05777449160814285,
      "logits/rejected": 0.296842098236084,
      "logps/chosen": -693.0511474609375,
      "logps/rejected": -451.3416748046875,
      "loss": 0.4484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.2574386596679688,
      "rewards/margins": 1.6944375038146973,
      "rewards/rejected": -0.43699896335601807,
      "step": 5986
    },
    {
      "epoch": 0.7834464709250021,
      "grad_norm": 7.833932399749756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17646031081676483,
      "logits/rejected": -0.21337106823921204,
      "logps/chosen": -335.9297790527344,
      "logps/rejected": -416.1093444824219,
      "loss": 0.6746,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29001617431640625,
      "rewards/margins": 0.7389549016952515,
      "rewards/rejected": -1.0289710760116577,
      "step": 5987
    },
    {
      "epoch": 0.7835773288623538,
      "grad_norm": 13.372847557067871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19009172916412354,
      "logits/rejected": -0.3964212238788605,
      "logps/chosen": -558.3489379882812,
      "logps/rejected": -431.1979064941406,
      "loss": 0.9159,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4110866189002991,
      "rewards/margins": 0.10801604390144348,
      "rewards/rejected": -0.5191026926040649,
      "step": 5988
    },
    {
      "epoch": 0.7837081867997056,
      "grad_norm": 7.044205665588379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07637536525726318,
      "logits/rejected": -0.11545592546463013,
      "logps/chosen": -609.8111572265625,
      "logps/rejected": -396.4619140625,
      "loss": 0.4286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10587558150291443,
      "rewards/margins": 1.114626407623291,
      "rewards/rejected": -1.0087507963180542,
      "step": 5989
    },
    {
      "epoch": 0.7838390447370573,
      "grad_norm": 10.47585678100586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21240977942943573,
      "logits/rejected": 0.061307113617658615,
      "logps/chosen": -564.6741943359375,
      "logps/rejected": -445.02508544921875,
      "loss": 0.7616,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.10630761086940765,
      "rewards/margins": 0.2167019098997116,
      "rewards/rejected": -0.11039425432682037,
      "step": 5990
    },
    {
      "epoch": 0.7839699026744091,
      "grad_norm": 11.27603530883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17263387143611908,
      "logits/rejected": -0.07994207739830017,
      "logps/chosen": -582.9635009765625,
      "logps/rejected": -456.59307861328125,
      "loss": 0.9435,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23637047410011292,
      "rewards/margins": -0.2016395926475525,
      "rewards/rejected": -0.03473088890314102,
      "step": 5991
    },
    {
      "epoch": 0.7841007606117608,
      "grad_norm": 5.6848955154418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09793765842914581,
      "logits/rejected": 0.08038709312677383,
      "logps/chosen": -399.64605712890625,
      "logps/rejected": -337.92071533203125,
      "loss": 0.4078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.882669985294342,
      "rewards/margins": 1.296278953552246,
      "rewards/rejected": -0.4136089086532593,
      "step": 5992
    },
    {
      "epoch": 0.7842316185491126,
      "grad_norm": 7.425836086273193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024839140474796295,
      "logits/rejected": -0.051288098096847534,
      "logps/chosen": -447.62384033203125,
      "logps/rejected": -500.3670654296875,
      "loss": 0.4722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8045652508735657,
      "rewards/margins": 0.8615763783454895,
      "rewards/rejected": -0.05701106786727905,
      "step": 5993
    },
    {
      "epoch": 0.7843624764864644,
      "grad_norm": 9.751984596252441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10902231186628342,
      "logits/rejected": 0.01727655529975891,
      "logps/chosen": -609.8668823242188,
      "logps/rejected": -566.6849365234375,
      "loss": 0.5933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3201114535331726,
      "rewards/margins": 0.6901880502700806,
      "rewards/rejected": -1.0102994441986084,
      "step": 5994
    },
    {
      "epoch": 0.7844933344238162,
      "grad_norm": 7.053305625915527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046806156635284424,
      "logits/rejected": 0.055036868900060654,
      "logps/chosen": -566.963134765625,
      "logps/rejected": -473.3443908691406,
      "loss": 0.3365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41421282291412354,
      "rewards/margins": 1.6543468236923218,
      "rewards/rejected": -2.0685598850250244,
      "step": 5995
    },
    {
      "epoch": 0.7846241923611679,
      "grad_norm": 9.490632057189941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3093552887439728,
      "logits/rejected": 0.16609638929367065,
      "logps/chosen": -454.2481994628906,
      "logps/rejected": -382.3869934082031,
      "loss": 0.5678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6720676422119141,
      "rewards/margins": 1.3560212850570679,
      "rewards/rejected": -0.6839536428451538,
      "step": 5996
    },
    {
      "epoch": 0.7847550502985197,
      "grad_norm": 6.502910137176514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08398658037185669,
      "logits/rejected": 0.06772342324256897,
      "logps/chosen": -479.5768127441406,
      "logps/rejected": -481.30303955078125,
      "loss": 0.429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5910613536834717,
      "rewards/margins": 0.8569999933242798,
      "rewards/rejected": -1.4480615854263306,
      "step": 5997
    },
    {
      "epoch": 0.7848859082358715,
      "grad_norm": 8.442008972167969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1819707155227661,
      "logits/rejected": -0.09443175792694092,
      "logps/chosen": -474.0444641113281,
      "logps/rejected": -503.72357177734375,
      "loss": 0.6428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1541120707988739,
      "rewards/margins": 0.5494924187660217,
      "rewards/rejected": -0.703604519367218,
      "step": 5998
    },
    {
      "epoch": 0.7850167661732232,
      "grad_norm": 6.0506744384765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026918824762105942,
      "logits/rejected": 0.08291742205619812,
      "logps/chosen": -432.7344970703125,
      "logps/rejected": -454.410400390625,
      "loss": 0.4932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5996620655059814,
      "rewards/margins": 1.3580728769302368,
      "rewards/rejected": -0.7584108710289001,
      "step": 5999
    },
    {
      "epoch": 0.7851476241105749,
      "grad_norm": 3.547501802444458,
      "learning_rate": 5e-06,
      "logits/chosen": -0.073638916015625,
      "logits/rejected": -0.1752897948026657,
      "logps/chosen": -363.8781433105469,
      "logps/rejected": -293.3729248046875,
      "loss": 0.2975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2145838588476181,
      "rewards/margins": 1.5533071756362915,
      "rewards/rejected": -1.7678911685943604,
      "step": 6000
    },
    {
      "epoch": 0.7852784820479267,
      "grad_norm": 8.098196983337402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2386331707239151,
      "logits/rejected": 0.2069886475801468,
      "logps/chosen": -471.72454833984375,
      "logps/rejected": -457.03057861328125,
      "loss": 0.5892,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.35224175453186035,
      "rewards/margins": 0.7542555928230286,
      "rewards/rejected": -1.1064974069595337,
      "step": 6001
    },
    {
      "epoch": 0.7854093399852785,
      "grad_norm": 7.356818675994873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18970730900764465,
      "logits/rejected": 0.20984041690826416,
      "logps/chosen": -383.8765563964844,
      "logps/rejected": -388.96246337890625,
      "loss": 0.5934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.14369378983974457,
      "rewards/margins": 0.7244052886962891,
      "rewards/rejected": -0.5807116031646729,
      "step": 6002
    },
    {
      "epoch": 0.7855401979226302,
      "grad_norm": 6.392996788024902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007156848907470703,
      "logits/rejected": -0.10175257176160812,
      "logps/chosen": -612.0576171875,
      "logps/rejected": -382.18939208984375,
      "loss": 0.4301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3587286174297333,
      "rewards/margins": 1.102532982826233,
      "rewards/rejected": -0.743804395198822,
      "step": 6003
    },
    {
      "epoch": 0.785671055859982,
      "grad_norm": 4.173234462738037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027641110122203827,
      "logits/rejected": -0.1595292091369629,
      "logps/chosen": -610.5503540039062,
      "logps/rejected": -445.6224365234375,
      "loss": 0.2178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18110808730125427,
      "rewards/margins": 1.5837996006011963,
      "rewards/rejected": -1.4026912450790405,
      "step": 6004
    },
    {
      "epoch": 0.7858019137973338,
      "grad_norm": 6.620136737823486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2749512791633606,
      "logits/rejected": -0.06715652346611023,
      "logps/chosen": -403.0892333984375,
      "logps/rejected": -369.5863037109375,
      "loss": 0.5896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2169208824634552,
      "rewards/margins": 0.2988712191581726,
      "rewards/rejected": -0.5157920718193054,
      "step": 6005
    },
    {
      "epoch": 0.7859327717346856,
      "grad_norm": 9.45681095123291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.217788964509964,
      "logits/rejected": -0.20893064141273499,
      "logps/chosen": -377.99456787109375,
      "logps/rejected": -448.66888427734375,
      "loss": 0.5367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5690147280693054,
      "rewards/margins": 0.6856362223625183,
      "rewards/rejected": -1.2546508312225342,
      "step": 6006
    },
    {
      "epoch": 0.7860636296720372,
      "grad_norm": 8.976227760314941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1211937814950943,
      "logits/rejected": 0.07692902535200119,
      "logps/chosen": -557.498779296875,
      "logps/rejected": -359.5560302734375,
      "loss": 0.6481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2672117352485657,
      "rewards/margins": 1.2270441055297852,
      "rewards/rejected": -0.9598323106765747,
      "step": 6007
    },
    {
      "epoch": 0.786194487609389,
      "grad_norm": 9.104669570922852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04071968048810959,
      "logits/rejected": -0.2267945408821106,
      "logps/chosen": -396.9853515625,
      "logps/rejected": -439.79461669921875,
      "loss": 0.5666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47264859080314636,
      "rewards/margins": 0.7768296003341675,
      "rewards/rejected": -1.2494783401489258,
      "step": 6008
    },
    {
      "epoch": 0.7863253455467408,
      "grad_norm": 10.987590789794922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1527058482170105,
      "logits/rejected": -0.08364669233560562,
      "logps/chosen": -547.5833740234375,
      "logps/rejected": -372.08721923828125,
      "loss": 0.7708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26667332649230957,
      "rewards/margins": 0.1589777022600174,
      "rewards/rejected": -0.42565101385116577,
      "step": 6009
    },
    {
      "epoch": 0.7864562034840926,
      "grad_norm": 6.602470874786377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0723942443728447,
      "logits/rejected": 0.14356191456317902,
      "logps/chosen": -531.463623046875,
      "logps/rejected": -478.4951171875,
      "loss": 0.3338,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4498922526836395,
      "rewards/margins": 1.2047933340072632,
      "rewards/rejected": -0.754901111125946,
      "step": 6010
    },
    {
      "epoch": 0.7865870614214443,
      "grad_norm": 7.384260654449463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027006763964891434,
      "logits/rejected": 0.056724920868873596,
      "logps/chosen": -566.9395141601562,
      "logps/rejected": -653.665771484375,
      "loss": 0.4864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22602424025535583,
      "rewards/margins": 0.7843243479728699,
      "rewards/rejected": -0.5583000779151917,
      "step": 6011
    },
    {
      "epoch": 0.7867179193587961,
      "grad_norm": 9.900465965270996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05607162415981293,
      "logits/rejected": -0.011738459579646587,
      "logps/chosen": -628.7752075195312,
      "logps/rejected": -452.8085021972656,
      "loss": 0.4708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10593448579311371,
      "rewards/margins": 1.1030690670013428,
      "rewards/rejected": -0.9971345663070679,
      "step": 6012
    },
    {
      "epoch": 0.7868487772961479,
      "grad_norm": 10.692158699035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019575972110033035,
      "logits/rejected": 0.15780137479305267,
      "logps/chosen": -350.5162353515625,
      "logps/rejected": -452.12823486328125,
      "loss": 0.8691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5895025730133057,
      "rewards/margins": 0.010829374194145203,
      "rewards/rejected": 0.5786731839179993,
      "step": 6013
    },
    {
      "epoch": 0.7869796352334997,
      "grad_norm": 5.996212959289551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05206218734383583,
      "logits/rejected": -0.17048782110214233,
      "logps/chosen": -248.98060607910156,
      "logps/rejected": -430.46002197265625,
      "loss": 0.4084,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24532948434352875,
      "rewards/margins": 1.5580158233642578,
      "rewards/rejected": -1.8033453226089478,
      "step": 6014
    },
    {
      "epoch": 0.7871104931708514,
      "grad_norm": 6.319192886352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2453812062740326,
      "logits/rejected": -0.20659197866916656,
      "logps/chosen": -396.21014404296875,
      "logps/rejected": -404.0814208984375,
      "loss": 0.5526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.313250333070755,
      "rewards/margins": 0.46843206882476807,
      "rewards/rejected": -0.7816823720932007,
      "step": 6015
    },
    {
      "epoch": 0.7872413511082031,
      "grad_norm": 9.090197563171387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14072194695472717,
      "logits/rejected": 0.09846101701259613,
      "logps/chosen": -461.36468505859375,
      "logps/rejected": -489.0292053222656,
      "loss": 0.6802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26459580659866333,
      "rewards/margins": 0.3314386308193207,
      "rewards/rejected": -0.5960344672203064,
      "step": 6016
    },
    {
      "epoch": 0.7873722090455549,
      "grad_norm": 9.279248237609863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07714633643627167,
      "logits/rejected": 0.011669781990349293,
      "logps/chosen": -462.88616943359375,
      "logps/rejected": -521.7118530273438,
      "loss": 0.8379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2526029348373413,
      "rewards/margins": 0.06583249568939209,
      "rewards/rejected": 0.1867704689502716,
      "step": 6017
    },
    {
      "epoch": 0.7875030669829067,
      "grad_norm": 8.622297286987305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2660526633262634,
      "logits/rejected": -0.129869282245636,
      "logps/chosen": -652.7085571289062,
      "logps/rejected": -634.5220947265625,
      "loss": 0.5902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.052873045206069946,
      "rewards/margins": 0.7708596587181091,
      "rewards/rejected": -0.8237327337265015,
      "step": 6018
    },
    {
      "epoch": 0.7876339249202584,
      "grad_norm": 6.620458602905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03172781690955162,
      "logits/rejected": -0.03198568895459175,
      "logps/chosen": -414.4683532714844,
      "logps/rejected": -391.5286560058594,
      "loss": 0.4244,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27704399824142456,
      "rewards/margins": 1.0476548671722412,
      "rewards/rejected": -0.7706108093261719,
      "step": 6019
    },
    {
      "epoch": 0.7877647828576102,
      "grad_norm": 8.368788719177246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15636739134788513,
      "logits/rejected": 0.028365682810544968,
      "logps/chosen": -445.5322570800781,
      "logps/rejected": -518.9747924804688,
      "loss": 0.6787,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6056926846504211,
      "rewards/margins": 0.2292594611644745,
      "rewards/rejected": -0.834952175617218,
      "step": 6020
    },
    {
      "epoch": 0.787895640794962,
      "grad_norm": 7.830328464508057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11193577200174332,
      "logits/rejected": -0.013726092875003815,
      "logps/chosen": -483.9366760253906,
      "logps/rejected": -569.5316772460938,
      "loss": 0.4175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.34832000732421875,
      "rewards/margins": 0.9582450985908508,
      "rewards/rejected": -0.6099250912666321,
      "step": 6021
    },
    {
      "epoch": 0.7880264987323138,
      "grad_norm": 6.413213729858398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3103010058403015,
      "logits/rejected": 0.11182959377765656,
      "logps/chosen": -602.23486328125,
      "logps/rejected": -504.125,
      "loss": 0.4355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39970701932907104,
      "rewards/margins": 1.0494277477264404,
      "rewards/rejected": -0.6497207880020142,
      "step": 6022
    },
    {
      "epoch": 0.7881573566696655,
      "grad_norm": 6.193953037261963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11745734512805939,
      "logits/rejected": 0.0068872347474098206,
      "logps/chosen": -512.2695922851562,
      "logps/rejected": -490.0843200683594,
      "loss": 0.3593,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07594261318445206,
      "rewards/margins": 1.2018005847930908,
      "rewards/rejected": -1.125857949256897,
      "step": 6023
    },
    {
      "epoch": 0.7882882146070173,
      "grad_norm": 6.379179954528809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14733542501926422,
      "logits/rejected": -0.06335582584142685,
      "logps/chosen": -274.6965637207031,
      "logps/rejected": -381.3439025878906,
      "loss": 0.4902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.011345863342285156,
      "rewards/margins": 0.5387697219848633,
      "rewards/rejected": -0.5274239182472229,
      "step": 6024
    },
    {
      "epoch": 0.788419072544369,
      "grad_norm": 7.351748466491699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3708127737045288,
      "logits/rejected": 0.3110560178756714,
      "logps/chosen": -415.5433044433594,
      "logps/rejected": -506.92333984375,
      "loss": 0.394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26188087463378906,
      "rewards/margins": 1.1130008697509766,
      "rewards/rejected": -0.8511199951171875,
      "step": 6025
    },
    {
      "epoch": 0.7885499304817207,
      "grad_norm": 7.3832621574401855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.059710219502449036,
      "logits/rejected": -0.03018823266029358,
      "logps/chosen": -535.2051391601562,
      "logps/rejected": -460.3937683105469,
      "loss": 0.613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.48411619663238525,
      "rewards/margins": 0.3364022970199585,
      "rewards/rejected": 0.14771384000778198,
      "step": 6026
    },
    {
      "epoch": 0.7886807884190725,
      "grad_norm": 4.621064186096191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18022942543029785,
      "logits/rejected": 0.018416747450828552,
      "logps/chosen": -358.8417053222656,
      "logps/rejected": -374.14715576171875,
      "loss": 0.4325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3546814024448395,
      "rewards/margins": 1.6897658109664917,
      "rewards/rejected": -1.3350844383239746,
      "step": 6027
    },
    {
      "epoch": 0.7888116463564243,
      "grad_norm": 6.595070838928223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021509625017642975,
      "logits/rejected": 0.08742094039916992,
      "logps/chosen": -438.85028076171875,
      "logps/rejected": -401.63507080078125,
      "loss": 0.4633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.882780134677887,
      "rewards/margins": 0.9180229306221008,
      "rewards/rejected": -1.8008029460906982,
      "step": 6028
    },
    {
      "epoch": 0.7889425042937761,
      "grad_norm": 4.201901912689209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3573272228240967,
      "logits/rejected": 0.17593561112880707,
      "logps/chosen": -462.6262512207031,
      "logps/rejected": -446.1919860839844,
      "loss": 0.1958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.035980463027954,
      "rewards/margins": 1.8599495887756348,
      "rewards/rejected": -0.8239690661430359,
      "step": 6029
    },
    {
      "epoch": 0.7890733622311278,
      "grad_norm": 8.50846004486084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020482122898101807,
      "logits/rejected": -0.08199884742498398,
      "logps/chosen": -392.58953857421875,
      "logps/rejected": -439.76751708984375,
      "loss": 0.6605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06653784960508347,
      "rewards/margins": 0.35043585300445557,
      "rewards/rejected": -0.2838980257511139,
      "step": 6030
    },
    {
      "epoch": 0.7892042201684796,
      "grad_norm": 5.554588794708252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20662787556648254,
      "logits/rejected": -0.2704125642776489,
      "logps/chosen": -488.34344482421875,
      "logps/rejected": -313.4002380371094,
      "loss": 0.379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1927347332239151,
      "rewards/margins": 1.6508383750915527,
      "rewards/rejected": -1.8435730934143066,
      "step": 6031
    },
    {
      "epoch": 0.7893350781058314,
      "grad_norm": 6.635054588317871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09816041588783264,
      "logits/rejected": 0.04909373074769974,
      "logps/chosen": -350.1154479980469,
      "logps/rejected": -538.604248046875,
      "loss": 0.5381,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2626497149467468,
      "rewards/margins": 1.3901702165603638,
      "rewards/rejected": -1.6528198719024658,
      "step": 6032
    },
    {
      "epoch": 0.7894659360431832,
      "grad_norm": 8.113701820373535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17911779880523682,
      "logits/rejected": 0.1942082643508911,
      "logps/chosen": -419.1624755859375,
      "logps/rejected": -441.4757080078125,
      "loss": 0.5715,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5242664217948914,
      "rewards/margins": 0.9102239608764648,
      "rewards/rejected": -0.3859575390815735,
      "step": 6033
    },
    {
      "epoch": 0.7895967939805348,
      "grad_norm": 5.805956840515137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12406192719936371,
      "logits/rejected": 0.028511449694633484,
      "logps/chosen": -369.2271423339844,
      "logps/rejected": -360.1578063964844,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7135601043701172,
      "rewards/margins": 1.0417242050170898,
      "rewards/rejected": -1.755284309387207,
      "step": 6034
    },
    {
      "epoch": 0.7897276519178866,
      "grad_norm": 6.811953067779541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15588660538196564,
      "logits/rejected": 0.26055580377578735,
      "logps/chosen": -333.8494567871094,
      "logps/rejected": -393.70147705078125,
      "loss": 0.4955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5876701474189758,
      "rewards/margins": 0.8501911163330078,
      "rewards/rejected": -0.262520968914032,
      "step": 6035
    },
    {
      "epoch": 0.7898585098552384,
      "grad_norm": 7.796963691711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10748683661222458,
      "logits/rejected": -0.2102632075548172,
      "logps/chosen": -342.6835021972656,
      "logps/rejected": -390.171630859375,
      "loss": 0.5811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08656160533428192,
      "rewards/margins": 0.8289703130722046,
      "rewards/rejected": -0.9155319333076477,
      "step": 6036
    },
    {
      "epoch": 0.7899893677925902,
      "grad_norm": 8.278878211975098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36133286356925964,
      "logits/rejected": -0.08925601094961166,
      "logps/chosen": -558.1512451171875,
      "logps/rejected": -364.2113037109375,
      "loss": 0.6552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6022297143936157,
      "rewards/margins": 1.3732365369796753,
      "rewards/rejected": -0.7710069417953491,
      "step": 6037
    },
    {
      "epoch": 0.7901202257299419,
      "grad_norm": 8.355108261108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13260483741760254,
      "logits/rejected": -0.12892469763755798,
      "logps/chosen": -366.23504638671875,
      "logps/rejected": -332.2400817871094,
      "loss": 0.6225,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.37410473823547363,
      "rewards/margins": 0.4677879214286804,
      "rewards/rejected": -0.09368322789669037,
      "step": 6038
    },
    {
      "epoch": 0.7902510836672937,
      "grad_norm": 3.441213846206665,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02201354131102562,
      "logits/rejected": -0.021295860409736633,
      "logps/chosen": -373.786376953125,
      "logps/rejected": -365.7143249511719,
      "loss": 0.345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10207080841064453,
      "rewards/margins": 1.4562678337097168,
      "rewards/rejected": -1.3541971445083618,
      "step": 6039
    },
    {
      "epoch": 0.7903819416046455,
      "grad_norm": 13.913150787353516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10208852589130402,
      "logits/rejected": -0.0448957234621048,
      "logps/chosen": -503.6316833496094,
      "logps/rejected": -311.9264831542969,
      "loss": 0.751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8016561269760132,
      "rewards/margins": 0.37008440494537354,
      "rewards/rejected": -1.1717406511306763,
      "step": 6040
    },
    {
      "epoch": 0.7905127995419973,
      "grad_norm": 7.311960220336914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2242383062839508,
      "logits/rejected": 0.020552027970552444,
      "logps/chosen": -380.3612976074219,
      "logps/rejected": -390.3311767578125,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7814983129501343,
      "rewards/margins": 1.33464777469635,
      "rewards/rejected": -0.553149402141571,
      "step": 6041
    },
    {
      "epoch": 0.790643657479349,
      "grad_norm": 8.629005432128906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28651443123817444,
      "logits/rejected": 0.13062676787376404,
      "logps/chosen": -546.186767578125,
      "logps/rejected": -588.9788818359375,
      "loss": 0.5193,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.686092734336853,
      "rewards/margins": 0.890041708946228,
      "rewards/rejected": -1.5761345624923706,
      "step": 6042
    },
    {
      "epoch": 0.7907745154167007,
      "grad_norm": 5.6851043701171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2741713225841522,
      "logits/rejected": 0.21170274913311005,
      "logps/chosen": -513.3352661132812,
      "logps/rejected": -474.3262939453125,
      "loss": 0.3338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9528654217720032,
      "rewards/margins": 1.3498101234436035,
      "rewards/rejected": -0.3969448208808899,
      "step": 6043
    },
    {
      "epoch": 0.7909053733540525,
      "grad_norm": 5.817917346954346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1928263008594513,
      "logits/rejected": -0.1643485128879547,
      "logps/chosen": -346.66778564453125,
      "logps/rejected": -379.85870361328125,
      "loss": 0.5489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19371624290943146,
      "rewards/margins": 0.46292629837989807,
      "rewards/rejected": -0.6566425561904907,
      "step": 6044
    },
    {
      "epoch": 0.7910362312914043,
      "grad_norm": 4.944937229156494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11212003231048584,
      "logits/rejected": -0.29878777265548706,
      "logps/chosen": -374.0781555175781,
      "logps/rejected": -388.5565490722656,
      "loss": 0.4267,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.364227831363678,
      "rewards/margins": 1.6614887714385986,
      "rewards/rejected": -1.2972608804702759,
      "step": 6045
    },
    {
      "epoch": 0.791167089228756,
      "grad_norm": 5.169951438903809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16757096350193024,
      "logits/rejected": 0.32886165380477905,
      "logps/chosen": -371.73406982421875,
      "logps/rejected": -412.41094970703125,
      "loss": 0.3719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9434690475463867,
      "rewards/margins": 2.1287307739257812,
      "rewards/rejected": -1.1852617263793945,
      "step": 6046
    },
    {
      "epoch": 0.7912979471661078,
      "grad_norm": 6.671395778656006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11001859605312347,
      "logits/rejected": -0.20156458020210266,
      "logps/chosen": -374.8029479980469,
      "logps/rejected": -434.456298828125,
      "loss": 0.5092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014544695615768433,
      "rewards/margins": 0.9040247201919556,
      "rewards/rejected": -0.9185694456100464,
      "step": 6047
    },
    {
      "epoch": 0.7914288051034596,
      "grad_norm": 6.1355133056640625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0034542009234428406,
      "logits/rejected": -0.03216502442955971,
      "logps/chosen": -373.4091796875,
      "logps/rejected": -544.9019165039062,
      "loss": 0.4045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31117480993270874,
      "rewards/margins": 1.1419296264648438,
      "rewards/rejected": -0.8307549357414246,
      "step": 6048
    },
    {
      "epoch": 0.7915596630408113,
      "grad_norm": 11.53265380859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39108243584632874,
      "logits/rejected": 0.30977120995521545,
      "logps/chosen": -536.9957885742188,
      "logps/rejected": -435.2848815917969,
      "loss": 0.8864,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5444076657295227,
      "rewards/margins": 0.023592010140419006,
      "rewards/rejected": -0.5679996609687805,
      "step": 6049
    },
    {
      "epoch": 0.7916905209781631,
      "grad_norm": 7.972679615020752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02559533715248108,
      "logits/rejected": 0.0625031590461731,
      "logps/chosen": -395.90728759765625,
      "logps/rejected": -534.9234619140625,
      "loss": 0.435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23041841387748718,
      "rewards/margins": 0.919745683670044,
      "rewards/rejected": -1.1501641273498535,
      "step": 6050
    },
    {
      "epoch": 0.7918213789155149,
      "grad_norm": 9.414302825927734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22932833433151245,
      "logits/rejected": 0.07701239734888077,
      "logps/chosen": -687.3563232421875,
      "logps/rejected": -683.3622436523438,
      "loss": 0.8121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41499829292297363,
      "rewards/margins": 0.40883976221084595,
      "rewards/rejected": -0.8238381147384644,
      "step": 6051
    },
    {
      "epoch": 0.7919522368528666,
      "grad_norm": 8.806443214416504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1293807029724121,
      "logits/rejected": -0.10291021317243576,
      "logps/chosen": -584.5906982421875,
      "logps/rejected": -490.79119873046875,
      "loss": 0.4402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11927281320095062,
      "rewards/margins": 1.4597729444503784,
      "rewards/rejected": -1.3405001163482666,
      "step": 6052
    },
    {
      "epoch": 0.7920830947902183,
      "grad_norm": 7.62669038772583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17068980634212494,
      "logits/rejected": -0.1616193652153015,
      "logps/chosen": -519.1063232421875,
      "logps/rejected": -424.07305908203125,
      "loss": 0.4573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2638073265552521,
      "rewards/margins": 0.7840217351913452,
      "rewards/rejected": -0.5202144384384155,
      "step": 6053
    },
    {
      "epoch": 0.7922139527275701,
      "grad_norm": 8.66594409942627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24870643019676208,
      "logits/rejected": -0.12420100718736649,
      "logps/chosen": -499.64569091796875,
      "logps/rejected": -312.33551025390625,
      "loss": 0.5588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15456333756446838,
      "rewards/margins": 0.5732633471488953,
      "rewards/rejected": -0.41870003938674927,
      "step": 6054
    },
    {
      "epoch": 0.7923448106649219,
      "grad_norm": 9.825806617736816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23929943144321442,
      "logits/rejected": 0.08744911849498749,
      "logps/chosen": -422.41943359375,
      "logps/rejected": -576.5119018554688,
      "loss": 0.7322,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9521800875663757,
      "rewards/margins": 0.18433362245559692,
      "rewards/rejected": -1.1365137100219727,
      "step": 6055
    },
    {
      "epoch": 0.7924756686022737,
      "grad_norm": 7.108356952667236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08232107758522034,
      "logits/rejected": 0.03505217283964157,
      "logps/chosen": -617.8087768554688,
      "logps/rejected": -655.0057373046875,
      "loss": 0.357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1736934930086136,
      "rewards/margins": 1.3410232067108154,
      "rewards/rejected": -1.1673297882080078,
      "step": 6056
    },
    {
      "epoch": 0.7926065265396254,
      "grad_norm": 9.320924758911133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14810368418693542,
      "logits/rejected": -0.006882406771183014,
      "logps/chosen": -559.9635009765625,
      "logps/rejected": -472.7833557128906,
      "loss": 0.6221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42165014147758484,
      "rewards/margins": 0.5962246656417847,
      "rewards/rejected": -0.17457443475723267,
      "step": 6057
    },
    {
      "epoch": 0.7927373844769772,
      "grad_norm": 7.549310684204102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0099315345287323,
      "logits/rejected": -0.17578843235969543,
      "logps/chosen": -444.12054443359375,
      "logps/rejected": -406.0335693359375,
      "loss": 0.6204,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5578654408454895,
      "rewards/margins": 0.22401714324951172,
      "rewards/rejected": -0.781882643699646,
      "step": 6058
    },
    {
      "epoch": 0.792868242414329,
      "grad_norm": 5.357588768005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15305210649967194,
      "logits/rejected": 0.019511567428708076,
      "logps/chosen": -358.4590759277344,
      "logps/rejected": -415.67242431640625,
      "loss": 0.398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2883787155151367,
      "rewards/margins": 1.0874173641204834,
      "rewards/rejected": -1.3757961988449097,
      "step": 6059
    },
    {
      "epoch": 0.7929991003516808,
      "grad_norm": 6.019255638122559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1197190135717392,
      "logits/rejected": -0.161794051527977,
      "logps/chosen": -478.54400634765625,
      "logps/rejected": -407.08087158203125,
      "loss": 0.2719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04862920939922333,
      "rewards/margins": 1.498092532157898,
      "rewards/rejected": -1.4494634866714478,
      "step": 6060
    },
    {
      "epoch": 0.7931299582890324,
      "grad_norm": 7.418084621429443,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05181771516799927,
      "logits/rejected": -0.05059202387928963,
      "logps/chosen": -324.9409484863281,
      "logps/rejected": -353.866943359375,
      "loss": 0.5603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1384241282939911,
      "rewards/margins": 0.4732675552368164,
      "rewards/rejected": -0.3348434567451477,
      "step": 6061
    },
    {
      "epoch": 0.7932608162263842,
      "grad_norm": 5.66431188583374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2867299020290375,
      "logits/rejected": 0.2387157827615738,
      "logps/chosen": -379.87451171875,
      "logps/rejected": -377.7094421386719,
      "loss": 0.3892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3512583076953888,
      "rewards/margins": 0.9759835004806519,
      "rewards/rejected": -0.6247251629829407,
      "step": 6062
    },
    {
      "epoch": 0.793391674163736,
      "grad_norm": 9.188090324401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027184322476387024,
      "logits/rejected": -0.1981753259897232,
      "logps/chosen": -451.5670166015625,
      "logps/rejected": -314.2653503417969,
      "loss": 0.6648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3360129594802856,
      "rewards/margins": 0.15833739936351776,
      "rewards/rejected": -1.4943503141403198,
      "step": 6063
    },
    {
      "epoch": 0.7935225321010878,
      "grad_norm": 7.833946228027344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3673802614212036,
      "logits/rejected": 0.4170418977737427,
      "logps/chosen": -382.8885498046875,
      "logps/rejected": -339.1539306640625,
      "loss": 0.5593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10488051176071167,
      "rewards/margins": 0.6716250777244568,
      "rewards/rejected": -0.5667445659637451,
      "step": 6064
    },
    {
      "epoch": 0.7936533900384395,
      "grad_norm": 8.971152305603027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06484077870845795,
      "logits/rejected": -0.15452232956886292,
      "logps/chosen": -560.5792846679688,
      "logps/rejected": -445.7106018066406,
      "loss": 0.614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35922569036483765,
      "rewards/margins": 1.2065473794937134,
      "rewards/rejected": -1.5657730102539062,
      "step": 6065
    },
    {
      "epoch": 0.7937842479757913,
      "grad_norm": 8.326544761657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14907978475093842,
      "logits/rejected": 0.07975037395954132,
      "logps/chosen": -304.251220703125,
      "logps/rejected": -397.4508056640625,
      "loss": 0.8433,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7827097773551941,
      "rewards/margins": 0.07276245951652527,
      "rewards/rejected": -0.8554721474647522,
      "step": 6066
    },
    {
      "epoch": 0.7939151059131431,
      "grad_norm": 11.774145126342773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18370483815670013,
      "logits/rejected": -0.20942333340644836,
      "logps/chosen": -423.6217956542969,
      "logps/rejected": -347.2102966308594,
      "loss": 0.8333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8078796863555908,
      "rewards/margins": 0.4163641333580017,
      "rewards/rejected": -1.2242437601089478,
      "step": 6067
    },
    {
      "epoch": 0.7940459638504948,
      "grad_norm": 7.774269104003906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3272612392902374,
      "logits/rejected": 0.22998538613319397,
      "logps/chosen": -349.4196472167969,
      "logps/rejected": -348.5515441894531,
      "loss": 0.6264,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2534691095352173,
      "rewards/margins": 0.5342726111412048,
      "rewards/rejected": -0.7877417802810669,
      "step": 6068
    },
    {
      "epoch": 0.7941768217878465,
      "grad_norm": 6.628662109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09290570020675659,
      "logits/rejected": -0.33026817440986633,
      "logps/chosen": -443.9785461425781,
      "logps/rejected": -623.1580200195312,
      "loss": 0.4045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1533759981393814,
      "rewards/margins": 0.9459859728813171,
      "rewards/rejected": -1.0993620157241821,
      "step": 6069
    },
    {
      "epoch": 0.7943076797251983,
      "grad_norm": 7.524639129638672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09223156422376633,
      "logits/rejected": -0.24675458669662476,
      "logps/chosen": -238.54847717285156,
      "logps/rejected": -299.5553894042969,
      "loss": 0.5456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5943554043769836,
      "rewards/margins": 0.7769562005996704,
      "rewards/rejected": -1.3713116645812988,
      "step": 6070
    },
    {
      "epoch": 0.7944385376625501,
      "grad_norm": 7.35377836227417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.140888512134552,
      "logits/rejected": -0.3184244930744171,
      "logps/chosen": -510.8282470703125,
      "logps/rejected": -632.039306640625,
      "loss": 0.5023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24525952339172363,
      "rewards/margins": 1.4764642715454102,
      "rewards/rejected": -1.7217237949371338,
      "step": 6071
    },
    {
      "epoch": 0.7945693955999018,
      "grad_norm": 8.107141494750977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12938764691352844,
      "logits/rejected": -0.14063198864459991,
      "logps/chosen": -330.74468994140625,
      "logps/rejected": -332.55419921875,
      "loss": 0.5762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16240844130516052,
      "rewards/margins": 0.49487170577049255,
      "rewards/rejected": -0.33246326446533203,
      "step": 6072
    },
    {
      "epoch": 0.7947002535372536,
      "grad_norm": 6.342197895050049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19731362164020538,
      "logits/rejected": 0.09179063886404037,
      "logps/chosen": -330.10858154296875,
      "logps/rejected": -289.4867248535156,
      "loss": 0.7037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33499184250831604,
      "rewards/margins": 0.7948899865150452,
      "rewards/rejected": -1.1298819780349731,
      "step": 6073
    },
    {
      "epoch": 0.7948311114746054,
      "grad_norm": 8.180121421813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2222619205713272,
      "logits/rejected": -0.29696160554885864,
      "logps/chosen": -620.0433349609375,
      "logps/rejected": -452.4469299316406,
      "loss": 0.355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02693997696042061,
      "rewards/margins": 1.3559083938598633,
      "rewards/rejected": -1.3828485012054443,
      "step": 6074
    },
    {
      "epoch": 0.7949619694119572,
      "grad_norm": 7.763189792633057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08610844612121582,
      "logits/rejected": 0.20922331511974335,
      "logps/chosen": -652.859375,
      "logps/rejected": -512.505615234375,
      "loss": 0.493,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3642423748970032,
      "rewards/margins": 0.7165843844413757,
      "rewards/rejected": -0.35234200954437256,
      "step": 6075
    },
    {
      "epoch": 0.7950928273493089,
      "grad_norm": 6.538786888122559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008810922503471375,
      "logits/rejected": 0.1604575514793396,
      "logps/chosen": -381.8106384277344,
      "logps/rejected": -352.0573425292969,
      "loss": 0.5982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.031181998550891876,
      "rewards/margins": 0.4971422851085663,
      "rewards/rejected": -0.4659602642059326,
      "step": 6076
    },
    {
      "epoch": 0.7952236852866607,
      "grad_norm": 8.602941513061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08946441113948822,
      "logits/rejected": -0.2279854416847229,
      "logps/chosen": -490.9443359375,
      "logps/rejected": -321.24371337890625,
      "loss": 0.5272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5525484085083008,
      "rewards/margins": 0.8941338062286377,
      "rewards/rejected": -1.4466822147369385,
      "step": 6077
    },
    {
      "epoch": 0.7953545432240124,
      "grad_norm": 10.459733009338379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012560063041746616,
      "logits/rejected": -0.0791684091091156,
      "logps/chosen": -354.5409240722656,
      "logps/rejected": -409.90509033203125,
      "loss": 0.5553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002914533019065857,
      "rewards/margins": 1.0616207122802734,
      "rewards/rejected": -1.064535140991211,
      "step": 6078
    },
    {
      "epoch": 0.7954854011613642,
      "grad_norm": 6.149116516113281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0923725813627243,
      "logits/rejected": -0.06419236958026886,
      "logps/chosen": -350.4404602050781,
      "logps/rejected": -307.2366027832031,
      "loss": 0.4659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05068626627326012,
      "rewards/margins": 0.9775397777557373,
      "rewards/rejected": -0.9268534183502197,
      "step": 6079
    },
    {
      "epoch": 0.7956162590987159,
      "grad_norm": 8.107747077941895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007838639430701733,
      "logits/rejected": 0.02296483889222145,
      "logps/chosen": -522.787841796875,
      "logps/rejected": -689.7647094726562,
      "loss": 0.5517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24814167618751526,
      "rewards/margins": 0.7823430299758911,
      "rewards/rejected": -1.0304847955703735,
      "step": 6080
    },
    {
      "epoch": 0.7957471170360677,
      "grad_norm": 5.9105610847473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11824139952659607,
      "logits/rejected": -0.1897035539150238,
      "logps/chosen": -504.4414978027344,
      "logps/rejected": -314.78485107421875,
      "loss": 0.5353,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8366482257843018,
      "rewards/margins": 1.2619357109069824,
      "rewards/rejected": -0.4252876043319702,
      "step": 6081
    },
    {
      "epoch": 0.7958779749734195,
      "grad_norm": 5.131214141845703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08382131159305573,
      "logits/rejected": -0.02144024148583412,
      "logps/chosen": -385.788330078125,
      "logps/rejected": -374.7353820800781,
      "loss": 0.4053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4899452328681946,
      "rewards/margins": 0.8571325540542603,
      "rewards/rejected": -1.34707772731781,
      "step": 6082
    },
    {
      "epoch": 0.7960088329107713,
      "grad_norm": 7.137597560882568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037071000784635544,
      "logits/rejected": 0.08346351981163025,
      "logps/chosen": -371.6065368652344,
      "logps/rejected": -426.5639953613281,
      "loss": 0.3642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23870356380939484,
      "rewards/margins": 1.2711213827133179,
      "rewards/rejected": -1.0324177742004395,
      "step": 6083
    },
    {
      "epoch": 0.796139690848123,
      "grad_norm": 5.673394680023193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018525490537285805,
      "logits/rejected": -0.1936573088169098,
      "logps/chosen": -423.6462707519531,
      "logps/rejected": -353.27239990234375,
      "loss": 0.344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11030101031064987,
      "rewards/margins": 1.7808128595352173,
      "rewards/rejected": -1.8911138772964478,
      "step": 6084
    },
    {
      "epoch": 0.7962705487854748,
      "grad_norm": 9.082066535949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1889469474554062,
      "logits/rejected": -0.10370688140392303,
      "logps/chosen": -413.7546081542969,
      "logps/rejected": -565.5831909179688,
      "loss": 0.6438,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6002477407455444,
      "rewards/margins": 0.4179975390434265,
      "rewards/rejected": -1.0182452201843262,
      "step": 6085
    },
    {
      "epoch": 0.7964014067228266,
      "grad_norm": 9.206666946411133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09047894179821014,
      "logits/rejected": -0.23742564022541046,
      "logps/chosen": -362.4227294921875,
      "logps/rejected": -467.3752136230469,
      "loss": 0.4006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5779587030410767,
      "rewards/margins": 1.2314525842666626,
      "rewards/rejected": -1.8094112873077393,
      "step": 6086
    },
    {
      "epoch": 0.7965322646601783,
      "grad_norm": 10.676773071289062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058412015438079834,
      "logits/rejected": 0.13405388593673706,
      "logps/chosen": -539.6524047851562,
      "logps/rejected": -386.6460876464844,
      "loss": 0.9185,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8487943410873413,
      "rewards/margins": 0.02735210955142975,
      "rewards/rejected": -0.8761465549468994,
      "step": 6087
    },
    {
      "epoch": 0.79666312259753,
      "grad_norm": 6.642673015594482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1436454802751541,
      "logits/rejected": -0.17235910892486572,
      "logps/chosen": -383.25848388671875,
      "logps/rejected": -344.6837158203125,
      "loss": 0.4654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17985382676124573,
      "rewards/margins": 1.0055447816848755,
      "rewards/rejected": -1.1853985786437988,
      "step": 6088
    },
    {
      "epoch": 0.7967939805348818,
      "grad_norm": 8.681285858154297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01537066325545311,
      "logits/rejected": 0.19564871490001678,
      "logps/chosen": -525.1990966796875,
      "logps/rejected": -494.9471740722656,
      "loss": 0.5889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8986847400665283,
      "rewards/margins": 0.8172060251235962,
      "rewards/rejected": 0.08147870004177094,
      "step": 6089
    },
    {
      "epoch": 0.7969248384722336,
      "grad_norm": 10.973231315612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05651175230741501,
      "logits/rejected": 0.012804113328456879,
      "logps/chosen": -559.3894653320312,
      "logps/rejected": -424.3305969238281,
      "loss": 0.7882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4268234670162201,
      "rewards/margins": 0.2019670158624649,
      "rewards/rejected": -0.6287904977798462,
      "step": 6090
    },
    {
      "epoch": 0.7970556964095853,
      "grad_norm": 7.137820243835449,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17427191138267517,
      "logits/rejected": 0.07896151393651962,
      "logps/chosen": -523.8963012695312,
      "logps/rejected": -515.4844970703125,
      "loss": 0.4458,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6734179854393005,
      "rewards/margins": 1.1690078973770142,
      "rewards/rejected": -0.49558982253074646,
      "step": 6091
    },
    {
      "epoch": 0.7971865543469371,
      "grad_norm": 12.130395889282227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09152571856975555,
      "logits/rejected": -0.052398987114429474,
      "logps/chosen": -617.6920776367188,
      "logps/rejected": -414.5663146972656,
      "loss": 0.8458,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5188135504722595,
      "rewards/margins": 0.06222990155220032,
      "rewards/rejected": -0.5810434818267822,
      "step": 6092
    },
    {
      "epoch": 0.7973174122842889,
      "grad_norm": 10.743647575378418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23271878063678741,
      "logits/rejected": 0.14001625776290894,
      "logps/chosen": -440.09490966796875,
      "logps/rejected": -500.6222229003906,
      "loss": 0.9014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005031488835811615,
      "rewards/margins": -0.0258493535220623,
      "rewards/rejected": 0.026352494955062866,
      "step": 6093
    },
    {
      "epoch": 0.7974482702216407,
      "grad_norm": 6.302670955657959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.40744131803512573,
      "logits/rejected": 0.2688119411468506,
      "logps/chosen": -526.611083984375,
      "logps/rejected": -345.28924560546875,
      "loss": 0.51,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5661007165908813,
      "rewards/margins": 0.8031313419342041,
      "rewards/rejected": -0.23703061044216156,
      "step": 6094
    },
    {
      "epoch": 0.7975791281589923,
      "grad_norm": 3.633127450942993,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22081232070922852,
      "logits/rejected": 0.06971748173236847,
      "logps/chosen": -435.4422302246094,
      "logps/rejected": -308.0637512207031,
      "loss": 0.2956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8160490393638611,
      "rewards/margins": 1.8051202297210693,
      "rewards/rejected": -0.9890711307525635,
      "step": 6095
    },
    {
      "epoch": 0.7977099860963441,
      "grad_norm": 11.367293357849121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3131680488586426,
      "logits/rejected": 0.0349758006632328,
      "logps/chosen": -649.6835327148438,
      "logps/rejected": -469.90264892578125,
      "loss": 0.7427,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.905478835105896,
      "rewards/margins": 0.42396658658981323,
      "rewards/rejected": -1.329445481300354,
      "step": 6096
    },
    {
      "epoch": 0.7978408440336959,
      "grad_norm": 7.773149013519287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1730060577392578,
      "logits/rejected": 0.1693720519542694,
      "logps/chosen": -621.8444213867188,
      "logps/rejected": -516.494873046875,
      "loss": 0.5245,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.015269801020622253,
      "rewards/margins": 0.6203685402870178,
      "rewards/rejected": -0.635638415813446,
      "step": 6097
    },
    {
      "epoch": 0.7979717019710477,
      "grad_norm": 8.581175804138184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2741111218929291,
      "logits/rejected": 0.06041578948497772,
      "logps/chosen": -443.0156555175781,
      "logps/rejected": -355.3835144042969,
      "loss": 0.6192,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0345118045806885,
      "rewards/margins": 0.46759697794914246,
      "rewards/rejected": -1.5021086931228638,
      "step": 6098
    },
    {
      "epoch": 0.7981025599083994,
      "grad_norm": 8.043286323547363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12937994301319122,
      "logits/rejected": 0.033318281173706055,
      "logps/chosen": -281.22015380859375,
      "logps/rejected": -324.9178161621094,
      "loss": 0.5803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46443238854408264,
      "rewards/margins": 1.038722038269043,
      "rewards/rejected": -1.5031543970108032,
      "step": 6099
    },
    {
      "epoch": 0.7982334178457512,
      "grad_norm": 7.081291675567627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11147737503051758,
      "logits/rejected": 0.16719064116477966,
      "logps/chosen": -373.712646484375,
      "logps/rejected": -396.01788330078125,
      "loss": 0.5257,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11110533773899078,
      "rewards/margins": 0.7204486131668091,
      "rewards/rejected": -0.6093431711196899,
      "step": 6100
    },
    {
      "epoch": 0.798364275783103,
      "grad_norm": 12.615276336669922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01610189490020275,
      "logits/rejected": -0.06614356487989426,
      "logps/chosen": -596.2924194335938,
      "logps/rejected": -491.175048828125,
      "loss": 0.9069,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.30954933166503906,
      "rewards/margins": 0.10574665665626526,
      "rewards/rejected": -0.4152960181236267,
      "step": 6101
    },
    {
      "epoch": 0.7984951337204548,
      "grad_norm": 10.268098831176758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19220362603664398,
      "logits/rejected": 0.1587512493133545,
      "logps/chosen": -484.832763671875,
      "logps/rejected": -398.4019470214844,
      "loss": 0.7749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3975559175014496,
      "rewards/margins": 0.3319551348686218,
      "rewards/rejected": -0.729511022567749,
      "step": 6102
    },
    {
      "epoch": 0.7986259916578065,
      "grad_norm": 7.066756725311279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1437022089958191,
      "logits/rejected": 0.08416683226823807,
      "logps/chosen": -500.6991271972656,
      "logps/rejected": -364.41680908203125,
      "loss": 0.4969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.043451905250549316,
      "rewards/margins": 0.621336817741394,
      "rewards/rejected": -0.6647887229919434,
      "step": 6103
    },
    {
      "epoch": 0.7987568495951582,
      "grad_norm": 6.846654891967773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10963474214076996,
      "logits/rejected": 0.08444812893867493,
      "logps/chosen": -354.38720703125,
      "logps/rejected": -365.042236328125,
      "loss": 0.4437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12266232073307037,
      "rewards/margins": 1.1838322877883911,
      "rewards/rejected": -1.3064947128295898,
      "step": 6104
    },
    {
      "epoch": 0.79888770753251,
      "grad_norm": 8.191171646118164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08101817965507507,
      "logits/rejected": -0.04646829143166542,
      "logps/chosen": -395.9278259277344,
      "logps/rejected": -421.507080078125,
      "loss": 0.6638,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20806846022605896,
      "rewards/margins": 0.4285978376865387,
      "rewards/rejected": -0.22052937746047974,
      "step": 6105
    },
    {
      "epoch": 0.7990185654698618,
      "grad_norm": 6.266623497009277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07213567197322845,
      "logits/rejected": -0.3200952410697937,
      "logps/chosen": -391.77935791015625,
      "logps/rejected": -493.27215576171875,
      "loss": 0.5891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33717119693756104,
      "rewards/margins": 1.0152086019515991,
      "rewards/rejected": -1.3523797988891602,
      "step": 6106
    },
    {
      "epoch": 0.7991494234072135,
      "grad_norm": 12.620481491088867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12220169603824615,
      "logits/rejected": -0.14025172591209412,
      "logps/chosen": -433.9155578613281,
      "logps/rejected": -331.3786926269531,
      "loss": 0.8946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5854273438453674,
      "rewards/margins": 0.007704868912696838,
      "rewards/rejected": -0.5931322574615479,
      "step": 6107
    },
    {
      "epoch": 0.7992802813445653,
      "grad_norm": 8.270833015441895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06533878296613693,
      "logits/rejected": -0.06408543884754181,
      "logps/chosen": -525.9341430664062,
      "logps/rejected": -429.6336669921875,
      "loss": 0.7085,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2554374933242798,
      "rewards/margins": 0.20951633155345917,
      "rewards/rejected": 0.045921146869659424,
      "step": 6108
    },
    {
      "epoch": 0.7994111392819171,
      "grad_norm": 11.935108184814453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23633776605129242,
      "logits/rejected": 0.07741297781467438,
      "logps/chosen": -645.1625366210938,
      "logps/rejected": -580.1148681640625,
      "loss": 0.7725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8487449884414673,
      "rewards/margins": 0.38139456510543823,
      "rewards/rejected": -1.2301396131515503,
      "step": 6109
    },
    {
      "epoch": 0.7995419972192689,
      "grad_norm": 4.374129772186279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011913672089576721,
      "logits/rejected": 0.016790442168712616,
      "logps/chosen": -474.68743896484375,
      "logps/rejected": -428.449951171875,
      "loss": 0.3581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5737852454185486,
      "rewards/margins": 1.1490435600280762,
      "rewards/rejected": -0.5752583146095276,
      "step": 6110
    },
    {
      "epoch": 0.7996728551566206,
      "grad_norm": 8.689393997192383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08312490582466125,
      "logits/rejected": -0.026367291808128357,
      "logps/chosen": -392.95263671875,
      "logps/rejected": -346.18133544921875,
      "loss": 0.62,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1053098738193512,
      "rewards/margins": 0.38568058609962463,
      "rewards/rejected": -0.28037071228027344,
      "step": 6111
    },
    {
      "epoch": 0.7998037130939724,
      "grad_norm": 11.678631782531738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08581006526947021,
      "logits/rejected": -0.014968746341764927,
      "logps/chosen": -270.4010925292969,
      "logps/rejected": -304.7645263671875,
      "loss": 0.7826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.40695858001708984,
      "rewards/margins": 0.1160145103931427,
      "rewards/rejected": 0.29094406962394714,
      "step": 6112
    },
    {
      "epoch": 0.7999345710313241,
      "grad_norm": 7.839886665344238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09062185883522034,
      "logits/rejected": -0.020893685519695282,
      "logps/chosen": -380.3856201171875,
      "logps/rejected": -521.8658447265625,
      "loss": 0.5453,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5135880708694458,
      "rewards/margins": 0.7726879119873047,
      "rewards/rejected": -1.2862759828567505,
      "step": 6113
    },
    {
      "epoch": 0.8000654289686758,
      "grad_norm": 6.656373023986816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2692420184612274,
      "logits/rejected": -0.31285688281059265,
      "logps/chosen": -478.0210266113281,
      "logps/rejected": -360.9471130371094,
      "loss": 0.3831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1979341208934784,
      "rewards/margins": 1.1560463905334473,
      "rewards/rejected": -0.9581121802330017,
      "step": 6114
    },
    {
      "epoch": 0.8001962869060276,
      "grad_norm": 8.05809497833252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03156203031539917,
      "logits/rejected": 0.059827469289302826,
      "logps/chosen": -420.1253662109375,
      "logps/rejected": -389.40972900390625,
      "loss": 0.4008,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07761305570602417,
      "rewards/margins": 1.4488568305969238,
      "rewards/rejected": -1.5264697074890137,
      "step": 6115
    },
    {
      "epoch": 0.8003271448433794,
      "grad_norm": 9.46142578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24987779557704926,
      "logits/rejected": -0.09697312116622925,
      "logps/chosen": -409.0896301269531,
      "logps/rejected": -466.2098693847656,
      "loss": 0.6406,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.37464749813079834,
      "rewards/margins": 0.7580184936523438,
      "rewards/rejected": -0.3833709955215454,
      "step": 6116
    },
    {
      "epoch": 0.8004580027807312,
      "grad_norm": 7.324875831604004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15448495745658875,
      "logits/rejected": 0.18948465585708618,
      "logps/chosen": -332.9906311035156,
      "logps/rejected": -330.92431640625,
      "loss": 0.5367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.76275634765625,
      "rewards/margins": 0.5306681394577026,
      "rewards/rejected": -1.2934244871139526,
      "step": 6117
    },
    {
      "epoch": 0.8005888607180829,
      "grad_norm": 6.9745025634765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27713966369628906,
      "logits/rejected": -0.3318660259246826,
      "logps/chosen": -387.95318603515625,
      "logps/rejected": -314.14154052734375,
      "loss": 0.4552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.013439152389764786,
      "rewards/margins": 1.2621513605117798,
      "rewards/rejected": -1.2755905389785767,
      "step": 6118
    },
    {
      "epoch": 0.8007197186554347,
      "grad_norm": 7.034898281097412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22945795953273773,
      "logits/rejected": 0.09906558692455292,
      "logps/chosen": -441.3402404785156,
      "logps/rejected": -527.5200805664062,
      "loss": 0.6108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1295570284128189,
      "rewards/margins": 0.8475339412689209,
      "rewards/rejected": -0.717976987361908,
      "step": 6119
    },
    {
      "epoch": 0.8008505765927865,
      "grad_norm": 7.243915557861328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1757563203573227,
      "logits/rejected": 0.148076593875885,
      "logps/chosen": -408.7469787597656,
      "logps/rejected": -449.9195861816406,
      "loss": 0.4793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14827117323875427,
      "rewards/margins": 0.84742671251297,
      "rewards/rejected": -0.9956977367401123,
      "step": 6120
    },
    {
      "epoch": 0.8009814345301383,
      "grad_norm": 6.960325241088867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29121875762939453,
      "logits/rejected": 0.04345235973596573,
      "logps/chosen": -553.5680541992188,
      "logps/rejected": -450.1388854980469,
      "loss": 0.4841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07025210559368134,
      "rewards/margins": 0.9456905722618103,
      "rewards/rejected": -1.0159426927566528,
      "step": 6121
    },
    {
      "epoch": 0.8011122924674899,
      "grad_norm": 4.651883125305176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010658442974090576,
      "logits/rejected": -0.2803138494491577,
      "logps/chosen": -455.0892333984375,
      "logps/rejected": -487.9530334472656,
      "loss": 0.2909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8235471248626709,
      "rewards/margins": 1.7777531147003174,
      "rewards/rejected": -0.9542058706283569,
      "step": 6122
    },
    {
      "epoch": 0.8012431504048417,
      "grad_norm": 6.202945232391357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00554247573018074,
      "logits/rejected": 0.13354411721229553,
      "logps/chosen": -398.1099548339844,
      "logps/rejected": -525.731689453125,
      "loss": 0.4036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.46945875883102417,
      "rewards/margins": 1.4988818168640137,
      "rewards/rejected": -1.0294231176376343,
      "step": 6123
    },
    {
      "epoch": 0.8013740083421935,
      "grad_norm": 7.589272499084473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06405223906040192,
      "logits/rejected": -0.3853917121887207,
      "logps/chosen": -398.3427429199219,
      "logps/rejected": -306.67950439453125,
      "loss": 0.4708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.024322509765625,
      "rewards/margins": 1.017137050628662,
      "rewards/rejected": -1.0414594411849976,
      "step": 6124
    },
    {
      "epoch": 0.8015048662795453,
      "grad_norm": 8.951419830322266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11144735664129257,
      "logits/rejected": -0.0598924458026886,
      "logps/chosen": -647.2940063476562,
      "logps/rejected": -494.64935302734375,
      "loss": 0.4775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10525226593017578,
      "rewards/margins": 1.0407435894012451,
      "rewards/rejected": -0.9354912042617798,
      "step": 6125
    },
    {
      "epoch": 0.801635724216897,
      "grad_norm": 7.99796199798584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14517799019813538,
      "logits/rejected": -0.363991379737854,
      "logps/chosen": -583.2157592773438,
      "logps/rejected": -522.5667724609375,
      "loss": 0.3817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2256435751914978,
      "rewards/margins": 1.169243335723877,
      "rewards/rejected": -1.3948867321014404,
      "step": 6126
    },
    {
      "epoch": 0.8017665821542488,
      "grad_norm": 7.485064506530762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006756842136383057,
      "logits/rejected": 0.04043281078338623,
      "logps/chosen": -447.1085510253906,
      "logps/rejected": -482.0423889160156,
      "loss": 0.531,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8207384347915649,
      "rewards/margins": 0.6544914245605469,
      "rewards/rejected": -1.4752297401428223,
      "step": 6127
    },
    {
      "epoch": 0.8018974400916006,
      "grad_norm": 9.778067588806152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08812219649553299,
      "logits/rejected": -0.11203426122665405,
      "logps/chosen": -703.0217895507812,
      "logps/rejected": -745.1485595703125,
      "loss": 0.6775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2750530242919922,
      "rewards/margins": 1.3190661668777466,
      "rewards/rejected": -1.044013261795044,
      "step": 6128
    },
    {
      "epoch": 0.8020282980289524,
      "grad_norm": 6.441987037658691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08373698592185974,
      "logits/rejected": -0.05694106966257095,
      "logps/chosen": -487.7393798828125,
      "logps/rejected": -435.0394287109375,
      "loss": 0.3642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.2624629735946655,
      "rewards/margins": 1.1139665842056274,
      "rewards/rejected": 0.14849643409252167,
      "step": 6129
    },
    {
      "epoch": 0.802159155966304,
      "grad_norm": 8.529573440551758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27650535106658936,
      "logits/rejected": -0.00339520163834095,
      "logps/chosen": -517.5252685546875,
      "logps/rejected": -329.85736083984375,
      "loss": 0.7687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3849193751811981,
      "rewards/margins": 0.20622920989990234,
      "rewards/rejected": -0.5911485552787781,
      "step": 6130
    },
    {
      "epoch": 0.8022900139036558,
      "grad_norm": 10.41301155090332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.033351123332977295,
      "logits/rejected": 0.1727127730846405,
      "logps/chosen": -633.015869140625,
      "logps/rejected": -412.9287414550781,
      "loss": 0.6041,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3658030927181244,
      "rewards/margins": 0.4304755926132202,
      "rewards/rejected": -0.7962787747383118,
      "step": 6131
    },
    {
      "epoch": 0.8024208718410076,
      "grad_norm": 7.787684917449951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21562159061431885,
      "logits/rejected": -0.026408875361084938,
      "logps/chosen": -393.69921875,
      "logps/rejected": -403.46142578125,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8766392469406128,
      "rewards/margins": 0.23532944917678833,
      "rewards/rejected": -1.1119686365127563,
      "step": 6132
    },
    {
      "epoch": 0.8025517297783594,
      "grad_norm": 9.057389259338379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16692611575126648,
      "logits/rejected": 0.10168322920799255,
      "logps/chosen": -359.51324462890625,
      "logps/rejected": -452.2855529785156,
      "loss": 0.6463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6421831846237183,
      "rewards/margins": 0.4544822573661804,
      "rewards/rejected": -1.096665382385254,
      "step": 6133
    },
    {
      "epoch": 0.8026825877157111,
      "grad_norm": 9.575346946716309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08586926758289337,
      "logits/rejected": -0.13744455575942993,
      "logps/chosen": -469.25006103515625,
      "logps/rejected": -475.07470703125,
      "loss": 0.5339,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.21791669726371765,
      "rewards/margins": 0.9468515515327454,
      "rewards/rejected": -0.7289349436759949,
      "step": 6134
    },
    {
      "epoch": 0.8028134456530629,
      "grad_norm": 5.724068641662598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4766574800014496,
      "logits/rejected": 0.3552775979042053,
      "logps/chosen": -447.4329528808594,
      "logps/rejected": -343.9991455078125,
      "loss": 0.4504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2760128080844879,
      "rewards/margins": 0.7314678430557251,
      "rewards/rejected": -0.4554550349712372,
      "step": 6135
    },
    {
      "epoch": 0.8029443035904147,
      "grad_norm": 8.534533500671387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025218021124601364,
      "logits/rejected": 0.2833135426044464,
      "logps/chosen": -482.129638671875,
      "logps/rejected": -353.21697998046875,
      "loss": 0.5777,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6438856720924377,
      "rewards/margins": 0.41736337542533875,
      "rewards/rejected": -1.0612488985061646,
      "step": 6136
    },
    {
      "epoch": 0.8030751615277664,
      "grad_norm": 5.713720321655273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08884280174970627,
      "logits/rejected": -0.10682341456413269,
      "logps/chosen": -272.384033203125,
      "logps/rejected": -250.36268615722656,
      "loss": 0.4428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5664028525352478,
      "rewards/margins": 1.0377159118652344,
      "rewards/rejected": -1.6041184663772583,
      "step": 6137
    },
    {
      "epoch": 0.8032060194651182,
      "grad_norm": 6.6833391189575195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023054402321577072,
      "logits/rejected": 0.14796476066112518,
      "logps/chosen": -332.7474060058594,
      "logps/rejected": -450.3458251953125,
      "loss": 0.4422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.30473387241363525,
      "rewards/margins": 1.1047428846359253,
      "rewards/rejected": -0.80000901222229,
      "step": 6138
    },
    {
      "epoch": 0.80333687740247,
      "grad_norm": 6.91253137588501,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18006953597068787,
      "logits/rejected": -0.049973614513874054,
      "logps/chosen": -453.10504150390625,
      "logps/rejected": -381.59173583984375,
      "loss": 0.3872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0946727842092514,
      "rewards/margins": 1.2187286615371704,
      "rewards/rejected": -1.313401460647583,
      "step": 6139
    },
    {
      "epoch": 0.8034677353398217,
      "grad_norm": 8.431233406066895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17651012539863586,
      "logits/rejected": -0.13946273922920227,
      "logps/chosen": -769.4700317382812,
      "logps/rejected": -468.888427734375,
      "loss": 0.4452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2808416485786438,
      "rewards/margins": 0.9079277515411377,
      "rewards/rejected": -1.1887694597244263,
      "step": 6140
    },
    {
      "epoch": 0.8035985932771734,
      "grad_norm": 7.591647624969482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11056911945343018,
      "logits/rejected": -0.13492898643016815,
      "logps/chosen": -576.6441040039062,
      "logps/rejected": -414.5979309082031,
      "loss": 0.4831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2883892357349396,
      "rewards/margins": 0.7142770886421204,
      "rewards/rejected": -0.4258878827095032,
      "step": 6141
    },
    {
      "epoch": 0.8037294512145252,
      "grad_norm": 7.158050537109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28700026869773865,
      "logits/rejected": -0.2619401216506958,
      "logps/chosen": -342.8830871582031,
      "logps/rejected": -359.3452453613281,
      "loss": 0.7424,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12578964233398438,
      "rewards/margins": 0.1758071929216385,
      "rewards/rejected": -0.30159682035446167,
      "step": 6142
    },
    {
      "epoch": 0.803860309151877,
      "grad_norm": 7.8304924964904785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008869484066963196,
      "logits/rejected": 0.02002948895096779,
      "logps/chosen": -462.579345703125,
      "logps/rejected": -514.7684936523438,
      "loss": 0.4649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07448083162307739,
      "rewards/margins": 1.3489868640899658,
      "rewards/rejected": -1.2745059728622437,
      "step": 6143
    },
    {
      "epoch": 0.8039911670892288,
      "grad_norm": 8.915619850158691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09664830565452576,
      "logits/rejected": 0.1778845340013504,
      "logps/chosen": -437.1336364746094,
      "logps/rejected": -404.69537353515625,
      "loss": 0.8133,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.17696742713451385,
      "rewards/margins": 0.24553032219409943,
      "rewards/rejected": -0.06856289505958557,
      "step": 6144
    },
    {
      "epoch": 0.8041220250265805,
      "grad_norm": 7.0653157234191895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10587754845619202,
      "logits/rejected": -0.020367972552776337,
      "logps/chosen": -487.4533386230469,
      "logps/rejected": -419.85260009765625,
      "loss": 0.4175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5211265683174133,
      "rewards/margins": 1.089951515197754,
      "rewards/rejected": -1.611078143119812,
      "step": 6145
    },
    {
      "epoch": 0.8042528829639323,
      "grad_norm": 8.450518608093262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03800436109304428,
      "logits/rejected": -0.23250870406627655,
      "logps/chosen": -472.90130615234375,
      "logps/rejected": -440.70208740234375,
      "loss": 0.5261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7933072447776794,
      "rewards/margins": 0.8553923964500427,
      "rewards/rejected": -1.6486997604370117,
      "step": 6146
    },
    {
      "epoch": 0.8043837409012841,
      "grad_norm": 8.212299346923828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05558210238814354,
      "logits/rejected": -0.21890325844287872,
      "logps/chosen": -446.0774230957031,
      "logps/rejected": -633.17431640625,
      "loss": 0.4747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19151955842971802,
      "rewards/margins": 0.8515123724937439,
      "rewards/rejected": -1.043031930923462,
      "step": 6147
    },
    {
      "epoch": 0.8045145988386359,
      "grad_norm": 5.572644233703613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04555845260620117,
      "logits/rejected": -0.27063649892807007,
      "logps/chosen": -337.4278869628906,
      "logps/rejected": -353.1708068847656,
      "loss": 0.4667,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29718148708343506,
      "rewards/margins": 0.8004717826843262,
      "rewards/rejected": -1.0976531505584717,
      "step": 6148
    },
    {
      "epoch": 0.8046454567759875,
      "grad_norm": 11.959218978881836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.032057277858257294,
      "logits/rejected": -0.08186804503202438,
      "logps/chosen": -624.7163696289062,
      "logps/rejected": -479.1947021484375,
      "loss": 1.0199,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5303220748901367,
      "rewards/margins": -0.45144712924957275,
      "rewards/rejected": -0.07887496054172516,
      "step": 6149
    },
    {
      "epoch": 0.8047763147133393,
      "grad_norm": 10.067740440368652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09965713322162628,
      "logits/rejected": -0.03260523080825806,
      "logps/chosen": -532.0849609375,
      "logps/rejected": -374.4482116699219,
      "loss": 0.7498,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6980503797531128,
      "rewards/margins": 0.2134767472743988,
      "rewards/rejected": -0.9115270376205444,
      "step": 6150
    },
    {
      "epoch": 0.8049071726506911,
      "grad_norm": 8.046788215637207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1324843466281891,
      "logits/rejected": -0.25833263993263245,
      "logps/chosen": -619.6273193359375,
      "logps/rejected": -615.24609375,
      "loss": 0.6623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2619243562221527,
      "rewards/margins": 0.7683746218681335,
      "rewards/rejected": -1.0302989482879639,
      "step": 6151
    },
    {
      "epoch": 0.8050380305880429,
      "grad_norm": 10.839067459106445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13155719637870789,
      "logits/rejected": -0.38052618503570557,
      "logps/chosen": -515.5321044921875,
      "logps/rejected": -461.0730895996094,
      "loss": 0.6528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2672019600868225,
      "rewards/margins": 0.5455693006515503,
      "rewards/rejected": -0.812771201133728,
      "step": 6152
    },
    {
      "epoch": 0.8051688885253946,
      "grad_norm": 6.270903587341309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0066132089123129845,
      "logits/rejected": -0.06472907960414886,
      "logps/chosen": -473.8494873046875,
      "logps/rejected": -446.6221618652344,
      "loss": 0.442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6460550427436829,
      "rewards/margins": 1.3359185457229614,
      "rewards/rejected": -0.6898635625839233,
      "step": 6153
    },
    {
      "epoch": 0.8052997464627464,
      "grad_norm": 6.384034633636475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04501722380518913,
      "logits/rejected": -0.04384307190775871,
      "logps/chosen": -341.7120056152344,
      "logps/rejected": -509.5606994628906,
      "loss": 0.4327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.020579546689987183,
      "rewards/margins": 1.5410182476043701,
      "rewards/rejected": -1.5615978240966797,
      "step": 6154
    },
    {
      "epoch": 0.8054306044000982,
      "grad_norm": 7.511656761169434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2197074294090271,
      "logits/rejected": -0.21633955836296082,
      "logps/chosen": -342.3523864746094,
      "logps/rejected": -356.3497314453125,
      "loss": 0.7498,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7037903070449829,
      "rewards/margins": 0.2296404242515564,
      "rewards/rejected": -0.9334307909011841,
      "step": 6155
    },
    {
      "epoch": 0.8055614623374499,
      "grad_norm": 6.3697028160095215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008588007651269436,
      "logits/rejected": -0.14007572829723358,
      "logps/chosen": -429.96429443359375,
      "logps/rejected": -311.5149230957031,
      "loss": 0.6091,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09483679383993149,
      "rewards/margins": 0.6283249258995056,
      "rewards/rejected": -0.5334880948066711,
      "step": 6156
    },
    {
      "epoch": 0.8056923202748016,
      "grad_norm": 7.588779926300049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23305481672286987,
      "logits/rejected": -0.10652349144220352,
      "logps/chosen": -398.8891906738281,
      "logps/rejected": -359.3686218261719,
      "loss": 0.4356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5494522452354431,
      "rewards/margins": 0.7624151706695557,
      "rewards/rejected": -1.311867356300354,
      "step": 6157
    },
    {
      "epoch": 0.8058231782121534,
      "grad_norm": 7.461601734161377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0307137593626976,
      "logits/rejected": -0.11507369577884674,
      "logps/chosen": -580.1257934570312,
      "logps/rejected": -530.4157104492188,
      "loss": 0.5643,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2415054440498352,
      "rewards/margins": 0.3825610876083374,
      "rewards/rejected": -0.6240665316581726,
      "step": 6158
    },
    {
      "epoch": 0.8059540361495052,
      "grad_norm": 7.51160192489624,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00284781027585268,
      "logits/rejected": -0.11958447098731995,
      "logps/chosen": -418.88983154296875,
      "logps/rejected": -441.2101135253906,
      "loss": 0.5135,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6219741702079773,
      "rewards/margins": 0.6041643023490906,
      "rewards/rejected": 0.017809880897402763,
      "step": 6159
    },
    {
      "epoch": 0.8060848940868569,
      "grad_norm": 6.305579662322998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27654317021369934,
      "logits/rejected": 0.03535313159227371,
      "logps/chosen": -506.70050048828125,
      "logps/rejected": -415.35418701171875,
      "loss": 0.5378,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3114982843399048,
      "rewards/margins": 1.1186169385910034,
      "rewards/rejected": -1.4301152229309082,
      "step": 6160
    },
    {
      "epoch": 0.8062157520242087,
      "grad_norm": 4.8398332595825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07719221711158752,
      "logits/rejected": -0.1460675299167633,
      "logps/chosen": -433.179931640625,
      "logps/rejected": -421.40240478515625,
      "loss": 0.3177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1470564752817154,
      "rewards/margins": 1.4111509323120117,
      "rewards/rejected": -1.5582072734832764,
      "step": 6161
    },
    {
      "epoch": 0.8063466099615605,
      "grad_norm": 2.2723798751831055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17400653660297394,
      "logits/rejected": 0.02125432714819908,
      "logps/chosen": -333.77734375,
      "logps/rejected": -389.2215881347656,
      "loss": 0.1111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.459286093711853,
      "rewards/margins": 2.366426467895508,
      "rewards/rejected": -1.9071404933929443,
      "step": 6162
    },
    {
      "epoch": 0.8064774678989123,
      "grad_norm": 7.671090126037598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23208558559417725,
      "logits/rejected": -0.1309525966644287,
      "logps/chosen": -524.43896484375,
      "logps/rejected": -410.64910888671875,
      "loss": 0.5514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5288723111152649,
      "rewards/margins": 0.675762951374054,
      "rewards/rejected": -1.2046352624893188,
      "step": 6163
    },
    {
      "epoch": 0.806608325836264,
      "grad_norm": 5.372653007507324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13839486241340637,
      "logits/rejected": -0.2953275740146637,
      "logps/chosen": -340.377685546875,
      "logps/rejected": -247.57041931152344,
      "loss": 0.5478,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017599118873476982,
      "rewards/margins": 0.589411199092865,
      "rewards/rejected": -0.571812093257904,
      "step": 6164
    },
    {
      "epoch": 0.8067391837736158,
      "grad_norm": 7.2957444190979,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11613660305738449,
      "logits/rejected": -0.1671690195798874,
      "logps/chosen": -406.3077392578125,
      "logps/rejected": -383.88128662109375,
      "loss": 0.4046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44707033038139343,
      "rewards/margins": 1.0582327842712402,
      "rewards/rejected": -1.5053030252456665,
      "step": 6165
    },
    {
      "epoch": 0.8068700417109675,
      "grad_norm": 7.900999546051025,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03162931278347969,
      "logits/rejected": 0.053504738956689835,
      "logps/chosen": -582.1074829101562,
      "logps/rejected": -430.52447509765625,
      "loss": 0.4412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1903570145368576,
      "rewards/margins": 0.9153972864151001,
      "rewards/rejected": -1.1057542562484741,
      "step": 6166
    },
    {
      "epoch": 0.8070008996483193,
      "grad_norm": 7.776062965393066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03878437727689743,
      "logits/rejected": 0.010435018688440323,
      "logps/chosen": -381.6441650390625,
      "logps/rejected": -473.2181091308594,
      "loss": 0.4357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15997923910617828,
      "rewards/margins": 1.565273642539978,
      "rewards/rejected": -1.7252528667449951,
      "step": 6167
    },
    {
      "epoch": 0.807131757585671,
      "grad_norm": 8.196686744689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028555627912282944,
      "logits/rejected": -0.23882901668548584,
      "logps/chosen": -507.5229797363281,
      "logps/rejected": -339.88409423828125,
      "loss": 0.5061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1850578337907791,
      "rewards/margins": 0.8745973110198975,
      "rewards/rejected": -0.6895395517349243,
      "step": 6168
    },
    {
      "epoch": 0.8072626155230228,
      "grad_norm": 7.583499908447266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14219880104064941,
      "logits/rejected": 0.10913193225860596,
      "logps/chosen": -502.9834899902344,
      "logps/rejected": -478.7446594238281,
      "loss": 0.4798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20630985498428345,
      "rewards/margins": 0.8925020694732666,
      "rewards/rejected": -1.0988118648529053,
      "step": 6169
    },
    {
      "epoch": 0.8073934734603746,
      "grad_norm": 11.068490028381348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006217353045940399,
      "logits/rejected": -0.16419914364814758,
      "logps/chosen": -356.3504638671875,
      "logps/rejected": -308.35955810546875,
      "loss": 0.9059,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28396183252334595,
      "rewards/margins": -0.20563794672489166,
      "rewards/rejected": -0.07832392305135727,
      "step": 6170
    },
    {
      "epoch": 0.8075243313977264,
      "grad_norm": 10.185708999633789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1532481610774994,
      "logits/rejected": -0.23568037152290344,
      "logps/chosen": -453.5607604980469,
      "logps/rejected": -726.2743530273438,
      "loss": 0.5652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.40376055240631104,
      "rewards/margins": 1.0775444507598877,
      "rewards/rejected": -0.6737838983535767,
      "step": 6171
    },
    {
      "epoch": 0.8076551893350781,
      "grad_norm": 6.644502639770508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14610865712165833,
      "logits/rejected": -0.05589863657951355,
      "logps/chosen": -502.6341552734375,
      "logps/rejected": -487.4541015625,
      "loss": 0.3974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39716070890426636,
      "rewards/margins": 0.9709354639053345,
      "rewards/rejected": -1.368096113204956,
      "step": 6172
    },
    {
      "epoch": 0.8077860472724299,
      "grad_norm": 7.082613468170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11217735707759857,
      "logits/rejected": -0.23063801229000092,
      "logps/chosen": -447.136962890625,
      "logps/rejected": -554.5410766601562,
      "loss": 0.5285,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19456349313259125,
      "rewards/margins": 0.7304533123970032,
      "rewards/rejected": -0.9250168800354004,
      "step": 6173
    },
    {
      "epoch": 0.8079169052097817,
      "grad_norm": 7.664604187011719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2781778872013092,
      "logits/rejected": 0.3308180570602417,
      "logps/chosen": -467.853271484375,
      "logps/rejected": -475.02935791015625,
      "loss": 0.6031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1588878631591797,
      "rewards/margins": 0.6442772150039673,
      "rewards/rejected": -0.803165078163147,
      "step": 6174
    },
    {
      "epoch": 0.8080477631471334,
      "grad_norm": 5.003294944763184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017741214483976364,
      "logits/rejected": -0.14846055209636688,
      "logps/chosen": -473.1891784667969,
      "logps/rejected": -443.7294921875,
      "loss": 0.3337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2897211015224457,
      "rewards/margins": 1.1887879371643066,
      "rewards/rejected": -1.4785089492797852,
      "step": 6175
    },
    {
      "epoch": 0.8081786210844851,
      "grad_norm": 8.854083061218262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0802706778049469,
      "logits/rejected": 0.05585087463259697,
      "logps/chosen": -387.1759033203125,
      "logps/rejected": -393.23370361328125,
      "loss": 0.7037,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.859639585018158,
      "rewards/margins": 0.3207296133041382,
      "rewards/rejected": -1.1803691387176514,
      "step": 6176
    },
    {
      "epoch": 0.8083094790218369,
      "grad_norm": 7.7259697914123535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044926442205905914,
      "logits/rejected": -0.22195231914520264,
      "logps/chosen": -336.78936767578125,
      "logps/rejected": -368.12384033203125,
      "loss": 0.5864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.36982518434524536,
      "rewards/margins": 0.44484806060791016,
      "rewards/rejected": -0.8146732449531555,
      "step": 6177
    },
    {
      "epoch": 0.8084403369591887,
      "grad_norm": 3.5004191398620605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1179313138127327,
      "logits/rejected": 0.10067202895879745,
      "logps/chosen": -501.8753356933594,
      "logps/rejected": -309.93817138671875,
      "loss": 0.3158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7170364260673523,
      "rewards/margins": 1.4790139198303223,
      "rewards/rejected": -0.7619774341583252,
      "step": 6178
    },
    {
      "epoch": 0.8085711948965404,
      "grad_norm": 6.608018398284912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1078784316778183,
      "logits/rejected": 0.15303124487400055,
      "logps/chosen": -530.1780395507812,
      "logps/rejected": -673.01416015625,
      "loss": 0.3898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2586151361465454,
      "rewards/margins": 1.3292264938354492,
      "rewards/rejected": -1.587841510772705,
      "step": 6179
    },
    {
      "epoch": 0.8087020528338922,
      "grad_norm": 8.013618469238281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11863519996404648,
      "logits/rejected": -0.15569572150707245,
      "logps/chosen": -392.205078125,
      "logps/rejected": -323.6466979980469,
      "loss": 0.6129,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.568194568157196,
      "rewards/margins": 0.4346906840801239,
      "rewards/rejected": -1.002885341644287,
      "step": 6180
    },
    {
      "epoch": 0.808832910771244,
      "grad_norm": 4.89748477935791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10423228144645691,
      "logits/rejected": -0.05080432817339897,
      "logps/chosen": -522.3642578125,
      "logps/rejected": -515.4188232421875,
      "loss": 0.3373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30010825395584106,
      "rewards/margins": 1.5478414297103882,
      "rewards/rejected": -1.847949504852295,
      "step": 6181
    },
    {
      "epoch": 0.8089637687085958,
      "grad_norm": 7.775894641876221,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1179753988981247,
      "logits/rejected": -0.08880023658275604,
      "logps/chosen": -244.17340087890625,
      "logps/rejected": -311.7882080078125,
      "loss": 0.5389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.039603427052497864,
      "rewards/margins": 0.7002450823783875,
      "rewards/rejected": -0.6606417298316956,
      "step": 6182
    },
    {
      "epoch": 0.8090946266459474,
      "grad_norm": 13.98147201538086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035551246255636215,
      "logits/rejected": -0.22935201227664948,
      "logps/chosen": -396.9501037597656,
      "logps/rejected": -369.7340393066406,
      "loss": 1.3874,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.145758867263794,
      "rewards/margins": -0.9493227005004883,
      "rewards/rejected": -0.19643613696098328,
      "step": 6183
    },
    {
      "epoch": 0.8092254845832992,
      "grad_norm": 6.239151954650879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47986799478530884,
      "logits/rejected": 0.4151700437068939,
      "logps/chosen": -508.14544677734375,
      "logps/rejected": -587.2987060546875,
      "loss": 0.3167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20914040505886078,
      "rewards/margins": 1.6744145154953003,
      "rewards/rejected": -1.8835546970367432,
      "step": 6184
    },
    {
      "epoch": 0.809356342520651,
      "grad_norm": 5.738187313079834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25885093212127686,
      "logits/rejected": 0.061917323619127274,
      "logps/chosen": -327.328369140625,
      "logps/rejected": -351.0191650390625,
      "loss": 0.3414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1355886459350586,
      "rewards/margins": 1.2673836946487427,
      "rewards/rejected": -1.1317949295043945,
      "step": 6185
    },
    {
      "epoch": 0.8094872004580028,
      "grad_norm": 6.104333400726318,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15869677066802979,
      "logits/rejected": -0.18752315640449524,
      "logps/chosen": -583.5074462890625,
      "logps/rejected": -511.44622802734375,
      "loss": 0.4767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9255248308181763,
      "rewards/margins": 0.6956636905670166,
      "rewards/rejected": 0.2298612743616104,
      "step": 6186
    },
    {
      "epoch": 0.8096180583953545,
      "grad_norm": 8.547439575195312,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17998145520687103,
      "logits/rejected": 0.2322385609149933,
      "logps/chosen": -396.75421142578125,
      "logps/rejected": -402.703125,
      "loss": 0.7642,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14063425362110138,
      "rewards/margins": 0.2590458393096924,
      "rewards/rejected": -0.39968013763427734,
      "step": 6187
    },
    {
      "epoch": 0.8097489163327063,
      "grad_norm": 6.532267093658447,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05977587029337883,
      "logits/rejected": 0.032165687531232834,
      "logps/chosen": -293.91278076171875,
      "logps/rejected": -299.8616027832031,
      "loss": 0.3364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5459840893745422,
      "rewards/margins": 1.4894448518753052,
      "rewards/rejected": -2.035428762435913,
      "step": 6188
    },
    {
      "epoch": 0.8098797742700581,
      "grad_norm": 6.375497341156006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004418365657329559,
      "logits/rejected": -0.09665033221244812,
      "logps/chosen": -501.75738525390625,
      "logps/rejected": -436.11370849609375,
      "loss": 0.3875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3475528657436371,
      "rewards/margins": 0.9307810068130493,
      "rewards/rejected": -0.5832281112670898,
      "step": 6189
    },
    {
      "epoch": 0.8100106322074099,
      "grad_norm": 4.8319091796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1476336121559143,
      "logits/rejected": -0.2961917817592621,
      "logps/chosen": -699.077880859375,
      "logps/rejected": -449.92376708984375,
      "loss": 0.2111,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37758520245552063,
      "rewards/margins": 2.1771156787872314,
      "rewards/rejected": -1.7995307445526123,
      "step": 6190
    },
    {
      "epoch": 0.8101414901447616,
      "grad_norm": 8.234923362731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04786160960793495,
      "logits/rejected": -0.09052304923534393,
      "logps/chosen": -625.03857421875,
      "logps/rejected": -504.40728759765625,
      "loss": 0.5891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31043511629104614,
      "rewards/margins": 0.4282954931259155,
      "rewards/rejected": -0.7387306094169617,
      "step": 6191
    },
    {
      "epoch": 0.8102723480821133,
      "grad_norm": 6.025783061981201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5313307642936707,
      "logits/rejected": -0.25905513763427734,
      "logps/chosen": -357.3464050292969,
      "logps/rejected": -524.6739501953125,
      "loss": 0.3387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4634738862514496,
      "rewards/margins": 1.1139739751815796,
      "rewards/rejected": -1.5774478912353516,
      "step": 6192
    },
    {
      "epoch": 0.8104032060194651,
      "grad_norm": 5.977348804473877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09259185194969177,
      "logits/rejected": 0.2009120136499405,
      "logps/chosen": -458.21331787109375,
      "logps/rejected": -462.1827392578125,
      "loss": 0.5381,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8412355184555054,
      "rewards/margins": 0.8463122844696045,
      "rewards/rejected": -1.6875478029251099,
      "step": 6193
    },
    {
      "epoch": 0.8105340639568169,
      "grad_norm": 11.524065971374512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08234912902116776,
      "logits/rejected": -0.17300623655319214,
      "logps/chosen": -540.7627563476562,
      "logps/rejected": -590.3538818359375,
      "loss": 0.8027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7290528416633606,
      "rewards/margins": 0.09668943285942078,
      "rewards/rejected": -0.8257423639297485,
      "step": 6194
    },
    {
      "epoch": 0.8106649218941686,
      "grad_norm": 5.727067470550537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2795459032058716,
      "logits/rejected": -0.1333911120891571,
      "logps/chosen": -414.7002868652344,
      "logps/rejected": -450.81341552734375,
      "loss": 0.427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14684927463531494,
      "rewards/margins": 1.5480583906173706,
      "rewards/rejected": -1.4012091159820557,
      "step": 6195
    },
    {
      "epoch": 0.8107957798315204,
      "grad_norm": 4.619360446929932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07112779468297958,
      "logits/rejected": -0.20410338044166565,
      "logps/chosen": -470.01922607421875,
      "logps/rejected": -424.8250427246094,
      "loss": 0.3436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2643245756626129,
      "rewards/margins": 1.1632261276245117,
      "rewards/rejected": -1.4275507926940918,
      "step": 6196
    },
    {
      "epoch": 0.8109266377688722,
      "grad_norm": 8.669483184814453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08724243938922882,
      "logits/rejected": -0.2845610976219177,
      "logps/chosen": -552.885986328125,
      "logps/rejected": -400.1076965332031,
      "loss": 0.4898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0664510652422905,
      "rewards/margins": 1.12000572681427,
      "rewards/rejected": -1.1864567995071411,
      "step": 6197
    },
    {
      "epoch": 0.811057495706224,
      "grad_norm": 5.166367053985596,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1166340708732605,
      "logits/rejected": 0.12926115095615387,
      "logps/chosen": -397.1042785644531,
      "logps/rejected": -493.6290283203125,
      "loss": 0.2938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14387553930282593,
      "rewards/margins": 1.8685230016708374,
      "rewards/rejected": -2.0123984813690186,
      "step": 6198
    },
    {
      "epoch": 0.8111883536435757,
      "grad_norm": 4.096929550170898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05777914822101593,
      "logits/rejected": -0.05711698904633522,
      "logps/chosen": -452.42877197265625,
      "logps/rejected": -513.3928833007812,
      "loss": 0.2418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.546901285648346,
      "rewards/margins": 2.127880096435547,
      "rewards/rejected": -1.5809789896011353,
      "step": 6199
    },
    {
      "epoch": 0.8113192115809275,
      "grad_norm": 6.996924877166748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04127448424696922,
      "logits/rejected": 0.07511641085147858,
      "logps/chosen": -502.24658203125,
      "logps/rejected": -551.723388671875,
      "loss": 0.4537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02428703010082245,
      "rewards/margins": 0.9623256325721741,
      "rewards/rejected": -0.9866126775741577,
      "step": 6200
    },
    {
      "epoch": 0.8114500695182792,
      "grad_norm": 6.006531715393066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12881776690483093,
      "logits/rejected": 0.19008147716522217,
      "logps/chosen": -457.6412658691406,
      "logps/rejected": -478.0318603515625,
      "loss": 0.3899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5873456597328186,
      "rewards/margins": 1.3635034561157227,
      "rewards/rejected": -1.9508490562438965,
      "step": 6201
    },
    {
      "epoch": 0.8115809274556309,
      "grad_norm": 6.773149013519287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20026172697544098,
      "logits/rejected": -0.18467465043067932,
      "logps/chosen": -398.0389099121094,
      "logps/rejected": -449.2351989746094,
      "loss": 0.6251,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7012838125228882,
      "rewards/margins": 0.3913339376449585,
      "rewards/rejected": -1.0926177501678467,
      "step": 6202
    },
    {
      "epoch": 0.8117117853929827,
      "grad_norm": 6.000072956085205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05169972404837608,
      "logits/rejected": -0.07182519137859344,
      "logps/chosen": -341.141357421875,
      "logps/rejected": -361.9170837402344,
      "loss": 0.3906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10206738114356995,
      "rewards/margins": 1.5491822957992554,
      "rewards/rejected": -1.4471150636672974,
      "step": 6203
    },
    {
      "epoch": 0.8118426433303345,
      "grad_norm": 5.634113311767578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02690296806395054,
      "logits/rejected": 0.1551869958639145,
      "logps/chosen": -525.6759033203125,
      "logps/rejected": -446.5899963378906,
      "loss": 0.2519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09237252175807953,
      "rewards/margins": 1.7263264656066895,
      "rewards/rejected": -1.6339538097381592,
      "step": 6204
    },
    {
      "epoch": 0.8119735012676863,
      "grad_norm": 16.90241241455078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16272424161434174,
      "logits/rejected": 0.15271039307117462,
      "logps/chosen": -475.1338806152344,
      "logps/rejected": -315.87066650390625,
      "loss": 0.5836,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7951433658599854,
      "rewards/margins": 0.9727462530136108,
      "rewards/rejected": -1.7678894996643066,
      "step": 6205
    },
    {
      "epoch": 0.812104359205038,
      "grad_norm": 7.82653284072876,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011310763657093048,
      "logits/rejected": -0.03446229174733162,
      "logps/chosen": -429.6619567871094,
      "logps/rejected": -492.2726745605469,
      "loss": 0.5935,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.583276629447937,
      "rewards/margins": 0.6480287313461304,
      "rewards/rejected": -1.2313053607940674,
      "step": 6206
    },
    {
      "epoch": 0.8122352171423898,
      "grad_norm": 10.707749366760254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31054019927978516,
      "logits/rejected": -0.4381905198097229,
      "logps/chosen": -357.70166015625,
      "logps/rejected": -400.41082763671875,
      "loss": 0.7176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0095959901809692,
      "rewards/margins": 0.3019859492778778,
      "rewards/rejected": -1.3115818500518799,
      "step": 6207
    },
    {
      "epoch": 0.8123660750797416,
      "grad_norm": 11.097867012023926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0021544452756643295,
      "logits/rejected": -0.03577995300292969,
      "logps/chosen": -456.65167236328125,
      "logps/rejected": -516.4016723632812,
      "loss": 0.6767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1861966848373413,
      "rewards/margins": 0.5043489933013916,
      "rewards/rejected": -0.6905456781387329,
      "step": 6208
    },
    {
      "epoch": 0.8124969330170934,
      "grad_norm": 8.678732872009277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.281197726726532,
      "logits/rejected": 0.22460755705833435,
      "logps/chosen": -350.4656066894531,
      "logps/rejected": -406.796142578125,
      "loss": 0.8776,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6256636381149292,
      "rewards/margins": 0.002203509211540222,
      "rewards/rejected": -0.6278671622276306,
      "step": 6209
    },
    {
      "epoch": 0.812627790954445,
      "grad_norm": 2.9518866539001465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11977237462997437,
      "logits/rejected": -0.3622247874736786,
      "logps/chosen": -685.6779174804688,
      "logps/rejected": -566.1923217773438,
      "loss": 0.1403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20117683708667755,
      "rewards/margins": 2.375291109085083,
      "rewards/rejected": -2.1741139888763428,
      "step": 6210
    },
    {
      "epoch": 0.8127586488917968,
      "grad_norm": 7.982006072998047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2139490395784378,
      "logits/rejected": 0.08931490033864975,
      "logps/chosen": -482.9425048828125,
      "logps/rejected": -391.90081787109375,
      "loss": 0.4481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2799581289291382,
      "rewards/margins": 1.6965304613113403,
      "rewards/rejected": -1.9764883518218994,
      "step": 6211
    },
    {
      "epoch": 0.8128895068291486,
      "grad_norm": 9.670489311218262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02375439926981926,
      "logits/rejected": -0.10124373435974121,
      "logps/chosen": -682.3936157226562,
      "logps/rejected": -483.02777099609375,
      "loss": 0.6064,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9167481660842896,
      "rewards/margins": 0.5319905281066895,
      "rewards/rejected": -1.4487388134002686,
      "step": 6212
    },
    {
      "epoch": 0.8130203647665004,
      "grad_norm": 9.733624458312988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28057414293289185,
      "logits/rejected": -0.00651931669563055,
      "logps/chosen": -413.2099609375,
      "logps/rejected": -564.8838500976562,
      "loss": 0.6212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.291319727897644,
      "rewards/margins": 0.61299729347229,
      "rewards/rejected": -1.9043169021606445,
      "step": 6213
    },
    {
      "epoch": 0.8131512227038521,
      "grad_norm": 8.4725923538208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02625124901533127,
      "logits/rejected": -0.2440853863954544,
      "logps/chosen": -331.9481506347656,
      "logps/rejected": -413.5338134765625,
      "loss": 0.6674,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9132446646690369,
      "rewards/margins": 0.6586739420890808,
      "rewards/rejected": -1.5719186067581177,
      "step": 6214
    },
    {
      "epoch": 0.8132820806412039,
      "grad_norm": 4.862232208251953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2571679353713989,
      "logits/rejected": -0.20572954416275024,
      "logps/chosen": -337.96746826171875,
      "logps/rejected": -387.8133239746094,
      "loss": 0.3692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.011670693755149841,
      "rewards/margins": 0.9983859062194824,
      "rewards/rejected": -1.0100566148757935,
      "step": 6215
    },
    {
      "epoch": 0.8134129385785557,
      "grad_norm": 4.860593318939209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10720860958099365,
      "logits/rejected": 0.18560180068016052,
      "logps/chosen": -531.9893798828125,
      "logps/rejected": -452.5379943847656,
      "loss": 0.3444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21170808374881744,
      "rewards/margins": 1.5216320753097534,
      "rewards/rejected": -1.7333401441574097,
      "step": 6216
    },
    {
      "epoch": 0.8135437965159075,
      "grad_norm": 8.5909423828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13028210401535034,
      "logits/rejected": 0.08449845016002655,
      "logps/chosen": -382.81304931640625,
      "logps/rejected": -314.484130859375,
      "loss": 0.5523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0021164417266846,
      "rewards/margins": 0.630867600440979,
      "rewards/rejected": -1.632983922958374,
      "step": 6217
    },
    {
      "epoch": 0.8136746544532591,
      "grad_norm": 12.250006675720215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1470748484134674,
      "logits/rejected": 0.20986585319042206,
      "logps/chosen": -416.56842041015625,
      "logps/rejected": -431.8760681152344,
      "loss": 0.7801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11318433284759521,
      "rewards/margins": 0.9937867522239685,
      "rewards/rejected": -1.1069711446762085,
      "step": 6218
    },
    {
      "epoch": 0.8138055123906109,
      "grad_norm": 5.939116954803467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014212086796760559,
      "logits/rejected": -0.17681342363357544,
      "logps/chosen": -597.6101684570312,
      "logps/rejected": -395.202880859375,
      "loss": 0.3964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06848640739917755,
      "rewards/margins": 0.9705591797828674,
      "rewards/rejected": -1.0390455722808838,
      "step": 6219
    },
    {
      "epoch": 0.8139363703279627,
      "grad_norm": 5.668907165527344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3356136381626129,
      "logits/rejected": -0.21531151235103607,
      "logps/chosen": -424.8271789550781,
      "logps/rejected": -414.635009765625,
      "loss": 0.4367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12335719913244247,
      "rewards/margins": 1.2499476671218872,
      "rewards/rejected": -1.3733049631118774,
      "step": 6220
    },
    {
      "epoch": 0.8140672282653144,
      "grad_norm": 8.062246322631836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17149800062179565,
      "logits/rejected": -0.049208082258701324,
      "logps/chosen": -579.073974609375,
      "logps/rejected": -393.4719543457031,
      "loss": 0.5833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.38291609287261963,
      "rewards/margins": 0.4438199996948242,
      "rewards/rejected": -0.06090393662452698,
      "step": 6221
    },
    {
      "epoch": 0.8141980862026662,
      "grad_norm": 7.777126789093018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14421145617961884,
      "logits/rejected": -0.071357861161232,
      "logps/chosen": -369.3804016113281,
      "logps/rejected": -416.0185546875,
      "loss": 0.5318,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17907586693763733,
      "rewards/margins": 0.7859897613525391,
      "rewards/rejected": -0.965065598487854,
      "step": 6222
    },
    {
      "epoch": 0.814328944140018,
      "grad_norm": 7.715551853179932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18236319720745087,
      "logits/rejected": -0.05936586484313011,
      "logps/chosen": -456.7231750488281,
      "logps/rejected": -432.9893493652344,
      "loss": 0.4577,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13447076082229614,
      "rewards/margins": 0.7222846746444702,
      "rewards/rejected": -0.5878139138221741,
      "step": 6223
    },
    {
      "epoch": 0.8144598020773698,
      "grad_norm": 3.6720635890960693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45800766348838806,
      "logits/rejected": -0.33670714497566223,
      "logps/chosen": -538.9467163085938,
      "logps/rejected": -428.8689880371094,
      "loss": 0.2641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09847298264503479,
      "rewards/margins": 1.7444920539855957,
      "rewards/rejected": -1.6460189819335938,
      "step": 6224
    },
    {
      "epoch": 0.8145906600147215,
      "grad_norm": 6.326622486114502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2686009407043457,
      "logits/rejected": -0.14132246375083923,
      "logps/chosen": -351.27130126953125,
      "logps/rejected": -404.8138427734375,
      "loss": 0.327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8163934946060181,
      "rewards/margins": 1.8326166868209839,
      "rewards/rejected": -2.649009943008423,
      "step": 6225
    },
    {
      "epoch": 0.8147215179520733,
      "grad_norm": 7.963384628295898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06549263745546341,
      "logits/rejected": -0.06645340472459793,
      "logps/chosen": -514.132568359375,
      "logps/rejected": -570.171630859375,
      "loss": 0.5776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6429312229156494,
      "rewards/margins": 0.4017433226108551,
      "rewards/rejected": -1.0446743965148926,
      "step": 6226
    },
    {
      "epoch": 0.814852375889425,
      "grad_norm": 5.153400897979736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26231449842453003,
      "logits/rejected": 0.34071606397628784,
      "logps/chosen": -425.98785400390625,
      "logps/rejected": -441.26666259765625,
      "loss": 0.4594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27937909960746765,
      "rewards/margins": 0.6987920999526978,
      "rewards/rejected": -0.4194130003452301,
      "step": 6227
    },
    {
      "epoch": 0.8149832338267768,
      "grad_norm": 6.328569412231445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16703137755393982,
      "logits/rejected": 0.22598324716091156,
      "logps/chosen": -365.7045593261719,
      "logps/rejected": -440.49786376953125,
      "loss": 0.3885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38203543424606323,
      "rewards/margins": 0.9288241267204285,
      "rewards/rejected": -0.5467885732650757,
      "step": 6228
    },
    {
      "epoch": 0.8151140917641285,
      "grad_norm": 4.288189888000488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1734994649887085,
      "logits/rejected": 0.2437296211719513,
      "logps/chosen": -364.9457092285156,
      "logps/rejected": -369.4041748046875,
      "loss": 0.2127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2001919001340866,
      "rewards/margins": 2.142024517059326,
      "rewards/rejected": -1.9418326616287231,
      "step": 6229
    },
    {
      "epoch": 0.8152449497014803,
      "grad_norm": 6.934292316436768,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1480758786201477,
      "logits/rejected": -0.2612943649291992,
      "logps/chosen": -502.04290771484375,
      "logps/rejected": -367.5062255859375,
      "loss": 0.5693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8847726583480835,
      "rewards/margins": 1.0254979133605957,
      "rewards/rejected": -1.9102706909179688,
      "step": 6230
    },
    {
      "epoch": 0.8153758076388321,
      "grad_norm": 8.39197063446045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011969588696956635,
      "logits/rejected": -0.14503681659698486,
      "logps/chosen": -342.73681640625,
      "logps/rejected": -568.335205078125,
      "loss": 0.5843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4180982112884521,
      "rewards/margins": 1.581636905670166,
      "rewards/rejected": -2.999734878540039,
      "step": 6231
    },
    {
      "epoch": 0.8155066655761839,
      "grad_norm": 8.071584701538086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19171014428138733,
      "logits/rejected": 0.06845186650753021,
      "logps/chosen": -528.0847778320312,
      "logps/rejected": -400.6995849609375,
      "loss": 0.5471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08297256380319595,
      "rewards/margins": 0.6109621524810791,
      "rewards/rejected": -0.6939345598220825,
      "step": 6232
    },
    {
      "epoch": 0.8156375235135356,
      "grad_norm": 6.872188091278076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049867045134305954,
      "logits/rejected": -0.1223415732383728,
      "logps/chosen": -324.64337158203125,
      "logps/rejected": -328.9737548828125,
      "loss": 0.4669,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.060833707451820374,
      "rewards/margins": 0.9123755097389221,
      "rewards/rejected": -0.9732092618942261,
      "step": 6233
    },
    {
      "epoch": 0.8157683814508874,
      "grad_norm": 9.567635536193848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034011948853731155,
      "logits/rejected": -0.05283406376838684,
      "logps/chosen": -330.93707275390625,
      "logps/rejected": -342.1748352050781,
      "loss": 0.6674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.463474303483963,
      "rewards/margins": 0.5470823645591736,
      "rewards/rejected": -1.010556697845459,
      "step": 6234
    },
    {
      "epoch": 0.8158992393882392,
      "grad_norm": 6.583640098571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03777331858873367,
      "logits/rejected": -0.04898728430271149,
      "logps/chosen": -445.4689025878906,
      "logps/rejected": -493.3137512207031,
      "loss": 0.4526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20853538811206818,
      "rewards/margins": 1.0827175378799438,
      "rewards/rejected": -0.8741822242736816,
      "step": 6235
    },
    {
      "epoch": 0.816030097325591,
      "grad_norm": 9.789799690246582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5626389980316162,
      "logits/rejected": 0.3559896647930145,
      "logps/chosen": -566.883544921875,
      "logps/rejected": -397.9572448730469,
      "loss": 0.5568,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.436387836933136,
      "rewards/margins": 1.0482033491134644,
      "rewards/rejected": -1.484591007232666,
      "step": 6236
    },
    {
      "epoch": 0.8161609552629426,
      "grad_norm": 6.064359664916992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.300005167722702,
      "logits/rejected": 0.3638405203819275,
      "logps/chosen": -463.52386474609375,
      "logps/rejected": -571.0892944335938,
      "loss": 0.4493,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.19321556389331818,
      "rewards/margins": 1.5960373878479004,
      "rewards/rejected": -1.4028217792510986,
      "step": 6237
    },
    {
      "epoch": 0.8162918132002944,
      "grad_norm": 5.428371906280518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47584012150764465,
      "logits/rejected": 0.2416604608297348,
      "logps/chosen": -380.80670166015625,
      "logps/rejected": -434.8777770996094,
      "loss": 0.3741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22988101840019226,
      "rewards/margins": 1.1419107913970947,
      "rewards/rejected": -1.3717918395996094,
      "step": 6238
    },
    {
      "epoch": 0.8164226711376462,
      "grad_norm": 2.943485736846924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0987309068441391,
      "logits/rejected": -0.10650014877319336,
      "logps/chosen": -440.87762451171875,
      "logps/rejected": -427.4561462402344,
      "loss": 0.1811,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2593855559825897,
      "rewards/margins": 2.0965988636016846,
      "rewards/rejected": -2.3559839725494385,
      "step": 6239
    },
    {
      "epoch": 0.816553529074998,
      "grad_norm": 7.054494380950928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0368533581495285,
      "logits/rejected": -0.2873549461364746,
      "logps/chosen": -492.3692932128906,
      "logps/rejected": -502.888916015625,
      "loss": 0.3701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5619024038314819,
      "rewards/margins": 1.2573535442352295,
      "rewards/rejected": -1.8192558288574219,
      "step": 6240
    },
    {
      "epoch": 0.8166843870123497,
      "grad_norm": 7.406871318817139,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08119483292102814,
      "logits/rejected": 0.04134778678417206,
      "logps/chosen": -431.5197448730469,
      "logps/rejected": -522.7048950195312,
      "loss": 0.542,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010696608573198318,
      "rewards/margins": 1.0234668254852295,
      "rewards/rejected": -1.012770175933838,
      "step": 6241
    },
    {
      "epoch": 0.8168152449497015,
      "grad_norm": 5.565514087677002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1512017399072647,
      "logits/rejected": 0.013191677629947662,
      "logps/chosen": -459.4351501464844,
      "logps/rejected": -430.04425048828125,
      "loss": 0.399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4604291021823883,
      "rewards/margins": 1.2973071336746216,
      "rewards/rejected": -1.7577362060546875,
      "step": 6242
    },
    {
      "epoch": 0.8169461028870533,
      "grad_norm": 11.079204559326172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12657254934310913,
      "logits/rejected": -0.1974238157272339,
      "logps/chosen": -433.9729309082031,
      "logps/rejected": -329.97467041015625,
      "loss": 0.8733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8181159496307373,
      "rewards/margins": 0.027726978063583374,
      "rewards/rejected": -0.8458429574966431,
      "step": 6243
    },
    {
      "epoch": 0.817076960824405,
      "grad_norm": 9.228833198547363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06555162370204926,
      "logits/rejected": -0.03702588751912117,
      "logps/chosen": -349.7947998046875,
      "logps/rejected": -361.44781494140625,
      "loss": 0.7079,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3094583749771118,
      "rewards/margins": 0.33504897356033325,
      "rewards/rejected": -0.6445074677467346,
      "step": 6244
    },
    {
      "epoch": 0.8172078187617567,
      "grad_norm": 8.032145500183105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0767274722456932,
      "logits/rejected": 0.07347305119037628,
      "logps/chosen": -405.28875732421875,
      "logps/rejected": -656.8258666992188,
      "loss": 0.524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32523077726364136,
      "rewards/margins": 0.8812936544418335,
      "rewards/rejected": -1.20652437210083,
      "step": 6245
    },
    {
      "epoch": 0.8173386766991085,
      "grad_norm": 8.09840202331543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08187343180179596,
      "logits/rejected": -0.18207670748233795,
      "logps/chosen": -354.57305908203125,
      "logps/rejected": -360.7174987792969,
      "loss": 0.7768,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3253885507583618,
      "rewards/margins": 0.18928313255310059,
      "rewards/rejected": -1.514671802520752,
      "step": 6246
    },
    {
      "epoch": 0.8174695346364603,
      "grad_norm": 6.903451919555664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06055145710706711,
      "logits/rejected": 0.1451176106929779,
      "logps/chosen": -343.361328125,
      "logps/rejected": -444.252685546875,
      "loss": 0.4189,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20419998466968536,
      "rewards/margins": 1.3797621726989746,
      "rewards/rejected": -1.583962082862854,
      "step": 6247
    },
    {
      "epoch": 0.817600392573812,
      "grad_norm": 9.497568130493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1471254825592041,
      "logits/rejected": -0.2796838581562042,
      "logps/chosen": -377.54168701171875,
      "logps/rejected": -314.2875061035156,
      "loss": 0.5885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48508337140083313,
      "rewards/margins": 1.1644320487976074,
      "rewards/rejected": -1.6495153903961182,
      "step": 6248
    },
    {
      "epoch": 0.8177312505111638,
      "grad_norm": 8.776165962219238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006832032464444637,
      "logits/rejected": -0.14080549776554108,
      "logps/chosen": -511.914306640625,
      "logps/rejected": -509.8025817871094,
      "loss": 0.6433,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2510637640953064,
      "rewards/margins": 0.5893926620483398,
      "rewards/rejected": -0.8404564261436462,
      "step": 6249
    },
    {
      "epoch": 0.8178621084485156,
      "grad_norm": 8.313874244689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13235703110694885,
      "logits/rejected": -0.08100700378417969,
      "logps/chosen": -520.1530151367188,
      "logps/rejected": -456.0389709472656,
      "loss": 0.7249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18173010647296906,
      "rewards/margins": 0.3288151025772095,
      "rewards/rejected": -0.1470850110054016,
      "step": 6250
    },
    {
      "epoch": 0.8179929663858674,
      "grad_norm": 6.498152732849121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.223514586687088,
      "logits/rejected": -0.12058701366186142,
      "logps/chosen": -545.6439208984375,
      "logps/rejected": -434.9513244628906,
      "loss": 0.4484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6752018332481384,
      "rewards/margins": 0.7522504925727844,
      "rewards/rejected": -1.4274522066116333,
      "step": 6251
    },
    {
      "epoch": 0.8181238243232191,
      "grad_norm": 7.5493927001953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17213475704193115,
      "logits/rejected": 0.13016173243522644,
      "logps/chosen": -538.8123779296875,
      "logps/rejected": -597.06201171875,
      "loss": 0.4818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0045108795166016,
      "rewards/margins": 0.6913862228393555,
      "rewards/rejected": -1.695897102355957,
      "step": 6252
    },
    {
      "epoch": 0.8182546822605709,
      "grad_norm": 6.125778675079346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051857516169548035,
      "logits/rejected": 0.0064867218025028706,
      "logps/chosen": -391.82891845703125,
      "logps/rejected": -328.1112976074219,
      "loss": 0.4531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1205793246626854,
      "rewards/margins": 0.7404022216796875,
      "rewards/rejected": -0.8609815239906311,
      "step": 6253
    },
    {
      "epoch": 0.8183855401979226,
      "grad_norm": 7.41293478012085,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10790328681468964,
      "logits/rejected": 0.16595236957073212,
      "logps/chosen": -392.06964111328125,
      "logps/rejected": -429.580078125,
      "loss": 0.5482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22609099745750427,
      "rewards/margins": 0.8773260116577148,
      "rewards/rejected": -1.1034170389175415,
      "step": 6254
    },
    {
      "epoch": 0.8185163981352744,
      "grad_norm": 7.788702011108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09264128655195236,
      "logits/rejected": 0.014566488564014435,
      "logps/chosen": -358.8758239746094,
      "logps/rejected": -399.9172058105469,
      "loss": 0.5443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2510945796966553,
      "rewards/margins": 0.46266719698905945,
      "rewards/rejected": -0.7137617468833923,
      "step": 6255
    },
    {
      "epoch": 0.8186472560726261,
      "grad_norm": 8.036397933959961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07907930761575699,
      "logits/rejected": -0.1274707019329071,
      "logps/chosen": -482.6211242675781,
      "logps/rejected": -469.334716796875,
      "loss": 0.5418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.049483101814985275,
      "rewards/margins": 0.9083936810493469,
      "rewards/rejected": -0.9578768014907837,
      "step": 6256
    },
    {
      "epoch": 0.8187781140099779,
      "grad_norm": 5.6352620124816895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03478037565946579,
      "logits/rejected": -0.059678297489881516,
      "logps/chosen": -582.8983764648438,
      "logps/rejected": -397.14166259765625,
      "loss": 0.3674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17742767930030823,
      "rewards/margins": 1.4009191989898682,
      "rewards/rejected": -1.2234915494918823,
      "step": 6257
    },
    {
      "epoch": 0.8189089719473297,
      "grad_norm": 11.770699501037598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1372535675764084,
      "logits/rejected": 0.183303564786911,
      "logps/chosen": -562.4535522460938,
      "logps/rejected": -535.1845703125,
      "loss": 0.8622,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.3832935392856598,
      "rewards/margins": 0.5088691711425781,
      "rewards/rejected": -0.12557563185691833,
      "step": 6258
    },
    {
      "epoch": 0.8190398298846815,
      "grad_norm": 6.91513729095459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19257217645645142,
      "logits/rejected": -0.17553216218948364,
      "logps/chosen": -486.17724609375,
      "logps/rejected": -521.5123901367188,
      "loss": 0.469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.965463161468506e-05,
      "rewards/margins": 0.8252534866333008,
      "rewards/rejected": -0.8252338767051697,
      "step": 6259
    },
    {
      "epoch": 0.8191706878220332,
      "grad_norm": 9.186287879943848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17782415449619293,
      "logits/rejected": -0.04481366276741028,
      "logps/chosen": -357.3207702636719,
      "logps/rejected": -407.092041015625,
      "loss": 0.6714,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7613632678985596,
      "rewards/margins": 0.4862635135650635,
      "rewards/rejected": -1.247626781463623,
      "step": 6260
    },
    {
      "epoch": 0.819301545759385,
      "grad_norm": 8.975862503051758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04483247548341751,
      "logits/rejected": 0.11802737414836884,
      "logps/chosen": -389.9907531738281,
      "logps/rejected": -362.9102478027344,
      "loss": 0.5469,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.009968563914299011,
      "rewards/margins": 0.9772787094116211,
      "rewards/rejected": -0.9872473478317261,
      "step": 6261
    },
    {
      "epoch": 0.8194324036967368,
      "grad_norm": 9.32762336730957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07514193654060364,
      "logits/rejected": 0.09234967082738876,
      "logps/chosen": -340.1234436035156,
      "logps/rejected": -465.5467834472656,
      "loss": 0.9234,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5693138241767883,
      "rewards/margins": 0.2289489507675171,
      "rewards/rejected": -0.7982627749443054,
      "step": 6262
    },
    {
      "epoch": 0.8195632616340885,
      "grad_norm": 8.111886024475098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19160334765911102,
      "logits/rejected": -0.12450997531414032,
      "logps/chosen": -452.84515380859375,
      "logps/rejected": -346.4001770019531,
      "loss": 0.4181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23122538626194,
      "rewards/margins": 1.1691272258758545,
      "rewards/rejected": -1.4003524780273438,
      "step": 6263
    },
    {
      "epoch": 0.8196941195714402,
      "grad_norm": 3.1441638469696045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2123308777809143,
      "logits/rejected": -0.39955365657806396,
      "logps/chosen": -652.083740234375,
      "logps/rejected": -552.6419677734375,
      "loss": 0.2405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.462856650352478,
      "rewards/margins": 1.752112627029419,
      "rewards/rejected": -1.2892560958862305,
      "step": 6264
    },
    {
      "epoch": 0.819824977508792,
      "grad_norm": 4.556426048278809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3696346580982208,
      "logits/rejected": -0.621440589427948,
      "logps/chosen": -340.8766784667969,
      "logps/rejected": -380.01708984375,
      "loss": 0.2791,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6251488327980042,
      "rewards/margins": 1.6966686248779297,
      "rewards/rejected": -2.32181715965271,
      "step": 6265
    },
    {
      "epoch": 0.8199558354461438,
      "grad_norm": 13.390905380249023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11091110855340958,
      "logits/rejected": -0.30871084332466125,
      "logps/chosen": -590.1342163085938,
      "logps/rejected": -678.4353637695312,
      "loss": 0.9885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1805883646011353,
      "rewards/margins": 0.0646856427192688,
      "rewards/rejected": -1.2452740669250488,
      "step": 6266
    },
    {
      "epoch": 0.8200866933834955,
      "grad_norm": 6.609428882598877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1929233819246292,
      "logits/rejected": -0.481711745262146,
      "logps/chosen": -342.3446044921875,
      "logps/rejected": -380.27093505859375,
      "loss": 0.4146,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15987321734428406,
      "rewards/margins": 1.1868236064910889,
      "rewards/rejected": -1.3466968536376953,
      "step": 6267
    },
    {
      "epoch": 0.8202175513208473,
      "grad_norm": 3.431934356689453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040903717279434204,
      "logits/rejected": -0.07948767393827438,
      "logps/chosen": -487.1007080078125,
      "logps/rejected": -340.4422607421875,
      "loss": 0.2775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09699344635009766,
      "rewards/margins": 1.823091983795166,
      "rewards/rejected": -1.7260985374450684,
      "step": 6268
    },
    {
      "epoch": 0.8203484092581991,
      "grad_norm": 8.847529411315918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013671611435711384,
      "logits/rejected": -0.01891792006790638,
      "logps/chosen": -588.2996215820312,
      "logps/rejected": -773.4977416992188,
      "loss": 0.6927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7929840087890625,
      "rewards/margins": 0.5797700881958008,
      "rewards/rejected": -1.3727540969848633,
      "step": 6269
    },
    {
      "epoch": 0.8204792671955509,
      "grad_norm": 5.700995445251465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11430960893630981,
      "logits/rejected": -0.17319616675376892,
      "logps/chosen": -650.8882446289062,
      "logps/rejected": -526.5093383789062,
      "loss": 0.4556,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.042283594608306885,
      "rewards/margins": 1.3818905353546143,
      "rewards/rejected": -1.3396071195602417,
      "step": 6270
    },
    {
      "epoch": 0.8206101251329025,
      "grad_norm": 5.076353073120117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07429968565702438,
      "logits/rejected": 0.18322083353996277,
      "logps/chosen": -539.6395263671875,
      "logps/rejected": -448.79705810546875,
      "loss": 0.3351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6778835654258728,
      "rewards/margins": 1.6769579648971558,
      "rewards/rejected": -0.9990743398666382,
      "step": 6271
    },
    {
      "epoch": 0.8207409830702543,
      "grad_norm": 5.092657089233398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25252780318260193,
      "logits/rejected": 0.14264962077140808,
      "logps/chosen": -339.37969970703125,
      "logps/rejected": -566.9410400390625,
      "loss": 0.1716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7444206476211548,
      "rewards/margins": 2.0872256755828857,
      "rewards/rejected": -2.83164644241333,
      "step": 6272
    },
    {
      "epoch": 0.8208718410076061,
      "grad_norm": 8.430978775024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08729913830757141,
      "logits/rejected": -0.05209999531507492,
      "logps/chosen": -539.1372680664062,
      "logps/rejected": -503.0475769042969,
      "loss": 0.7044,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5068626403808594,
      "rewards/margins": 0.5399094223976135,
      "rewards/rejected": -1.0467720031738281,
      "step": 6273
    },
    {
      "epoch": 0.8210026989449579,
      "grad_norm": 8.468840599060059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4186556339263916,
      "logits/rejected": -0.38416922092437744,
      "logps/chosen": -808.5615234375,
      "logps/rejected": -434.8807373046875,
      "loss": 0.3198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17803995311260223,
      "rewards/margins": 1.8819050788879395,
      "rewards/rejected": -1.7038650512695312,
      "step": 6274
    },
    {
      "epoch": 0.8211335568823096,
      "grad_norm": 7.567924499511719,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12034492939710617,
      "logits/rejected": -0.018670419231057167,
      "logps/chosen": -358.70233154296875,
      "logps/rejected": -438.8226623535156,
      "loss": 0.4234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04730721563100815,
      "rewards/margins": 0.9862155914306641,
      "rewards/rejected": -1.0335228443145752,
      "step": 6275
    },
    {
      "epoch": 0.8212644148196614,
      "grad_norm": 5.220200061798096,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27073803544044495,
      "logits/rejected": -0.14332248270511627,
      "logps/chosen": -393.39703369140625,
      "logps/rejected": -417.0381774902344,
      "loss": 0.321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.037179216742515564,
      "rewards/margins": 1.3063158988952637,
      "rewards/rejected": -1.269136667251587,
      "step": 6276
    },
    {
      "epoch": 0.8213952727570132,
      "grad_norm": 5.567864894866943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046613745391368866,
      "logits/rejected": 0.03406168892979622,
      "logps/chosen": -438.5924072265625,
      "logps/rejected": -465.0000305175781,
      "loss": 0.3201,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26123103499412537,
      "rewards/margins": 1.4786078929901123,
      "rewards/rejected": -1.217376708984375,
      "step": 6277
    },
    {
      "epoch": 0.821526130694365,
      "grad_norm": 8.371088981628418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2236417829990387,
      "logits/rejected": 0.09716923534870148,
      "logps/chosen": -441.43341064453125,
      "logps/rejected": -322.3501892089844,
      "loss": 0.4556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4162290692329407,
      "rewards/margins": 1.6983717679977417,
      "rewards/rejected": -1.2821426391601562,
      "step": 6278
    },
    {
      "epoch": 0.8216569886317167,
      "grad_norm": 6.351694583892822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39625412225723267,
      "logits/rejected": -0.43869858980178833,
      "logps/chosen": -300.1264953613281,
      "logps/rejected": -425.3191223144531,
      "loss": 0.3504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6441211700439453,
      "rewards/margins": 1.2043310403823853,
      "rewards/rejected": -1.8484523296356201,
      "step": 6279
    },
    {
      "epoch": 0.8217878465690684,
      "grad_norm": 5.671972751617432,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1695055365562439,
      "logits/rejected": -0.12979817390441895,
      "logps/chosen": -340.10809326171875,
      "logps/rejected": -392.27703857421875,
      "loss": 0.3424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22159862518310547,
      "rewards/margins": 1.4239246845245361,
      "rewards/rejected": -1.6455233097076416,
      "step": 6280
    },
    {
      "epoch": 0.8219187045064202,
      "grad_norm": 6.61704683303833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15255260467529297,
      "logits/rejected": -0.2927699685096741,
      "logps/chosen": -492.1481018066406,
      "logps/rejected": -497.2753601074219,
      "loss": 0.4385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9868120551109314,
      "rewards/margins": 0.9077392816543579,
      "rewards/rejected": -1.894551396369934,
      "step": 6281
    },
    {
      "epoch": 0.822049562443772,
      "grad_norm": 5.786417007446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0692412406206131,
      "logits/rejected": -0.2275255024433136,
      "logps/chosen": -467.1991271972656,
      "logps/rejected": -382.3414001464844,
      "loss": 0.4808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1296968311071396,
      "rewards/margins": 0.8569076657295227,
      "rewards/rejected": -0.7272108793258667,
      "step": 6282
    },
    {
      "epoch": 0.8221804203811237,
      "grad_norm": 9.03707504272461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0016110390424728394,
      "logits/rejected": -0.19010856747627258,
      "logps/chosen": -413.196533203125,
      "logps/rejected": -505.8656005859375,
      "loss": 0.5914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04667644202709198,
      "rewards/margins": 1.2712042331695557,
      "rewards/rejected": -1.2245278358459473,
      "step": 6283
    },
    {
      "epoch": 0.8223112783184755,
      "grad_norm": 6.766580581665039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0015169326215982437,
      "logits/rejected": 0.06109532713890076,
      "logps/chosen": -563.9751586914062,
      "logps/rejected": -413.5439453125,
      "loss": 0.4556,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16939619183540344,
      "rewards/margins": 0.833889365196228,
      "rewards/rejected": -1.0032856464385986,
      "step": 6284
    },
    {
      "epoch": 0.8224421362558273,
      "grad_norm": 9.408242225646973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12249984592199326,
      "logits/rejected": 0.1399802267551422,
      "logps/chosen": -664.4727783203125,
      "logps/rejected": -708.0635986328125,
      "loss": 0.3407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13746798038482666,
      "rewards/margins": 2.0324032306671143,
      "rewards/rejected": -2.1698713302612305,
      "step": 6285
    },
    {
      "epoch": 0.822572994193179,
      "grad_norm": 8.685478210449219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08745019882917404,
      "logits/rejected": -0.03631014749407768,
      "logps/chosen": -376.7535400390625,
      "logps/rejected": -352.5985107421875,
      "loss": 0.5773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04331417381763458,
      "rewards/margins": 0.6477218270301819,
      "rewards/rejected": -0.6910360455513,
      "step": 6286
    },
    {
      "epoch": 0.8227038521305308,
      "grad_norm": 10.888897895812988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13381436467170715,
      "logits/rejected": -0.031306855380535126,
      "logps/chosen": -372.8304443359375,
      "logps/rejected": -366.62298583984375,
      "loss": 1.1651,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1042673587799072,
      "rewards/margins": -0.4342411160469055,
      "rewards/rejected": -0.6700262427330017,
      "step": 6287
    },
    {
      "epoch": 0.8228347100678826,
      "grad_norm": 6.292236804962158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12913616001605988,
      "logits/rejected": 0.13113196194171906,
      "logps/chosen": -319.9422302246094,
      "logps/rejected": -308.6768798828125,
      "loss": 0.5627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.592040479183197,
      "rewards/margins": 0.7486375570297241,
      "rewards/rejected": -1.340678095817566,
      "step": 6288
    },
    {
      "epoch": 0.8229655680052343,
      "grad_norm": 3.0338680744171143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03734715282917023,
      "logits/rejected": -0.34568607807159424,
      "logps/chosen": -557.8740844726562,
      "logps/rejected": -770.173583984375,
      "loss": 0.1506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4758535623550415,
      "rewards/margins": 2.8287506103515625,
      "rewards/rejected": -2.3528971672058105,
      "step": 6289
    },
    {
      "epoch": 0.823096425942586,
      "grad_norm": 8.714401245117188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20853069424629211,
      "logits/rejected": -0.2693738043308258,
      "logps/chosen": -498.2649230957031,
      "logps/rejected": -445.9981994628906,
      "loss": 0.584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7205512523651123,
      "rewards/margins": 0.7639186978340149,
      "rewards/rejected": -1.4844698905944824,
      "step": 6290
    },
    {
      "epoch": 0.8232272838799378,
      "grad_norm": 6.35429573059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04687323421239853,
      "logits/rejected": -0.09401835501194,
      "logps/chosen": -462.7903747558594,
      "logps/rejected": -371.2695007324219,
      "loss": 0.3927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8594870567321777,
      "rewards/margins": 0.9855813980102539,
      "rewards/rejected": -1.8450685739517212,
      "step": 6291
    },
    {
      "epoch": 0.8233581418172896,
      "grad_norm": 8.91612720489502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2252013385295868,
      "logits/rejected": 0.3054620623588562,
      "logps/chosen": -447.2846374511719,
      "logps/rejected": -450.0785217285156,
      "loss": 0.9025,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019727319478988647,
      "rewards/margins": 0.9230751395225525,
      "rewards/rejected": -0.9033477306365967,
      "step": 6292
    },
    {
      "epoch": 0.8234889997546414,
      "grad_norm": 9.048920631408691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23315078020095825,
      "logits/rejected": 0.45199134945869446,
      "logps/chosen": -604.93994140625,
      "logps/rejected": -599.9437866210938,
      "loss": 0.6983,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5943787097930908,
      "rewards/margins": 0.10913429409265518,
      "rewards/rejected": -0.703512966632843,
      "step": 6293
    },
    {
      "epoch": 0.8236198576919931,
      "grad_norm": 6.365211486816406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27684563398361206,
      "logits/rejected": -0.33028095960617065,
      "logps/chosen": -509.2036437988281,
      "logps/rejected": -560.1060791015625,
      "loss": 0.2676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14036008715629578,
      "rewards/margins": 1.897965431213379,
      "rewards/rejected": -2.038325309753418,
      "step": 6294
    },
    {
      "epoch": 0.8237507156293449,
      "grad_norm": 9.918900489807129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08369431644678116,
      "logits/rejected": 0.024269111454486847,
      "logps/chosen": -663.635498046875,
      "logps/rejected": -411.33837890625,
      "loss": 0.4428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3767755627632141,
      "rewards/margins": 1.509819746017456,
      "rewards/rejected": -1.886595368385315,
      "step": 6295
    },
    {
      "epoch": 0.8238815735666967,
      "grad_norm": 7.259384632110596,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20936483144760132,
      "logits/rejected": -0.026102915406227112,
      "logps/chosen": -376.31903076171875,
      "logps/rejected": -437.44049072265625,
      "loss": 0.6706,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2764898538589478,
      "rewards/margins": 0.2978021502494812,
      "rewards/rejected": -1.5742919445037842,
      "step": 6296
    },
    {
      "epoch": 0.8240124315040485,
      "grad_norm": 6.811391353607178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1402619630098343,
      "logits/rejected": -0.29149162769317627,
      "logps/chosen": -365.18865966796875,
      "logps/rejected": -350.6683044433594,
      "loss": 0.3578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.505458414554596,
      "rewards/margins": 1.40939199924469,
      "rewards/rejected": -1.9148504734039307,
      "step": 6297
    },
    {
      "epoch": 0.8241432894414001,
      "grad_norm": 10.921675682067871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2519138753414154,
      "logits/rejected": -0.0585264191031456,
      "logps/chosen": -490.9160461425781,
      "logps/rejected": -331.3808898925781,
      "loss": 0.6582,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6771219372749329,
      "rewards/margins": 0.22520828247070312,
      "rewards/rejected": -0.9023301601409912,
      "step": 6298
    },
    {
      "epoch": 0.8242741473787519,
      "grad_norm": 13.321624755859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07315418124198914,
      "logits/rejected": -0.14217661321163177,
      "logps/chosen": -557.1185302734375,
      "logps/rejected": -456.6376647949219,
      "loss": 1.348,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4015344381332397,
      "rewards/margins": -0.5064981579780579,
      "rewards/rejected": -0.8950363397598267,
      "step": 6299
    },
    {
      "epoch": 0.8244050053161037,
      "grad_norm": 9.567146301269531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14042215049266815,
      "logits/rejected": -0.1167452335357666,
      "logps/chosen": -424.430419921875,
      "logps/rejected": -501.21807861328125,
      "loss": 0.8522,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.35400623083114624,
      "rewards/margins": 0.39065343141555786,
      "rewards/rejected": -0.7446596622467041,
      "step": 6300
    },
    {
      "epoch": 0.8245358632534555,
      "grad_norm": 11.1137056350708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11078500747680664,
      "logits/rejected": -0.02586439996957779,
      "logps/chosen": -529.6163330078125,
      "logps/rejected": -482.93408203125,
      "loss": 0.7487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.096616730093956,
      "rewards/margins": 0.814483642578125,
      "rewards/rejected": -0.717866837978363,
      "step": 6301
    },
    {
      "epoch": 0.8246667211908072,
      "grad_norm": 10.124911308288574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2787646949291229,
      "logits/rejected": -0.08205408602952957,
      "logps/chosen": -472.92572021484375,
      "logps/rejected": -411.3801574707031,
      "loss": 0.7593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4767388105392456,
      "rewards/margins": 0.5296777486801147,
      "rewards/rejected": -1.0064165592193604,
      "step": 6302
    },
    {
      "epoch": 0.824797579128159,
      "grad_norm": 8.018428802490234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12158557027578354,
      "logits/rejected": -0.09235291928052902,
      "logps/chosen": -634.634521484375,
      "logps/rejected": -577.7664184570312,
      "loss": 0.5696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37718239426612854,
      "rewards/margins": 0.9114257097244263,
      "rewards/rejected": -1.288608193397522,
      "step": 6303
    },
    {
      "epoch": 0.8249284370655108,
      "grad_norm": 10.53050708770752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24291136860847473,
      "logits/rejected": -0.17930397391319275,
      "logps/chosen": -417.6314392089844,
      "logps/rejected": -326.85601806640625,
      "loss": 0.6641,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14087507128715515,
      "rewards/margins": 0.6741421222686768,
      "rewards/rejected": -0.8150172233581543,
      "step": 6304
    },
    {
      "epoch": 0.8250592950028626,
      "grad_norm": 11.509000778198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5886203646659851,
      "logits/rejected": -0.453965425491333,
      "logps/chosen": -481.32611083984375,
      "logps/rejected": -393.441162109375,
      "loss": 0.7117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.070913553237915,
      "rewards/margins": 0.22593460977077484,
      "rewards/rejected": -1.2968480587005615,
      "step": 6305
    },
    {
      "epoch": 0.8251901529402142,
      "grad_norm": 9.419376373291016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05890890583395958,
      "logits/rejected": -0.2161930352449417,
      "logps/chosen": -386.87017822265625,
      "logps/rejected": -462.00042724609375,
      "loss": 0.6552,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28420865535736084,
      "rewards/margins": 1.1248793601989746,
      "rewards/rejected": -1.4090880155563354,
      "step": 6306
    },
    {
      "epoch": 0.825321010877566,
      "grad_norm": 11.745972633361816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022723639383912086,
      "logits/rejected": -0.5407790541648865,
      "logps/chosen": -534.3243408203125,
      "logps/rejected": -693.3840942382812,
      "loss": 0.4701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0177202224731445,
      "rewards/margins": 0.8348222970962524,
      "rewards/rejected": -1.8525424003601074,
      "step": 6307
    },
    {
      "epoch": 0.8254518688149178,
      "grad_norm": 4.901045799255371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3015696704387665,
      "logits/rejected": -0.10977181047201157,
      "logps/chosen": -347.0517578125,
      "logps/rejected": -360.30059814453125,
      "loss": 0.2844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.058223143219947815,
      "rewards/margins": 1.669947624206543,
      "rewards/rejected": -1.72817063331604,
      "step": 6308
    },
    {
      "epoch": 0.8255827267522695,
      "grad_norm": 5.521054267883301,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09577895700931549,
      "logits/rejected": 0.13356173038482666,
      "logps/chosen": -429.7304382324219,
      "logps/rejected": -448.7552795410156,
      "loss": 0.2591,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.46116843819618225,
      "rewards/margins": 1.4414539337158203,
      "rewards/rejected": -1.9026224613189697,
      "step": 6309
    },
    {
      "epoch": 0.8257135846896213,
      "grad_norm": 4.485215187072754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14929743111133575,
      "logits/rejected": -0.23517030477523804,
      "logps/chosen": -607.1417236328125,
      "logps/rejected": -627.4140625,
      "loss": 0.2744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3425390422344208,
      "rewards/margins": 1.311915397644043,
      "rewards/rejected": -1.6544544696807861,
      "step": 6310
    },
    {
      "epoch": 0.8258444426269731,
      "grad_norm": 7.504159450531006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05022523179650307,
      "logits/rejected": 0.15776222944259644,
      "logps/chosen": -738.10400390625,
      "logps/rejected": -501.57269287109375,
      "loss": 0.5033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13146626949310303,
      "rewards/margins": 0.9016643762588501,
      "rewards/rejected": -0.7701980471611023,
      "step": 6311
    },
    {
      "epoch": 0.8259753005643249,
      "grad_norm": 10.747376441955566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09931830316781998,
      "logits/rejected": -0.05949478596448898,
      "logps/chosen": -465.7454833984375,
      "logps/rejected": -475.135498046875,
      "loss": 0.7944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9286330938339233,
      "rewards/margins": 0.7686833143234253,
      "rewards/rejected": -1.6973164081573486,
      "step": 6312
    },
    {
      "epoch": 0.8261061585016766,
      "grad_norm": 4.701009273529053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09005008637905121,
      "logits/rejected": 0.055130600929260254,
      "logps/chosen": -368.6594543457031,
      "logps/rejected": -350.8037109375,
      "loss": 0.2786,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21289290487766266,
      "rewards/margins": 1.791387677192688,
      "rewards/rejected": -2.0042805671691895,
      "step": 6313
    },
    {
      "epoch": 0.8262370164390284,
      "grad_norm": 7.091182708740234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09859208762645721,
      "logits/rejected": 0.07244880497455597,
      "logps/chosen": -427.27239990234375,
      "logps/rejected": -444.0923156738281,
      "loss": 0.4978,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4229871928691864,
      "rewards/margins": 1.0561134815216064,
      "rewards/rejected": -1.4791005849838257,
      "step": 6314
    },
    {
      "epoch": 0.8263678743763802,
      "grad_norm": 5.968897342681885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006471551954746246,
      "logits/rejected": -0.15077802538871765,
      "logps/chosen": -389.9980163574219,
      "logps/rejected": -374.5648193359375,
      "loss": 0.3784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44168341159820557,
      "rewards/margins": 1.061182975769043,
      "rewards/rejected": -1.502866268157959,
      "step": 6315
    },
    {
      "epoch": 0.8264987323137319,
      "grad_norm": 6.7031025886535645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20750918984413147,
      "logits/rejected": -0.1722380518913269,
      "logps/chosen": -300.07965087890625,
      "logps/rejected": -406.1917724609375,
      "loss": 0.5456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5344519019126892,
      "rewards/margins": 0.6162094473838806,
      "rewards/rejected": -1.1506612300872803,
      "step": 6316
    },
    {
      "epoch": 0.8266295902510836,
      "grad_norm": 9.811013221740723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15872415900230408,
      "logits/rejected": -0.010017625987529755,
      "logps/chosen": -455.87451171875,
      "logps/rejected": -501.839599609375,
      "loss": 0.8462,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4319374561309814,
      "rewards/margins": 0.17083263397216797,
      "rewards/rejected": -1.602770209312439,
      "step": 6317
    },
    {
      "epoch": 0.8267604481884354,
      "grad_norm": 8.025704383850098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09784989804029465,
      "logits/rejected": -0.04165351763367653,
      "logps/chosen": -519.4163818359375,
      "logps/rejected": -384.374267578125,
      "loss": 0.5636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04245644807815552,
      "rewards/margins": 0.7185064554214478,
      "rewards/rejected": -0.7609629034996033,
      "step": 6318
    },
    {
      "epoch": 0.8268913061257872,
      "grad_norm": 6.296946048736572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.397916704416275,
      "logits/rejected": -0.20763486623764038,
      "logps/chosen": -269.216552734375,
      "logps/rejected": -365.19384765625,
      "loss": 0.5812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7662813663482666,
      "rewards/margins": 0.8951568603515625,
      "rewards/rejected": -1.661438226699829,
      "step": 6319
    },
    {
      "epoch": 0.827022164063139,
      "grad_norm": 8.465627670288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26502469182014465,
      "logits/rejected": -0.14925312995910645,
      "logps/chosen": -547.5477294921875,
      "logps/rejected": -381.7667236328125,
      "loss": 0.5399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7590273022651672,
      "rewards/margins": 1.101399302482605,
      "rewards/rejected": -1.860426664352417,
      "step": 6320
    },
    {
      "epoch": 0.8271530220004907,
      "grad_norm": 5.224625587463379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15765070915222168,
      "logits/rejected": -0.1148955300450325,
      "logps/chosen": -480.58367919921875,
      "logps/rejected": -422.25885009765625,
      "loss": 0.295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6924031972885132,
      "rewards/margins": 1.2521425485610962,
      "rewards/rejected": -1.944545865058899,
      "step": 6321
    },
    {
      "epoch": 0.8272838799378425,
      "grad_norm": 11.365547180175781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11648957431316376,
      "logits/rejected": -0.13947930932044983,
      "logps/chosen": -448.1084289550781,
      "logps/rejected": -575.130615234375,
      "loss": 0.7288,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.922926902770996,
      "rewards/margins": 0.14057061076164246,
      "rewards/rejected": -2.063497543334961,
      "step": 6322
    },
    {
      "epoch": 0.8274147378751943,
      "grad_norm": 11.341714859008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023702874779701233,
      "logits/rejected": -0.1399959772825241,
      "logps/chosen": -551.9393310546875,
      "logps/rejected": -562.15478515625,
      "loss": 0.8331,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8438293933868408,
      "rewards/margins": 0.6296131014823914,
      "rewards/rejected": -1.473442554473877,
      "step": 6323
    },
    {
      "epoch": 0.827545595812546,
      "grad_norm": 6.118496417999268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27162960171699524,
      "logits/rejected": -0.26930516958236694,
      "logps/chosen": -353.888916015625,
      "logps/rejected": -352.5743103027344,
      "loss": 0.4523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32381343841552734,
      "rewards/margins": 1.1850502490997314,
      "rewards/rejected": -1.5088636875152588,
      "step": 6324
    },
    {
      "epoch": 0.8276764537498977,
      "grad_norm": 6.861193656921387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14191851019859314,
      "logits/rejected": -0.1414274126291275,
      "logps/chosen": -292.1717529296875,
      "logps/rejected": -370.54888916015625,
      "loss": 0.4489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6818844079971313,
      "rewards/margins": 1.00314199924469,
      "rewards/rejected": -1.6850264072418213,
      "step": 6325
    },
    {
      "epoch": 0.8278073116872495,
      "grad_norm": 7.799355506896973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016491301357746124,
      "logits/rejected": 0.07700949162244797,
      "logps/chosen": -588.959228515625,
      "logps/rejected": -461.737060546875,
      "loss": 0.3942,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3608803749084473,
      "rewards/margins": 0.9744669198989868,
      "rewards/rejected": -2.3353471755981445,
      "step": 6326
    },
    {
      "epoch": 0.8279381696246013,
      "grad_norm": 4.141341209411621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016050197184085846,
      "logits/rejected": -0.059176914393901825,
      "logps/chosen": -474.93359375,
      "logps/rejected": -472.96466064453125,
      "loss": 0.1949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44699686765670776,
      "rewards/margins": 2.228121519088745,
      "rewards/rejected": -1.7811245918273926,
      "step": 6327
    },
    {
      "epoch": 0.8280690275619531,
      "grad_norm": 11.638840675354004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05724448338150978,
      "logits/rejected": 0.018469251692295074,
      "logps/chosen": -263.30645751953125,
      "logps/rejected": -297.9870300292969,
      "loss": 1.0766,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7333996891975403,
      "rewards/margins": 0.2302876114845276,
      "rewards/rejected": -0.9636873602867126,
      "step": 6328
    },
    {
      "epoch": 0.8281998854993048,
      "grad_norm": 3.94846510887146,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16357049345970154,
      "logits/rejected": -0.29837724566459656,
      "logps/chosen": -421.2337646484375,
      "logps/rejected": -469.9073486328125,
      "loss": 0.22,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17861519753932953,
      "rewards/margins": 1.6586110591888428,
      "rewards/rejected": -1.8372262716293335,
      "step": 6329
    },
    {
      "epoch": 0.8283307434366566,
      "grad_norm": 6.936401844024658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14125052094459534,
      "logits/rejected": -0.3066025376319885,
      "logps/chosen": -475.7729187011719,
      "logps/rejected": -497.93292236328125,
      "loss": 0.5284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6838940978050232,
      "rewards/margins": 0.941629946231842,
      "rewards/rejected": -1.6255240440368652,
      "step": 6330
    },
    {
      "epoch": 0.8284616013740084,
      "grad_norm": 4.143482208251953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1718854010105133,
      "logits/rejected": -0.24505352973937988,
      "logps/chosen": -325.82476806640625,
      "logps/rejected": -355.4886169433594,
      "loss": 0.3757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15852834284305573,
      "rewards/margins": 1.3832223415374756,
      "rewards/rejected": -1.2246938943862915,
      "step": 6331
    },
    {
      "epoch": 0.82859245931136,
      "grad_norm": 7.547713756561279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0412181094288826,
      "logits/rejected": -0.2257155478000641,
      "logps/chosen": -455.11126708984375,
      "logps/rejected": -418.19384765625,
      "loss": 0.5892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08186284452676773,
      "rewards/margins": 0.463583379983902,
      "rewards/rejected": -0.5454462170600891,
      "step": 6332
    },
    {
      "epoch": 0.8287233172487118,
      "grad_norm": 7.135590076446533,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016519973054528236,
      "logits/rejected": -0.16611884534358978,
      "logps/chosen": -583.4436645507812,
      "logps/rejected": -417.62823486328125,
      "loss": 0.3876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3775283694267273,
      "rewards/margins": 1.3436390161514282,
      "rewards/rejected": -0.9661105871200562,
      "step": 6333
    },
    {
      "epoch": 0.8288541751860636,
      "grad_norm": 6.615993022918701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4505150318145752,
      "logits/rejected": -0.21552124619483948,
      "logps/chosen": -474.55059814453125,
      "logps/rejected": -555.4721069335938,
      "loss": 0.4603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.142869770526886,
      "rewards/margins": 1.4022624492645264,
      "rewards/rejected": -1.2593927383422852,
      "step": 6334
    },
    {
      "epoch": 0.8289850331234154,
      "grad_norm": 8.087482452392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20503458380699158,
      "logits/rejected": 0.07176858186721802,
      "logps/chosen": -504.06707763671875,
      "logps/rejected": -553.03173828125,
      "loss": 0.4469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0199633836746216,
      "rewards/margins": 1.579050898551941,
      "rewards/rejected": -2.5990140438079834,
      "step": 6335
    },
    {
      "epoch": 0.8291158910607671,
      "grad_norm": 8.323320388793945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1757410764694214,
      "logits/rejected": -0.10438144207000732,
      "logps/chosen": -469.5805358886719,
      "logps/rejected": -379.39117431640625,
      "loss": 0.6381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2736482620239258,
      "rewards/margins": 0.7859698534011841,
      "rewards/rejected": -0.5123216509819031,
      "step": 6336
    },
    {
      "epoch": 0.8292467489981189,
      "grad_norm": 7.893144130706787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11327792704105377,
      "logits/rejected": -0.14163245260715485,
      "logps/chosen": -489.0782470703125,
      "logps/rejected": -674.0914306640625,
      "loss": 0.5942,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3695131540298462,
      "rewards/margins": 0.5303637981414795,
      "rewards/rejected": -0.8998769521713257,
      "step": 6337
    },
    {
      "epoch": 0.8293776069354707,
      "grad_norm": 17.242834091186523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1632271558046341,
      "logits/rejected": -0.21263208985328674,
      "logps/chosen": -477.5743408203125,
      "logps/rejected": -498.8682861328125,
      "loss": 0.7946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0939196348190308,
      "rewards/margins": 0.8939607739448547,
      "rewards/rejected": -1.9878807067871094,
      "step": 6338
    },
    {
      "epoch": 0.8295084648728225,
      "grad_norm": 6.297145366668701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.056983187794685364,
      "logits/rejected": -0.04146605730056763,
      "logps/chosen": -372.0552978515625,
      "logps/rejected": -439.462158203125,
      "loss": 0.4325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6505951285362244,
      "rewards/margins": 0.8866400122642517,
      "rewards/rejected": -1.5372350215911865,
      "step": 6339
    },
    {
      "epoch": 0.8296393228101742,
      "grad_norm": 16.55504035949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17685580253601074,
      "logits/rejected": -0.15106850862503052,
      "logps/chosen": -555.95703125,
      "logps/rejected": -677.1287841796875,
      "loss": 0.9841,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2428735494613647,
      "rewards/margins": 0.27677422761917114,
      "rewards/rejected": -1.5196478366851807,
      "step": 6340
    },
    {
      "epoch": 0.829770180747526,
      "grad_norm": 7.131919860839844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06046193838119507,
      "logits/rejected": -0.12038198113441467,
      "logps/chosen": -456.0924987792969,
      "logps/rejected": -490.12884521484375,
      "loss": 0.5091,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.402444839477539,
      "rewards/margins": 0.6708507537841797,
      "rewards/rejected": -2.073295831680298,
      "step": 6341
    },
    {
      "epoch": 0.8299010386848777,
      "grad_norm": 6.244165420532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051346272230148315,
      "logits/rejected": -0.3022831678390503,
      "logps/chosen": -395.67742919921875,
      "logps/rejected": -388.10455322265625,
      "loss": 0.3539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7936227917671204,
      "rewards/margins": 1.7366929054260254,
      "rewards/rejected": -2.53031587600708,
      "step": 6342
    },
    {
      "epoch": 0.8300318966222295,
      "grad_norm": 5.899433612823486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03314374387264252,
      "logits/rejected": -0.13498300313949585,
      "logps/chosen": -402.406982421875,
      "logps/rejected": -383.71124267578125,
      "loss": 0.45,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003117939457297325,
      "rewards/margins": 0.9083405137062073,
      "rewards/rejected": -0.9114583730697632,
      "step": 6343
    },
    {
      "epoch": 0.8301627545595812,
      "grad_norm": 6.4705891609191895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.183711975812912,
      "logits/rejected": -0.13623538613319397,
      "logps/chosen": -449.5741882324219,
      "logps/rejected": -364.5966796875,
      "loss": 0.3634,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24085292220115662,
      "rewards/margins": 1.3100165128707886,
      "rewards/rejected": -1.550869345664978,
      "step": 6344
    },
    {
      "epoch": 0.830293612496933,
      "grad_norm": 10.860589981079102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23021000623703003,
      "logits/rejected": -0.142071932554245,
      "logps/chosen": -477.2860412597656,
      "logps/rejected": -604.20068359375,
      "loss": 0.9089,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6695141196250916,
      "rewards/margins": -0.1000327616930008,
      "rewards/rejected": -0.5694812536239624,
      "step": 6345
    },
    {
      "epoch": 0.8304244704342848,
      "grad_norm": 9.298721313476562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30573463439941406,
      "logits/rejected": -0.37410715222358704,
      "logps/chosen": -828.09521484375,
      "logps/rejected": -566.2015991210938,
      "loss": 0.3815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7360647916793823,
      "rewards/margins": 1.1975411176681519,
      "rewards/rejected": -1.9336057901382446,
      "step": 6346
    },
    {
      "epoch": 0.8305553283716366,
      "grad_norm": 5.165582180023193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2804509401321411,
      "logits/rejected": -0.18541352450847626,
      "logps/chosen": -627.303466796875,
      "logps/rejected": -525.2408447265625,
      "loss": 0.301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1043565571308136,
      "rewards/margins": 1.8205888271331787,
      "rewards/rejected": -1.7162322998046875,
      "step": 6347
    },
    {
      "epoch": 0.8306861863089883,
      "grad_norm": 9.62000846862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06689070165157318,
      "logits/rejected": -0.12370620667934418,
      "logps/chosen": -503.2814636230469,
      "logps/rejected": -647.710205078125,
      "loss": 0.4906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45239871740341187,
      "rewards/margins": 1.4812098741531372,
      "rewards/rejected": -1.9336086511611938,
      "step": 6348
    },
    {
      "epoch": 0.8308170442463401,
      "grad_norm": 5.195652961730957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01722007989883423,
      "logits/rejected": -0.2979619801044464,
      "logps/chosen": -382.42352294921875,
      "logps/rejected": -346.0350036621094,
      "loss": 0.41,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0165523290634155,
      "rewards/margins": 1.2701205015182495,
      "rewards/rejected": -2.286672830581665,
      "step": 6349
    },
    {
      "epoch": 0.8309479021836919,
      "grad_norm": 8.803640365600586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03330586105585098,
      "logits/rejected": -0.09892904758453369,
      "logps/chosen": -371.4918518066406,
      "logps/rejected": -388.9577331542969,
      "loss": 0.7487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3986893594264984,
      "rewards/margins": 0.5935847759246826,
      "rewards/rejected": -0.9922741651535034,
      "step": 6350
    },
    {
      "epoch": 0.8310787601210436,
      "grad_norm": 2.939446210861206,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09112890809774399,
      "logits/rejected": -0.21477949619293213,
      "logps/chosen": -620.859619140625,
      "logps/rejected": -385.4283142089844,
      "loss": 0.2296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6593628525733948,
      "rewards/margins": 2.0636632442474365,
      "rewards/rejected": -1.404300570487976,
      "step": 6351
    },
    {
      "epoch": 0.8312096180583953,
      "grad_norm": 5.397464752197266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01841450110077858,
      "logits/rejected": -0.09006418287754059,
      "logps/chosen": -617.864501953125,
      "logps/rejected": -444.2779235839844,
      "loss": 0.2724,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18800462782382965,
      "rewards/margins": 1.8624235391616821,
      "rewards/rejected": -1.6744186878204346,
      "step": 6352
    },
    {
      "epoch": 0.8313404759957471,
      "grad_norm": 7.607936382293701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13662099838256836,
      "logits/rejected": -0.195487380027771,
      "logps/chosen": -474.6333312988281,
      "logps/rejected": -533.9099731445312,
      "loss": 0.3264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6357536315917969,
      "rewards/margins": 1.7206989526748657,
      "rewards/rejected": -1.0849453210830688,
      "step": 6353
    },
    {
      "epoch": 0.8314713339330989,
      "grad_norm": 4.204554080963135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12901005148887634,
      "logits/rejected": -0.053424470126628876,
      "logps/chosen": -597.2626953125,
      "logps/rejected": -433.362060546875,
      "loss": 0.218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09352722018957138,
      "rewards/margins": 1.857882022857666,
      "rewards/rejected": -1.7643547058105469,
      "step": 6354
    },
    {
      "epoch": 0.8316021918704506,
      "grad_norm": 9.361581802368164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08938968926668167,
      "logits/rejected": 0.040333688259124756,
      "logps/chosen": -762.3582153320312,
      "logps/rejected": -611.6781005859375,
      "loss": 0.5046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4199499785900116,
      "rewards/margins": 1.0253032445907593,
      "rewards/rejected": -1.4452531337738037,
      "step": 6355
    },
    {
      "epoch": 0.8317330498078024,
      "grad_norm": 10.255473136901855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040529824793338776,
      "logits/rejected": 0.05036904662847519,
      "logps/chosen": -503.2225646972656,
      "logps/rejected": -502.9997863769531,
      "loss": 0.9925,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.07583294063806534,
      "rewards/margins": -0.305110365152359,
      "rewards/rejected": 0.22927743196487427,
      "step": 6356
    },
    {
      "epoch": 0.8318639077451542,
      "grad_norm": 12.414182662963867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008192412555217743,
      "logits/rejected": 0.010820895433425903,
      "logps/chosen": -456.024658203125,
      "logps/rejected": -564.641845703125,
      "loss": 1.0961,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6246524453163147,
      "rewards/margins": -0.3323737680912018,
      "rewards/rejected": -0.2922787368297577,
      "step": 6357
    },
    {
      "epoch": 0.831994765682506,
      "grad_norm": 6.416461944580078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05212407186627388,
      "logits/rejected": -0.1848020851612091,
      "logps/chosen": -502.45220947265625,
      "logps/rejected": -467.8558349609375,
      "loss": 0.4232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7445905208587646,
      "rewards/margins": 0.9546392560005188,
      "rewards/rejected": -1.6992299556732178,
      "step": 6358
    },
    {
      "epoch": 0.8321256236198576,
      "grad_norm": 5.47238302230835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11198576539754868,
      "logits/rejected": 0.0246119424700737,
      "logps/chosen": -322.34832763671875,
      "logps/rejected": -375.79815673828125,
      "loss": 0.3376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29134029150009155,
      "rewards/margins": 1.283470869064331,
      "rewards/rejected": -0.9921306371688843,
      "step": 6359
    },
    {
      "epoch": 0.8322564815572094,
      "grad_norm": 7.626089096069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07434489578008652,
      "logits/rejected": -0.081976979970932,
      "logps/chosen": -700.04443359375,
      "logps/rejected": -464.71624755859375,
      "loss": 0.3347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3325193524360657,
      "rewards/margins": 1.7942661046981812,
      "rewards/rejected": -1.4617468118667603,
      "step": 6360
    },
    {
      "epoch": 0.8323873394945612,
      "grad_norm": 6.41161584854126,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029687128961086273,
      "logits/rejected": 0.017548661679029465,
      "logps/chosen": -492.3737487792969,
      "logps/rejected": -473.0788879394531,
      "loss": 0.3916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24644356966018677,
      "rewards/margins": 1.289805293083191,
      "rewards/rejected": -1.0433616638183594,
      "step": 6361
    },
    {
      "epoch": 0.832518197431913,
      "grad_norm": 7.984194278717041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009546583518385887,
      "logits/rejected": -0.24551481008529663,
      "logps/chosen": -469.51934814453125,
      "logps/rejected": -453.3088684082031,
      "loss": 0.5461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14354190230369568,
      "rewards/margins": 1.267228364944458,
      "rewards/rejected": -1.4107704162597656,
      "step": 6362
    },
    {
      "epoch": 0.8326490553692647,
      "grad_norm": 5.891074180603027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07834944128990173,
      "logits/rejected": -0.10924237221479416,
      "logps/chosen": -575.9327392578125,
      "logps/rejected": -339.4415588378906,
      "loss": 0.3488,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3317855894565582,
      "rewards/margins": 1.4419617652893066,
      "rewards/rejected": -1.7737473249435425,
      "step": 6363
    },
    {
      "epoch": 0.8327799133066165,
      "grad_norm": 6.844786643981934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21094118058681488,
      "logits/rejected": 0.1247107982635498,
      "logps/chosen": -423.0069885253906,
      "logps/rejected": -497.1274719238281,
      "loss": 0.5064,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6439729928970337,
      "rewards/margins": 1.2330267429351807,
      "rewards/rejected": -1.8769997358322144,
      "step": 6364
    },
    {
      "epoch": 0.8329107712439683,
      "grad_norm": 8.127559661865234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2707270681858063,
      "logits/rejected": 0.06657445430755615,
      "logps/chosen": -400.1428527832031,
      "logps/rejected": -379.3943176269531,
      "loss": 0.7146,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6823005080223083,
      "rewards/margins": 0.2541579008102417,
      "rewards/rejected": -0.9364583492279053,
      "step": 6365
    },
    {
      "epoch": 0.8330416291813201,
      "grad_norm": 6.326128005981445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04507128894329071,
      "logits/rejected": 0.16859567165374756,
      "logps/chosen": -610.53759765625,
      "logps/rejected": -572.94482421875,
      "loss": 0.411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5386461615562439,
      "rewards/margins": 0.9429051876068115,
      "rewards/rejected": -1.481551170349121,
      "step": 6366
    },
    {
      "epoch": 0.8331724871186718,
      "grad_norm": 6.439513683319092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14385420083999634,
      "logits/rejected": -0.1711009293794632,
      "logps/chosen": -448.259033203125,
      "logps/rejected": -468.83740234375,
      "loss": 0.5167,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0880235806107521,
      "rewards/margins": 0.9458118677139282,
      "rewards/rejected": -0.8577883243560791,
      "step": 6367
    },
    {
      "epoch": 0.8333033450560235,
      "grad_norm": 8.62427043914795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16719019412994385,
      "logits/rejected": -0.1341429203748703,
      "logps/chosen": -387.29510498046875,
      "logps/rejected": -414.7806091308594,
      "loss": 0.4242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9835480451583862,
      "rewards/margins": 1.2662622928619385,
      "rewards/rejected": -2.249810218811035,
      "step": 6368
    },
    {
      "epoch": 0.8334342029933753,
      "grad_norm": 3.8377490043640137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20029661059379578,
      "logits/rejected": -0.2131933718919754,
      "logps/chosen": -506.36639404296875,
      "logps/rejected": -488.4109191894531,
      "loss": 0.3253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17984123528003693,
      "rewards/margins": 2.062835693359375,
      "rewards/rejected": -2.2426769733428955,
      "step": 6369
    },
    {
      "epoch": 0.8335650609307271,
      "grad_norm": 11.572249412536621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04054497927427292,
      "logits/rejected": -0.12369628995656967,
      "logps/chosen": -577.4130249023438,
      "logps/rejected": -608.8079223632812,
      "loss": 0.7469,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17061442136764526,
      "rewards/margins": 0.2764778137207031,
      "rewards/rejected": -0.44709229469299316,
      "step": 6370
    },
    {
      "epoch": 0.8336959188680788,
      "grad_norm": 7.435694217681885,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12797212600708008,
      "logits/rejected": 0.016717683523893356,
      "logps/chosen": -446.2333068847656,
      "logps/rejected": -494.1175231933594,
      "loss": 0.4737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7535465955734253,
      "rewards/margins": 0.8507082462310791,
      "rewards/rejected": -1.6042548418045044,
      "step": 6371
    },
    {
      "epoch": 0.8338267768054306,
      "grad_norm": 5.928276538848877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11910620331764221,
      "logits/rejected": -0.00037164241075515747,
      "logps/chosen": -430.9791259765625,
      "logps/rejected": -447.522216796875,
      "loss": 0.4516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.016260303556919098,
      "rewards/margins": 2.039875030517578,
      "rewards/rejected": -2.0236146450042725,
      "step": 6372
    },
    {
      "epoch": 0.8339576347427824,
      "grad_norm": 7.916579246520996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16542038321495056,
      "logits/rejected": 0.216334268450737,
      "logps/chosen": -554.7250366210938,
      "logps/rejected": -671.1036376953125,
      "loss": 0.5045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4451250433921814,
      "rewards/margins": 0.5573546290397644,
      "rewards/rejected": -0.11222954094409943,
      "step": 6373
    },
    {
      "epoch": 0.8340884926801341,
      "grad_norm": 8.66656494140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3912980854511261,
      "logits/rejected": -0.46522870659828186,
      "logps/chosen": -329.85699462890625,
      "logps/rejected": -298.7159729003906,
      "loss": 0.8043,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5036411285400391,
      "rewards/margins": -0.12308711558580399,
      "rewards/rejected": -0.3805540204048157,
      "step": 6374
    },
    {
      "epoch": 0.8342193506174859,
      "grad_norm": 9.398176193237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44790300726890564,
      "logits/rejected": -0.4784453511238098,
      "logps/chosen": -437.01129150390625,
      "logps/rejected": -663.9139404296875,
      "loss": 0.6593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2732112407684326,
      "rewards/margins": 0.3832830786705017,
      "rewards/rejected": -1.656494379043579,
      "step": 6375
    },
    {
      "epoch": 0.8343502085548377,
      "grad_norm": 7.312665939331055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04843435436487198,
      "logits/rejected": -0.18000748753547668,
      "logps/chosen": -584.4986572265625,
      "logps/rejected": -536.342529296875,
      "loss": 0.4214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18735560774803162,
      "rewards/margins": 1.418108582496643,
      "rewards/rejected": -1.605464220046997,
      "step": 6376
    },
    {
      "epoch": 0.8344810664921894,
      "grad_norm": 11.803282737731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12250863015651703,
      "logits/rejected": -0.05963905155658722,
      "logps/chosen": -500.3547058105469,
      "logps/rejected": -481.3043518066406,
      "loss": 0.8767,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2753422260284424,
      "rewards/margins": -0.034575074911117554,
      "rewards/rejected": -1.240767240524292,
      "step": 6377
    },
    {
      "epoch": 0.8346119244295411,
      "grad_norm": 12.082291603088379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0964103415608406,
      "logits/rejected": 0.14700743556022644,
      "logps/chosen": -572.137939453125,
      "logps/rejected": -527.3670654296875,
      "loss": 0.8097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2872040271759033,
      "rewards/margins": 1.4578968286514282,
      "rewards/rejected": -1.7451008558273315,
      "step": 6378
    },
    {
      "epoch": 0.8347427823668929,
      "grad_norm": 9.137571334838867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05679836496710777,
      "logits/rejected": 0.0019458833849057555,
      "logps/chosen": -397.4063720703125,
      "logps/rejected": -353.94342041015625,
      "loss": 0.4354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43198108673095703,
      "rewards/margins": 0.9789979457855225,
      "rewards/rejected": -1.410979151725769,
      "step": 6379
    },
    {
      "epoch": 0.8348736403042447,
      "grad_norm": 6.4261860847473145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1007966548204422,
      "logits/rejected": -0.25834232568740845,
      "logps/chosen": -432.40106201171875,
      "logps/rejected": -337.03680419921875,
      "loss": 0.5284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.020656801760196686,
      "rewards/margins": 0.9243137836456299,
      "rewards/rejected": -0.9449705481529236,
      "step": 6380
    },
    {
      "epoch": 0.8350044982415965,
      "grad_norm": 10.578512191772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15654559433460236,
      "logits/rejected": -0.1881706565618515,
      "logps/chosen": -389.26458740234375,
      "logps/rejected": -340.42047119140625,
      "loss": 0.8974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4427651166915894,
      "rewards/margins": -0.02458210289478302,
      "rewards/rejected": -1.4181828498840332,
      "step": 6381
    },
    {
      "epoch": 0.8351353561789482,
      "grad_norm": 8.603260040283203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06896381080150604,
      "logits/rejected": 0.2842209041118622,
      "logps/chosen": -413.3757629394531,
      "logps/rejected": -325.07086181640625,
      "loss": 0.5336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0855182632803917,
      "rewards/margins": 1.071453332901001,
      "rewards/rejected": -1.1569715738296509,
      "step": 6382
    },
    {
      "epoch": 0.8352662141163,
      "grad_norm": 6.926917552947998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009971065446734428,
      "logits/rejected": -0.032471172511577606,
      "logps/chosen": -560.5560302734375,
      "logps/rejected": -410.770263671875,
      "loss": 0.3735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3396965265274048,
      "rewards/margins": 1.5371863842010498,
      "rewards/rejected": -1.8768829107284546,
      "step": 6383
    },
    {
      "epoch": 0.8353970720536518,
      "grad_norm": 5.799095630645752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07886097580194473,
      "logits/rejected": -0.20300021767616272,
      "logps/chosen": -385.15374755859375,
      "logps/rejected": -382.5117492675781,
      "loss": 0.4922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5586837530136108,
      "rewards/margins": 1.2093031406402588,
      "rewards/rejected": -1.7679870128631592,
      "step": 6384
    },
    {
      "epoch": 0.8355279299910036,
      "grad_norm": 12.208501815795898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024456556886434555,
      "logits/rejected": -0.08374032378196716,
      "logps/chosen": -325.98175048828125,
      "logps/rejected": -320.078125,
      "loss": 0.8433,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.034948706626892,
      "rewards/margins": 0.3883260488510132,
      "rewards/rejected": -1.4232747554779053,
      "step": 6385
    },
    {
      "epoch": 0.8356587879283552,
      "grad_norm": 5.224243640899658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08951166272163391,
      "logits/rejected": -0.10419798642396927,
      "logps/chosen": -341.5455322265625,
      "logps/rejected": -334.81988525390625,
      "loss": 0.3186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18500232696533203,
      "rewards/margins": 1.5933170318603516,
      "rewards/rejected": -1.7783193588256836,
      "step": 6386
    },
    {
      "epoch": 0.835789645865707,
      "grad_norm": 7.130695819854736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.47325390577316284,
      "logits/rejected": -0.11503415554761887,
      "logps/chosen": -516.9404296875,
      "logps/rejected": -485.8077087402344,
      "loss": 0.352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47111207246780396,
      "rewards/margins": 0.9141908884048462,
      "rewards/rejected": -1.385303020477295,
      "step": 6387
    },
    {
      "epoch": 0.8359205038030588,
      "grad_norm": 7.355691432952881,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09483350068330765,
      "logits/rejected": -0.1815582811832428,
      "logps/chosen": -375.6317443847656,
      "logps/rejected": -428.868408203125,
      "loss": 0.4198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3892834484577179,
      "rewards/margins": 1.206526756286621,
      "rewards/rejected": -1.595810055732727,
      "step": 6388
    },
    {
      "epoch": 0.8360513617404106,
      "grad_norm": 2.8669521808624268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030253849923610687,
      "logits/rejected": -0.38929519057273865,
      "logps/chosen": -322.914306640625,
      "logps/rejected": -341.9429626464844,
      "loss": 0.2012,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20600205659866333,
      "rewards/margins": 2.363280773162842,
      "rewards/rejected": -2.1572787761688232,
      "step": 6389
    },
    {
      "epoch": 0.8361822196777623,
      "grad_norm": 7.575852870941162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05295426398515701,
      "logits/rejected": -0.14089836180210114,
      "logps/chosen": -507.7574157714844,
      "logps/rejected": -492.5450439453125,
      "loss": 0.5371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2823946177959442,
      "rewards/margins": 0.5533438324928284,
      "rewards/rejected": -0.8357384204864502,
      "step": 6390
    },
    {
      "epoch": 0.8363130776151141,
      "grad_norm": 5.70143461227417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17298507690429688,
      "logits/rejected": 0.14346662163734436,
      "logps/chosen": -372.6615295410156,
      "logps/rejected": -388.91741943359375,
      "loss": 0.253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9798269867897034,
      "rewards/margins": 2.060108184814453,
      "rewards/rejected": -1.080281376838684,
      "step": 6391
    },
    {
      "epoch": 0.8364439355524659,
      "grad_norm": 6.973207950592041,
      "learning_rate": 5e-06,
      "logits/chosen": 1.3055279850959778e-05,
      "logits/rejected": -0.09663597494363785,
      "logps/chosen": -547.1427612304688,
      "logps/rejected": -444.65966796875,
      "loss": 0.5202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.524510383605957,
      "rewards/margins": 1.2151007652282715,
      "rewards/rejected": -0.6905904412269592,
      "step": 6392
    },
    {
      "epoch": 0.8365747934898177,
      "grad_norm": 9.776822090148926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19456136226654053,
      "logits/rejected": 0.12569460272789001,
      "logps/chosen": -512.6525268554688,
      "logps/rejected": -387.94171142578125,
      "loss": 0.7782,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015673264861106873,
      "rewards/margins": 0.4449254870414734,
      "rewards/rejected": -0.46059876680374146,
      "step": 6393
    },
    {
      "epoch": 0.8367056514271694,
      "grad_norm": 8.741127014160156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16260230541229248,
      "logits/rejected": -0.3710845410823822,
      "logps/chosen": -518.7937622070312,
      "logps/rejected": -686.37890625,
      "loss": 0.5209,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40842628479003906,
      "rewards/margins": 0.859356164932251,
      "rewards/rejected": -1.26778244972229,
      "step": 6394
    },
    {
      "epoch": 0.8368365093645211,
      "grad_norm": 10.996915817260742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19579559564590454,
      "logits/rejected": -0.14804202318191528,
      "logps/chosen": -561.9740600585938,
      "logps/rejected": -450.25115966796875,
      "loss": 0.8613,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5274451971054077,
      "rewards/margins": 0.11033117771148682,
      "rewards/rejected": -0.6377763748168945,
      "step": 6395
    },
    {
      "epoch": 0.8369673673018729,
      "grad_norm": 8.243316650390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022243835031986237,
      "logits/rejected": 0.035030484199523926,
      "logps/chosen": -266.59161376953125,
      "logps/rejected": -363.97125244140625,
      "loss": 0.4579,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08332741260528564,
      "rewards/margins": 1.5231306552886963,
      "rewards/rejected": -1.6064579486846924,
      "step": 6396
    },
    {
      "epoch": 0.8370982252392246,
      "grad_norm": 8.917241096496582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1548926830291748,
      "logits/rejected": -0.173118457198143,
      "logps/chosen": -515.2033081054688,
      "logps/rejected": -453.3042297363281,
      "loss": 0.6809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1726512610912323,
      "rewards/margins": 0.8532553315162659,
      "rewards/rejected": -1.0259066820144653,
      "step": 6397
    },
    {
      "epoch": 0.8372290831765764,
      "grad_norm": 6.840787410736084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048528190702199936,
      "logits/rejected": -0.03847636282444,
      "logps/chosen": -377.6241760253906,
      "logps/rejected": -313.93072509765625,
      "loss": 0.6248,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5593340396881104,
      "rewards/margins": 0.4539591670036316,
      "rewards/rejected": -1.0132931470870972,
      "step": 6398
    },
    {
      "epoch": 0.8373599411139282,
      "grad_norm": 9.129213333129883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09807530790567398,
      "logits/rejected": -0.08155427128076553,
      "logps/chosen": -631.4766845703125,
      "logps/rejected": -770.7930908203125,
      "loss": 0.6147,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5827949643135071,
      "rewards/margins": 0.5789758563041687,
      "rewards/rejected": -1.1617708206176758,
      "step": 6399
    },
    {
      "epoch": 0.83749079905128,
      "grad_norm": 3.8242218494415283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17460575699806213,
      "logits/rejected": 0.12054368853569031,
      "logps/chosen": -492.42315673828125,
      "logps/rejected": -462.65838623046875,
      "loss": 0.2913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7878414392471313,
      "rewards/margins": 1.4949918985366821,
      "rewards/rejected": -0.7071504592895508,
      "step": 6400
    },
    {
      "epoch": 0.8376216569886317,
      "grad_norm": 10.249593734741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10752127319574356,
      "logits/rejected": 0.19342346489429474,
      "logps/chosen": -382.3492431640625,
      "logps/rejected": -477.7034912109375,
      "loss": 0.6668,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.39559996128082275,
      "rewards/margins": 0.5924661159515381,
      "rewards/rejected": -0.9880661368370056,
      "step": 6401
    },
    {
      "epoch": 0.8377525149259835,
      "grad_norm": 6.913357257843018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.048140451312065125,
      "logits/rejected": -0.24951623380184174,
      "logps/chosen": -473.56829833984375,
      "logps/rejected": -385.0411376953125,
      "loss": 0.4946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09604262560606003,
      "rewards/margins": 0.6183803081512451,
      "rewards/rejected": -0.5223377346992493,
      "step": 6402
    },
    {
      "epoch": 0.8378833728633353,
      "grad_norm": 9.461962699890137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36891406774520874,
      "logits/rejected": 0.4873986542224884,
      "logps/chosen": -352.1451110839844,
      "logps/rejected": -463.0543212890625,
      "loss": 0.7333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22244012355804443,
      "rewards/margins": 0.7522257566452026,
      "rewards/rejected": -0.9746658802032471,
      "step": 6403
    },
    {
      "epoch": 0.838014230800687,
      "grad_norm": 4.538422584533691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05188798904418945,
      "logits/rejected": -0.24661017954349518,
      "logps/chosen": -512.3908081054688,
      "logps/rejected": -513.6450805664062,
      "loss": 0.2143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0668235793709755,
      "rewards/margins": 1.963578224182129,
      "rewards/rejected": -1.8967546224594116,
      "step": 6404
    },
    {
      "epoch": 0.8381450887380387,
      "grad_norm": 5.287712097167969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031120985746383667,
      "logits/rejected": -0.07343389838933945,
      "logps/chosen": -469.29656982421875,
      "logps/rejected": -482.2010192871094,
      "loss": 0.422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25656530261039734,
      "rewards/margins": 1.153356671333313,
      "rewards/rejected": -1.4099218845367432,
      "step": 6405
    },
    {
      "epoch": 0.8382759466753905,
      "grad_norm": 8.195605278015137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04537644982337952,
      "logits/rejected": 0.03270421922206879,
      "logps/chosen": -555.4225463867188,
      "logps/rejected": -443.43658447265625,
      "loss": 0.6236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14122429490089417,
      "rewards/margins": 0.4961735010147095,
      "rewards/rejected": -0.6373977661132812,
      "step": 6406
    },
    {
      "epoch": 0.8384068046127423,
      "grad_norm": 4.840469837188721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14317269623279572,
      "logits/rejected": 0.028861042112112045,
      "logps/chosen": -421.062744140625,
      "logps/rejected": -280.7746887207031,
      "loss": 0.3125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.627016544342041,
      "rewards/margins": 1.5116658210754395,
      "rewards/rejected": -0.8846495747566223,
      "step": 6407
    },
    {
      "epoch": 0.8385376625500941,
      "grad_norm": 6.936586856842041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042017873376607895,
      "logits/rejected": -0.15387952327728271,
      "logps/chosen": -480.66937255859375,
      "logps/rejected": -347.2214050292969,
      "loss": 0.3708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6657585501670837,
      "rewards/margins": 0.8838783502578735,
      "rewards/rejected": -1.5496368408203125,
      "step": 6408
    },
    {
      "epoch": 0.8386685204874458,
      "grad_norm": 3.6596057415008545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06819389760494232,
      "logits/rejected": 0.2041683793067932,
      "logps/chosen": -504.88824462890625,
      "logps/rejected": -463.29888916015625,
      "loss": 0.2209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16269150376319885,
      "rewards/margins": 1.7687156200408936,
      "rewards/rejected": -1.6060240268707275,
      "step": 6409
    },
    {
      "epoch": 0.8387993784247976,
      "grad_norm": 5.652368545532227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06835977733135223,
      "logits/rejected": -0.20413261651992798,
      "logps/chosen": -420.9850769042969,
      "logps/rejected": -366.48187255859375,
      "loss": 0.2684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4598895311355591,
      "rewards/margins": 1.4778070449829102,
      "rewards/rejected": -1.9376966953277588,
      "step": 6410
    },
    {
      "epoch": 0.8389302363621494,
      "grad_norm": 4.489303112030029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29048407077789307,
      "logits/rejected": -0.2489335685968399,
      "logps/chosen": -718.0125122070312,
      "logps/rejected": -537.5084228515625,
      "loss": 0.2909,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8181073069572449,
      "rewards/margins": 2.082394599914551,
      "rewards/rejected": -1.2642872333526611,
      "step": 6411
    },
    {
      "epoch": 0.8390610942995012,
      "grad_norm": 5.116456508636475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11769215017557144,
      "logits/rejected": 0.12270574271678925,
      "logps/chosen": -244.56744384765625,
      "logps/rejected": -322.4691162109375,
      "loss": 0.3885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.009042352437973022,
      "rewards/margins": 0.9321555495262146,
      "rewards/rejected": -0.9411977529525757,
      "step": 6412
    },
    {
      "epoch": 0.8391919522368528,
      "grad_norm": 6.367210865020752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1176990270614624,
      "logits/rejected": 0.0029922518879175186,
      "logps/chosen": -374.1455993652344,
      "logps/rejected": -492.17547607421875,
      "loss": 0.4817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4430094361305237,
      "rewards/margins": 1.5607372522354126,
      "rewards/rejected": -1.1177277565002441,
      "step": 6413
    },
    {
      "epoch": 0.8393228101742046,
      "grad_norm": 10.5442533493042,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09150122106075287,
      "logits/rejected": 0.08761247992515564,
      "logps/chosen": -404.52630615234375,
      "logps/rejected": -343.37908935546875,
      "loss": 0.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.242933452129364,
      "rewards/margins": 0.6594592928886414,
      "rewards/rejected": -0.41652584075927734,
      "step": 6414
    },
    {
      "epoch": 0.8394536681115564,
      "grad_norm": 10.571423530578613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09113134443759918,
      "logits/rejected": 0.023716257885098457,
      "logps/chosen": -346.8650207519531,
      "logps/rejected": -322.6796875,
      "loss": 0.7609,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3141733407974243,
      "rewards/margins": 0.11133024096488953,
      "rewards/rejected": -0.42550355195999146,
      "step": 6415
    },
    {
      "epoch": 0.8395845260489082,
      "grad_norm": 6.968019962310791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09394639730453491,
      "logits/rejected": -0.17523592710494995,
      "logps/chosen": -397.2384033203125,
      "logps/rejected": -422.0755615234375,
      "loss": 0.5827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03862723708152771,
      "rewards/margins": 0.7102737426757812,
      "rewards/rejected": -0.7489009499549866,
      "step": 6416
    },
    {
      "epoch": 0.8397153839862599,
      "grad_norm": 4.887857913970947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0006998460739850998,
      "logits/rejected": 0.07008300721645355,
      "logps/chosen": -619.2529296875,
      "logps/rejected": -619.7718505859375,
      "loss": 0.2894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9465678930282593,
      "rewards/margins": 1.8652749061584473,
      "rewards/rejected": -0.9187070727348328,
      "step": 6417
    },
    {
      "epoch": 0.8398462419236117,
      "grad_norm": 5.717167377471924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13001874089241028,
      "logits/rejected": -0.3483842611312866,
      "logps/chosen": -493.7925109863281,
      "logps/rejected": -569.3545532226562,
      "loss": 0.355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2573366165161133,
      "rewards/margins": 1.5502500534057617,
      "rewards/rejected": -1.807586669921875,
      "step": 6418
    },
    {
      "epoch": 0.8399770998609635,
      "grad_norm": 5.87646484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0798482596874237,
      "logits/rejected": -0.09259030967950821,
      "logps/chosen": -417.52166748046875,
      "logps/rejected": -438.15692138671875,
      "loss": 0.4583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20239312946796417,
      "rewards/margins": 0.6714480519294739,
      "rewards/rejected": -0.4690549671649933,
      "step": 6419
    },
    {
      "epoch": 0.8401079577983152,
      "grad_norm": 7.729363918304443,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06031684949994087,
      "logits/rejected": -0.06281308829784393,
      "logps/chosen": -700.2405395507812,
      "logps/rejected": -714.0899047851562,
      "loss": 0.3702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5142622590065002,
      "rewards/margins": 1.598160982131958,
      "rewards/rejected": -1.083898663520813,
      "step": 6420
    },
    {
      "epoch": 0.8402388157356669,
      "grad_norm": 8.142982482910156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10178031027317047,
      "logits/rejected": 0.09569092839956284,
      "logps/chosen": -255.74717712402344,
      "logps/rejected": -441.82720947265625,
      "loss": 0.7439,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5726652145385742,
      "rewards/margins": 0.9211726188659668,
      "rewards/rejected": -1.4938379526138306,
      "step": 6421
    },
    {
      "epoch": 0.8403696736730187,
      "grad_norm": 9.408273696899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18849292397499084,
      "logits/rejected": -0.10593156516551971,
      "logps/chosen": -525.3350219726562,
      "logps/rejected": -392.28277587890625,
      "loss": 0.6517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5287666320800781,
      "rewards/margins": 0.9528284072875977,
      "rewards/rejected": -1.4815950393676758,
      "step": 6422
    },
    {
      "epoch": 0.8405005316103705,
      "grad_norm": 8.675249099731445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029272668063640594,
      "logits/rejected": -0.2329137623310089,
      "logps/chosen": -473.9521484375,
      "logps/rejected": -382.5970458984375,
      "loss": 0.591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21894285082817078,
      "rewards/margins": 0.9604547023773193,
      "rewards/rejected": -1.1793975830078125,
      "step": 6423
    },
    {
      "epoch": 0.8406313895477222,
      "grad_norm": 8.4482421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12375349551439285,
      "logits/rejected": -0.15699733793735504,
      "logps/chosen": -395.5152587890625,
      "logps/rejected": -496.46197509765625,
      "loss": 0.6308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11725673824548721,
      "rewards/margins": 0.5384618043899536,
      "rewards/rejected": -0.6557186245918274,
      "step": 6424
    },
    {
      "epoch": 0.840762247485074,
      "grad_norm": 12.05547046661377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1306462436914444,
      "logits/rejected": 0.10400815308094025,
      "logps/chosen": -329.0838317871094,
      "logps/rejected": -423.57952880859375,
      "loss": 1.0669,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15724609792232513,
      "rewards/margins": -0.3669776916503906,
      "rewards/rejected": 0.2097315788269043,
      "step": 6425
    },
    {
      "epoch": 0.8408931054224258,
      "grad_norm": 6.557092189788818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.055853310972452164,
      "logits/rejected": 0.13830913603305817,
      "logps/chosen": -417.459716796875,
      "logps/rejected": -352.1315002441406,
      "loss": 0.5377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45977649092674255,
      "rewards/margins": 0.8814669251441956,
      "rewards/rejected": -0.4216904044151306,
      "step": 6426
    },
    {
      "epoch": 0.8410239633597776,
      "grad_norm": 5.910667419433594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15163026750087738,
      "logits/rejected": -0.06686978042125702,
      "logps/chosen": -489.92584228515625,
      "logps/rejected": -426.4899597167969,
      "loss": 0.3361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.46880650520324707,
      "rewards/margins": 1.4541642665863037,
      "rewards/rejected": -0.9853578805923462,
      "step": 6427
    },
    {
      "epoch": 0.8411548212971293,
      "grad_norm": 9.35188102722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2326863408088684,
      "logits/rejected": -0.4371492862701416,
      "logps/chosen": -456.8118591308594,
      "logps/rejected": -306.5114440917969,
      "loss": 0.7164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37313714623451233,
      "rewards/margins": 0.7101505994796753,
      "rewards/rejected": -1.0832877159118652,
      "step": 6428
    },
    {
      "epoch": 0.841285679234481,
      "grad_norm": 13.564894676208496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011274078860878944,
      "logits/rejected": 0.017947088927030563,
      "logps/chosen": -491.4400634765625,
      "logps/rejected": -455.07952880859375,
      "loss": 1.1523,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.41751694679260254,
      "rewards/margins": -0.710425078868866,
      "rewards/rejected": 0.29290807247161865,
      "step": 6429
    },
    {
      "epoch": 0.8414165371718328,
      "grad_norm": 11.458157539367676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20568616688251495,
      "logits/rejected": 0.10491666197776794,
      "logps/chosen": -330.7082214355469,
      "logps/rejected": -370.9072265625,
      "loss": 0.8845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7044985294342041,
      "rewards/margins": 0.4514204263687134,
      "rewards/rejected": -1.155918836593628,
      "step": 6430
    },
    {
      "epoch": 0.8415473951091846,
      "grad_norm": 3.753227710723877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0789785236120224,
      "logits/rejected": -0.13793738186359406,
      "logps/chosen": -434.8544921875,
      "logps/rejected": -399.0043640136719,
      "loss": 0.2856,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.426203191280365,
      "rewards/margins": 1.9836339950561523,
      "rewards/rejected": -1.5574311017990112,
      "step": 6431
    },
    {
      "epoch": 0.8416782530465363,
      "grad_norm": 6.843465805053711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18856091797351837,
      "logits/rejected": -0.18141520023345947,
      "logps/chosen": -536.0943603515625,
      "logps/rejected": -411.8757019042969,
      "loss": 0.3655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8870325088500977,
      "rewards/margins": 1.1931129693984985,
      "rewards/rejected": -0.3060804605484009,
      "step": 6432
    },
    {
      "epoch": 0.8418091109838881,
      "grad_norm": 5.360570430755615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009459547698497772,
      "logits/rejected": -0.012450795620679855,
      "logps/chosen": -565.641357421875,
      "logps/rejected": -601.4153442382812,
      "loss": 0.3074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0131475925445557,
      "rewards/margins": 2.1115033626556396,
      "rewards/rejected": -1.098355770111084,
      "step": 6433
    },
    {
      "epoch": 0.8419399689212399,
      "grad_norm": 6.795627593994141,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03819917142391205,
      "logits/rejected": -0.11966967582702637,
      "logps/chosen": -308.0706481933594,
      "logps/rejected": -248.84405517578125,
      "loss": 0.6415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3435139060020447,
      "rewards/margins": 0.5909233093261719,
      "rewards/rejected": -0.9344371557235718,
      "step": 6434
    },
    {
      "epoch": 0.8420708268585917,
      "grad_norm": 7.170307159423828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0918673425912857,
      "logits/rejected": 0.0021935035474598408,
      "logps/chosen": -371.0791320800781,
      "logps/rejected": -416.8604431152344,
      "loss": 0.549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002002716064453125,
      "rewards/margins": 0.5532054305076599,
      "rewards/rejected": -0.5552082061767578,
      "step": 6435
    },
    {
      "epoch": 0.8422016847959434,
      "grad_norm": 7.7172346115112305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23894035816192627,
      "logits/rejected": 0.02824399620294571,
      "logps/chosen": -294.6651916503906,
      "logps/rejected": -300.8077087402344,
      "loss": 0.5797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12580834329128265,
      "rewards/margins": 1.038524866104126,
      "rewards/rejected": -1.1643332242965698,
      "step": 6436
    },
    {
      "epoch": 0.8423325427332952,
      "grad_norm": 7.708870887756348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16728045046329498,
      "logits/rejected": 0.022688673809170723,
      "logps/chosen": -388.9587097167969,
      "logps/rejected": -315.01190185546875,
      "loss": 0.7011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11911314725875854,
      "rewards/margins": 0.08728808164596558,
      "rewards/rejected": -0.20640119910240173,
      "step": 6437
    },
    {
      "epoch": 0.842463400670647,
      "grad_norm": 7.04817533493042,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1400081217288971,
      "logits/rejected": -0.0014512091875076294,
      "logps/chosen": -434.7646789550781,
      "logps/rejected": -390.2914123535156,
      "loss": 0.3314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3463398218154907,
      "rewards/margins": 1.3426545858383179,
      "rewards/rejected": -1.6889945268630981,
      "step": 6438
    },
    {
      "epoch": 0.8425942586079986,
      "grad_norm": 7.583162307739258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06839399039745331,
      "logits/rejected": 0.22342732548713684,
      "logps/chosen": -405.78912353515625,
      "logps/rejected": -365.07672119140625,
      "loss": 0.6225,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.09797002375125885,
      "rewards/margins": 0.6180530786514282,
      "rewards/rejected": -0.7160231471061707,
      "step": 6439
    },
    {
      "epoch": 0.8427251165453504,
      "grad_norm": 9.278328895568848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28465136885643005,
      "logits/rejected": -0.3216358423233032,
      "logps/chosen": -423.5328369140625,
      "logps/rejected": -320.60931396484375,
      "loss": 0.7497,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.41899797320365906,
      "rewards/margins": 0.5636457204818726,
      "rewards/rejected": -0.9826436042785645,
      "step": 6440
    },
    {
      "epoch": 0.8428559744827022,
      "grad_norm": 6.828072547912598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04364766180515289,
      "logits/rejected": 0.15274792909622192,
      "logps/chosen": -398.6221618652344,
      "logps/rejected": -357.9188537597656,
      "loss": 0.4972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.41899508237838745,
      "rewards/margins": 0.800611138343811,
      "rewards/rejected": -0.3816160559654236,
      "step": 6441
    },
    {
      "epoch": 0.842986832420054,
      "grad_norm": 8.694315910339355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0015909681096673012,
      "logits/rejected": -0.02597094513475895,
      "logps/chosen": -359.0208435058594,
      "logps/rejected": -401.8398742675781,
      "loss": 0.6488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3335437774658203,
      "rewards/margins": 0.8575552105903625,
      "rewards/rejected": -1.191098928451538,
      "step": 6442
    },
    {
      "epoch": 0.8431176903574057,
      "grad_norm": 12.325352668762207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15002581477165222,
      "logits/rejected": -0.0647619366645813,
      "logps/chosen": -508.5793762207031,
      "logps/rejected": -431.42681884765625,
      "loss": 0.7816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1697218269109726,
      "rewards/margins": 0.3000686764717102,
      "rewards/rejected": -0.1303468644618988,
      "step": 6443
    },
    {
      "epoch": 0.8432485482947575,
      "grad_norm": 17.322324752807617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08287332952022552,
      "logits/rejected": 0.01992497220635414,
      "logps/chosen": -638.2542114257812,
      "logps/rejected": -525.466796875,
      "loss": 0.9241,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5598778128623962,
      "rewards/margins": 0.24758762121200562,
      "rewards/rejected": 0.3122901916503906,
      "step": 6444
    },
    {
      "epoch": 0.8433794062321093,
      "grad_norm": 6.68752384185791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1968632936477661,
      "logits/rejected": -0.007428946904838085,
      "logps/chosen": -477.06488037109375,
      "logps/rejected": -360.00390625,
      "loss": 0.3728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5967437624931335,
      "rewards/margins": 1.039171576499939,
      "rewards/rejected": -0.4424278140068054,
      "step": 6445
    },
    {
      "epoch": 0.8435102641694611,
      "grad_norm": 4.147634983062744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1499955654144287,
      "logits/rejected": -0.04807976633310318,
      "logps/chosen": -539.9401245117188,
      "logps/rejected": -509.6217956542969,
      "loss": 0.2804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3520990014076233,
      "rewards/margins": 1.437967300415039,
      "rewards/rejected": -1.085868239402771,
      "step": 6446
    },
    {
      "epoch": 0.8436411221068127,
      "grad_norm": 8.501558303833008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31262853741645813,
      "logits/rejected": -0.40829265117645264,
      "logps/chosen": -389.89288330078125,
      "logps/rejected": -364.20330810546875,
      "loss": 0.5366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29490986466407776,
      "rewards/margins": 0.9704426527023315,
      "rewards/rejected": -1.265352487564087,
      "step": 6447
    },
    {
      "epoch": 0.8437719800441645,
      "grad_norm": 3.7835569381713867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06691983342170715,
      "logits/rejected": 0.016243819147348404,
      "logps/chosen": -278.6591491699219,
      "logps/rejected": -347.12652587890625,
      "loss": 0.2651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6187523603439331,
      "rewards/margins": 1.4455063343048096,
      "rewards/rejected": -0.8267539143562317,
      "step": 6448
    },
    {
      "epoch": 0.8439028379815163,
      "grad_norm": 4.161825656890869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0508466511964798,
      "logits/rejected": -0.1625167727470398,
      "logps/chosen": -363.2404479980469,
      "logps/rejected": -331.86004638671875,
      "loss": 0.2643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4091739654541016,
      "rewards/margins": 1.4893807172775269,
      "rewards/rejected": -0.08020669966936111,
      "step": 6449
    },
    {
      "epoch": 0.8440336959188681,
      "grad_norm": 8.005547523498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15632730722427368,
      "logits/rejected": 0.1458507478237152,
      "logps/chosen": -374.9404602050781,
      "logps/rejected": -444.6099853515625,
      "loss": 0.6647,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2934581935405731,
      "rewards/margins": 0.3675643503665924,
      "rewards/rejected": -0.6610226035118103,
      "step": 6450
    },
    {
      "epoch": 0.8441645538562198,
      "grad_norm": 7.285689830780029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10133378207683563,
      "logits/rejected": 0.0975581705570221,
      "logps/chosen": -291.4253234863281,
      "logps/rejected": -342.1418151855469,
      "loss": 0.5972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4129096567630768,
      "rewards/margins": 0.6541175842285156,
      "rewards/rejected": -0.24120786786079407,
      "step": 6451
    },
    {
      "epoch": 0.8442954117935716,
      "grad_norm": 6.4953203201293945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07790977507829666,
      "logits/rejected": 0.025866838172078133,
      "logps/chosen": -386.8402099609375,
      "logps/rejected": -369.8310241699219,
      "loss": 0.5861,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6192728877067566,
      "rewards/margins": 0.45350387692451477,
      "rewards/rejected": 0.16576902568340302,
      "step": 6452
    },
    {
      "epoch": 0.8444262697309234,
      "grad_norm": 7.8355913162231445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08642476797103882,
      "logits/rejected": -0.016680214554071426,
      "logps/chosen": -435.0544738769531,
      "logps/rejected": -378.32208251953125,
      "loss": 0.4899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03781837224960327,
      "rewards/margins": 0.775621235370636,
      "rewards/rejected": -0.8134395480155945,
      "step": 6453
    },
    {
      "epoch": 0.8445571276682752,
      "grad_norm": 6.905506134033203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.129495769739151,
      "logits/rejected": 0.03842693567276001,
      "logps/chosen": -345.51544189453125,
      "logps/rejected": -331.8286437988281,
      "loss": 0.593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5719479322433472,
      "rewards/margins": 0.5267457962036133,
      "rewards/rejected": 0.0452021062374115,
      "step": 6454
    },
    {
      "epoch": 0.8446879856056269,
      "grad_norm": 6.263241291046143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21539443731307983,
      "logits/rejected": -0.09165415912866592,
      "logps/chosen": -436.43658447265625,
      "logps/rejected": -499.3423767089844,
      "loss": 0.5135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1317666620016098,
      "rewards/margins": 0.7025764584541321,
      "rewards/rejected": -0.8343430757522583,
      "step": 6455
    },
    {
      "epoch": 0.8448188435429786,
      "grad_norm": 6.447560787200928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11419498920440674,
      "logits/rejected": 0.13237597048282623,
      "logps/chosen": -382.3596496582031,
      "logps/rejected": -448.56134033203125,
      "loss": 0.4413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8121179938316345,
      "rewards/margins": 0.8781282305717468,
      "rewards/rejected": -0.06601028144359589,
      "step": 6456
    },
    {
      "epoch": 0.8449497014803304,
      "grad_norm": 4.661666393280029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1427905559539795,
      "logits/rejected": -0.012332357466220856,
      "logps/chosen": -326.1532897949219,
      "logps/rejected": -311.41455078125,
      "loss": 0.3611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6718869209289551,
      "rewards/margins": 1.3952043056488037,
      "rewards/rejected": -0.7233173847198486,
      "step": 6457
    },
    {
      "epoch": 0.8450805594176822,
      "grad_norm": 6.320516586303711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20105324685573578,
      "logits/rejected": 0.07487177848815918,
      "logps/chosen": -357.18048095703125,
      "logps/rejected": -487.58966064453125,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0998146533966064,
      "rewards/margins": 1.216478705406189,
      "rewards/rejected": -0.1166641116142273,
      "step": 6458
    },
    {
      "epoch": 0.8452114173550339,
      "grad_norm": 9.582273483276367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027871878817677498,
      "logits/rejected": -0.2837599217891693,
      "logps/chosen": -755.2429809570312,
      "logps/rejected": -507.1282043457031,
      "loss": 0.4949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.30296236276626587,
      "rewards/margins": 0.7647562026977539,
      "rewards/rejected": -0.4617938995361328,
      "step": 6459
    },
    {
      "epoch": 0.8453422752923857,
      "grad_norm": 8.436654090881348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.304993599653244,
      "logits/rejected": -0.070657879114151,
      "logps/chosen": -403.2503662109375,
      "logps/rejected": -377.4652099609375,
      "loss": 0.4817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.355821818113327,
      "rewards/margins": 0.6826088428497314,
      "rewards/rejected": -1.0384305715560913,
      "step": 6460
    },
    {
      "epoch": 0.8454731332297375,
      "grad_norm": 6.085206031799316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20869965851306915,
      "logits/rejected": 0.004754897207021713,
      "logps/chosen": -473.523681640625,
      "logps/rejected": -442.2041015625,
      "loss": 0.3829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8544914722442627,
      "rewards/margins": 1.0428802967071533,
      "rewards/rejected": -0.18838882446289062,
      "step": 6461
    },
    {
      "epoch": 0.8456039911670892,
      "grad_norm": 7.344141960144043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12576478719711304,
      "logits/rejected": 0.0849643275141716,
      "logps/chosen": -550.5528564453125,
      "logps/rejected": -444.1693115234375,
      "loss": 0.5445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5924415588378906,
      "rewards/margins": 1.5970425605773926,
      "rewards/rejected": -1.0046008825302124,
      "step": 6462
    },
    {
      "epoch": 0.845734849104441,
      "grad_norm": 9.002397537231445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3105907440185547,
      "logits/rejected": -0.3565666675567627,
      "logps/chosen": -386.4085998535156,
      "logps/rejected": -384.957763671875,
      "loss": 0.6221,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4174959063529968,
      "rewards/margins": 0.3635908365249634,
      "rewards/rejected": -0.781086802482605,
      "step": 6463
    },
    {
      "epoch": 0.8458657070417928,
      "grad_norm": 9.501859664916992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25062358379364014,
      "logits/rejected": 0.1679898500442505,
      "logps/chosen": -520.6697387695312,
      "logps/rejected": -465.62890625,
      "loss": 0.7727,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2703983187675476,
      "rewards/margins": 0.2404499053955078,
      "rewards/rejected": -0.5108482837677002,
      "step": 6464
    },
    {
      "epoch": 0.8459965649791445,
      "grad_norm": 9.717791557312012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22526434063911438,
      "logits/rejected": -0.1959722936153412,
      "logps/chosen": -457.2073974609375,
      "logps/rejected": -568.4036865234375,
      "loss": 0.8412,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11894817650318146,
      "rewards/margins": 0.3031603991985321,
      "rewards/rejected": -0.18421223759651184,
      "step": 6465
    },
    {
      "epoch": 0.8461274229164962,
      "grad_norm": 10.368778228759766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16579054296016693,
      "logits/rejected": 0.07158070057630539,
      "logps/chosen": -346.45440673828125,
      "logps/rejected": -441.55120849609375,
      "loss": 0.957,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.9451122283935547,
      "rewards/margins": 0.08430865406990051,
      "rewards/rejected": 0.8608036041259766,
      "step": 6466
    },
    {
      "epoch": 0.846258280853848,
      "grad_norm": 6.699621677398682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16302981972694397,
      "logits/rejected": 0.12813685834407806,
      "logps/chosen": -527.0458984375,
      "logps/rejected": -391.943359375,
      "loss": 0.4657,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.646757960319519,
      "rewards/margins": 1.4451559782028198,
      "rewards/rejected": -0.7983980178833008,
      "step": 6467
    },
    {
      "epoch": 0.8463891387911998,
      "grad_norm": 7.28128719329834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.061923205852508545,
      "logits/rejected": -0.09489944577217102,
      "logps/chosen": -381.15985107421875,
      "logps/rejected": -431.5979919433594,
      "loss": 0.5377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6574190258979797,
      "rewards/margins": 0.5872714519500732,
      "rewards/rejected": 0.07014749944210052,
      "step": 6468
    },
    {
      "epoch": 0.8465199967285516,
      "grad_norm": 9.716675758361816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26710280776023865,
      "logits/rejected": -0.5058634877204895,
      "logps/chosen": -592.9656372070312,
      "logps/rejected": -511.8602294921875,
      "loss": 0.6601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.128363847732544,
      "rewards/margins": 0.20858418941497803,
      "rewards/rejected": -1.336948037147522,
      "step": 6469
    },
    {
      "epoch": 0.8466508546659033,
      "grad_norm": 9.50059700012207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06718896329402924,
      "logits/rejected": -0.041043348610401154,
      "logps/chosen": -515.9054565429688,
      "logps/rejected": -454.3353576660156,
      "loss": 0.5884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.020450647920370102,
      "rewards/margins": 1.0155105590820312,
      "rewards/rejected": -0.9950599670410156,
      "step": 6470
    },
    {
      "epoch": 0.8467817126032551,
      "grad_norm": 8.15788745880127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30654603242874146,
      "logits/rejected": -0.5707335472106934,
      "logps/chosen": -448.03564453125,
      "logps/rejected": -466.18426513671875,
      "loss": 0.727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3512510657310486,
      "rewards/margins": 0.5527946949005127,
      "rewards/rejected": -0.2015436291694641,
      "step": 6471
    },
    {
      "epoch": 0.8469125705406069,
      "grad_norm": 6.571529865264893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2550974488258362,
      "logits/rejected": 0.0788297951221466,
      "logps/chosen": -417.22113037109375,
      "logps/rejected": -427.0941467285156,
      "loss": 0.4956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5266075134277344,
      "rewards/margins": 1.0156421661376953,
      "rewards/rejected": -0.48903465270996094,
      "step": 6472
    },
    {
      "epoch": 0.8470434284779587,
      "grad_norm": 5.19451379776001,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12120780348777771,
      "logits/rejected": -0.2070980966091156,
      "logps/chosen": -304.7792663574219,
      "logps/rejected": -360.3736267089844,
      "loss": 0.3092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34553566575050354,
      "rewards/margins": 1.0963256359100342,
      "rewards/rejected": -0.7507900595664978,
      "step": 6473
    },
    {
      "epoch": 0.8471742864153103,
      "grad_norm": 8.117053985595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03428029641509056,
      "logits/rejected": -0.2824461758136749,
      "logps/chosen": -466.20458984375,
      "logps/rejected": -511.9939270019531,
      "loss": 0.5701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0254566669464111,
      "rewards/margins": 0.4588760733604431,
      "rewards/rejected": 0.566580593585968,
      "step": 6474
    },
    {
      "epoch": 0.8473051443526621,
      "grad_norm": 10.773052215576172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07493627071380615,
      "logits/rejected": -0.02751888521015644,
      "logps/chosen": -500.109619140625,
      "logps/rejected": -536.8558959960938,
      "loss": 0.6691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06180247664451599,
      "rewards/margins": 0.7166650891304016,
      "rewards/rejected": -0.6548625826835632,
      "step": 6475
    },
    {
      "epoch": 0.8474360022900139,
      "grad_norm": 5.7423248291015625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19895069301128387,
      "logits/rejected": -0.057804256677627563,
      "logps/chosen": -295.7591552734375,
      "logps/rejected": -296.72186279296875,
      "loss": 0.4119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1584489941596985,
      "rewards/margins": 0.9178301095962524,
      "rewards/rejected": -0.7593811750411987,
      "step": 6476
    },
    {
      "epoch": 0.8475668602273657,
      "grad_norm": 3.6448171138763428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4086333215236664,
      "logits/rejected": 0.18915362656116486,
      "logps/chosen": -498.8940124511719,
      "logps/rejected": -436.51837158203125,
      "loss": 0.1924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2661718130111694,
      "rewards/margins": 2.064671277999878,
      "rewards/rejected": -0.798499345779419,
      "step": 6477
    },
    {
      "epoch": 0.8476977181647174,
      "grad_norm": 8.92558765411377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3844301104545593,
      "logits/rejected": 0.4216576814651489,
      "logps/chosen": -395.6316223144531,
      "logps/rejected": -479.44683837890625,
      "loss": 0.745,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.8630588054656982,
      "rewards/margins": 0.7599036693572998,
      "rewards/rejected": 0.10315514355897903,
      "step": 6478
    },
    {
      "epoch": 0.8478285761020692,
      "grad_norm": 7.265335559844971,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17305845022201538,
      "logits/rejected": 0.05232233554124832,
      "logps/chosen": -515.0562744140625,
      "logps/rejected": -509.9075622558594,
      "loss": 0.5051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7670614123344421,
      "rewards/margins": 1.1665152311325073,
      "rewards/rejected": -0.3994537591934204,
      "step": 6479
    },
    {
      "epoch": 0.847959434039421,
      "grad_norm": 3.627340316772461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22487127780914307,
      "logits/rejected": 0.0790676400065422,
      "logps/chosen": -331.13616943359375,
      "logps/rejected": -412.0744323730469,
      "loss": 0.2945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9104040265083313,
      "rewards/margins": 1.2412010431289673,
      "rewards/rejected": -0.3307969868183136,
      "step": 6480
    },
    {
      "epoch": 0.8480902919767728,
      "grad_norm": 4.540770530700684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01965533196926117,
      "logits/rejected": -0.10405132919549942,
      "logps/chosen": -530.1358032226562,
      "logps/rejected": -348.8643798828125,
      "loss": 0.3193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7242656350135803,
      "rewards/margins": 1.2434431314468384,
      "rewards/rejected": -0.5191774368286133,
      "step": 6481
    },
    {
      "epoch": 0.8482211499141245,
      "grad_norm": 5.836579322814941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07875839620828629,
      "logits/rejected": -0.09603369235992432,
      "logps/chosen": -404.8977355957031,
      "logps/rejected": -411.3776550292969,
      "loss": 0.467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3981718122959137,
      "rewards/margins": 1.055843710899353,
      "rewards/rejected": -0.6576719284057617,
      "step": 6482
    },
    {
      "epoch": 0.8483520078514762,
      "grad_norm": 3.59614896774292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047126512974500656,
      "logits/rejected": -0.2361740618944168,
      "logps/chosen": -348.352294921875,
      "logps/rejected": -353.2426452636719,
      "loss": 0.3427,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5249625444412231,
      "rewards/margins": 1.4196758270263672,
      "rewards/rejected": -0.8947132229804993,
      "step": 6483
    },
    {
      "epoch": 0.848482865788828,
      "grad_norm": 5.579249382019043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24594062566757202,
      "logits/rejected": -0.316396564245224,
      "logps/chosen": -518.9793701171875,
      "logps/rejected": -405.1562194824219,
      "loss": 0.3194,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03707028925418854,
      "rewards/margins": 1.5083991289138794,
      "rewards/rejected": -1.4713289737701416,
      "step": 6484
    },
    {
      "epoch": 0.8486137237261797,
      "grad_norm": 4.122128963470459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034842319786548615,
      "logits/rejected": 0.12440667301416397,
      "logps/chosen": -486.6512756347656,
      "logps/rejected": -470.15875244140625,
      "loss": 0.2367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8251938819885254,
      "rewards/margins": 1.5185565948486328,
      "rewards/rejected": -0.6933627724647522,
      "step": 6485
    },
    {
      "epoch": 0.8487445816635315,
      "grad_norm": 11.64332103729248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12110821902751923,
      "logits/rejected": -0.2783718705177307,
      "logps/chosen": -478.4839172363281,
      "logps/rejected": -423.1355895996094,
      "loss": 0.8616,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.32431870698928833,
      "rewards/margins": 0.060138680040836334,
      "rewards/rejected": -0.38445740938186646,
      "step": 6486
    },
    {
      "epoch": 0.8488754396008833,
      "grad_norm": 5.747514724731445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14303357899188995,
      "logits/rejected": -0.2719804644584656,
      "logps/chosen": -454.1925964355469,
      "logps/rejected": -340.1732177734375,
      "loss": 0.3651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2716396152973175,
      "rewards/margins": 1.1247738599777222,
      "rewards/rejected": -0.8531341552734375,
      "step": 6487
    },
    {
      "epoch": 0.8490062975382351,
      "grad_norm": 6.025927543640137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051148369908332825,
      "logits/rejected": -0.19918015599250793,
      "logps/chosen": -494.9051818847656,
      "logps/rejected": -493.502685546875,
      "loss": 0.3875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5499682426452637,
      "rewards/margins": 1.0684138536453247,
      "rewards/rejected": -0.518445611000061,
      "step": 6488
    },
    {
      "epoch": 0.8491371554755868,
      "grad_norm": 6.311607360839844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1307457983493805,
      "logits/rejected": -0.15353775024414062,
      "logps/chosen": -378.2101745605469,
      "logps/rejected": -396.35552978515625,
      "loss": 0.4851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.31301653385162354,
      "rewards/margins": 0.776412308216095,
      "rewards/rejected": -0.4633958041667938,
      "step": 6489
    },
    {
      "epoch": 0.8492680134129386,
      "grad_norm": 6.020872592926025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06652988493442535,
      "logits/rejected": 0.08864286541938782,
      "logps/chosen": -295.3182373046875,
      "logps/rejected": -384.76153564453125,
      "loss": 0.4175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08807753771543503,
      "rewards/margins": 1.0802303552627563,
      "rewards/rejected": -0.9921528100967407,
      "step": 6490
    },
    {
      "epoch": 0.8493988713502904,
      "grad_norm": 13.332636833190918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2077002078294754,
      "logits/rejected": 0.046180643141269684,
      "logps/chosen": -583.3344116210938,
      "logps/rejected": -559.4242553710938,
      "loss": 1.5012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.49627918004989624,
      "rewards/margins": -0.6439718008041382,
      "rewards/rejected": 0.14769266545772552,
      "step": 6491
    },
    {
      "epoch": 0.8495297292876421,
      "grad_norm": 7.278959274291992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07317502796649933,
      "logits/rejected": 0.1197221428155899,
      "logps/chosen": -480.0096435546875,
      "logps/rejected": -528.06396484375,
      "loss": 0.4368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7777659893035889,
      "rewards/margins": 1.4360382556915283,
      "rewards/rejected": -2.213804244995117,
      "step": 6492
    },
    {
      "epoch": 0.8496605872249938,
      "grad_norm": 7.272441864013672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10566370189189911,
      "logits/rejected": -0.3287292718887329,
      "logps/chosen": -414.96484375,
      "logps/rejected": -429.7115173339844,
      "loss": 0.4695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.271200567483902,
      "rewards/margins": 0.6680353283882141,
      "rewards/rejected": -0.9392358660697937,
      "step": 6493
    },
    {
      "epoch": 0.8497914451623456,
      "grad_norm": 8.732261657714844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14136236906051636,
      "logits/rejected": -0.11375134438276291,
      "logps/chosen": -435.8166809082031,
      "logps/rejected": -493.8455810546875,
      "loss": 0.7284,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18332862854003906,
      "rewards/margins": -0.022250741720199585,
      "rewards/rejected": 0.20557937026023865,
      "step": 6494
    },
    {
      "epoch": 0.8499223030996974,
      "grad_norm": 6.412301540374756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19794625043869019,
      "logits/rejected": -0.15506988763809204,
      "logps/chosen": -517.9789428710938,
      "logps/rejected": -354.8331298828125,
      "loss": 0.4449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.48208028078079224,
      "rewards/margins": 1.0097647905349731,
      "rewards/rejected": -0.5276845693588257,
      "step": 6495
    },
    {
      "epoch": 0.8500531610370492,
      "grad_norm": 7.469067096710205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11399907618761063,
      "logits/rejected": -0.2330244481563568,
      "logps/chosen": -582.9990234375,
      "logps/rejected": -638.3819580078125,
      "loss": 0.4795,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1832767277956009,
      "rewards/margins": 1.0872772932052612,
      "rewards/rejected": -1.2705539464950562,
      "step": 6496
    },
    {
      "epoch": 0.8501840189744009,
      "grad_norm": 6.379288196563721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0771656483411789,
      "logits/rejected": -0.14722983539104462,
      "logps/chosen": -547.5055541992188,
      "logps/rejected": -550.2590942382812,
      "loss": 0.3686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.362283706665039,
      "rewards/margins": 1.2994661331176758,
      "rewards/rejected": 0.06281757354736328,
      "step": 6497
    },
    {
      "epoch": 0.8503148769117527,
      "grad_norm": 11.998260498046875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.348302960395813,
      "logits/rejected": -0.0030914321541786194,
      "logps/chosen": -457.87249755859375,
      "logps/rejected": -299.24554443359375,
      "loss": 0.9077,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6279353499412537,
      "rewards/margins": -0.12817516922950745,
      "rewards/rejected": -0.4997600018978119,
      "step": 6498
    },
    {
      "epoch": 0.8504457348491045,
      "grad_norm": 5.335366725921631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18271192908287048,
      "logits/rejected": -0.08380045741796494,
      "logps/chosen": -452.4380187988281,
      "logps/rejected": -459.37408447265625,
      "loss": 0.3007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34800225496292114,
      "rewards/margins": 1.5595024824142456,
      "rewards/rejected": -1.2115002870559692,
      "step": 6499
    },
    {
      "epoch": 0.8505765927864563,
      "grad_norm": 3.8718669414520264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1674901396036148,
      "logits/rejected": -0.22446507215499878,
      "logps/chosen": -364.4671325683594,
      "logps/rejected": -436.32659912109375,
      "loss": 0.2777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.261094868183136,
      "rewards/margins": 1.4011073112487793,
      "rewards/rejected": -1.66220223903656,
      "step": 6500
    },
    {
      "epoch": 0.8507074507238079,
      "grad_norm": 7.925266742706299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22474735975265503,
      "logits/rejected": -0.18102964758872986,
      "logps/chosen": -485.7182922363281,
      "logps/rejected": -345.2596130371094,
      "loss": 0.5608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5007975101470947,
      "rewards/margins": 0.570214033126831,
      "rewards/rejected": -0.06941656768321991,
      "step": 6501
    },
    {
      "epoch": 0.8508383086611597,
      "grad_norm": 7.083011627197266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010254979133605957,
      "logits/rejected": -0.021064892411231995,
      "logps/chosen": -493.6146240234375,
      "logps/rejected": -516.9466552734375,
      "loss": 0.6019,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20378519594669342,
      "rewards/margins": 1.0206708908081055,
      "rewards/rejected": -1.2244559526443481,
      "step": 6502
    },
    {
      "epoch": 0.8509691665985115,
      "grad_norm": 8.410835266113281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18438397347927094,
      "logits/rejected": 0.04555622488260269,
      "logps/chosen": -457.1072082519531,
      "logps/rejected": -344.1593017578125,
      "loss": 0.6687,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.30563467741012573,
      "rewards/margins": 0.6796943545341492,
      "rewards/rejected": -0.37405967712402344,
      "step": 6503
    },
    {
      "epoch": 0.8511000245358633,
      "grad_norm": 6.396984577178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09206438064575195,
      "logits/rejected": -0.20871922373771667,
      "logps/chosen": -610.5195922851562,
      "logps/rejected": -518.3617553710938,
      "loss": 0.347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.39140579104423523,
      "rewards/margins": 1.3281257152557373,
      "rewards/rejected": -0.9367199540138245,
      "step": 6504
    },
    {
      "epoch": 0.851230882473215,
      "grad_norm": 8.967398643493652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11218316853046417,
      "logits/rejected": 0.17203128337860107,
      "logps/chosen": -448.9229431152344,
      "logps/rejected": -357.809814453125,
      "loss": 0.5252,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21099892258644104,
      "rewards/margins": 0.9898296594619751,
      "rewards/rejected": -0.7788307666778564,
      "step": 6505
    },
    {
      "epoch": 0.8513617404105668,
      "grad_norm": 5.277978897094727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10757766664028168,
      "logits/rejected": -0.024883707985281944,
      "logps/chosen": -569.8961791992188,
      "logps/rejected": -481.8943786621094,
      "loss": 0.3401,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30175378918647766,
      "rewards/margins": 1.4270195960998535,
      "rewards/rejected": -1.1252657175064087,
      "step": 6506
    },
    {
      "epoch": 0.8514925983479186,
      "grad_norm": 6.745893955230713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20602947473526,
      "logits/rejected": -0.009539440274238586,
      "logps/chosen": -354.44976806640625,
      "logps/rejected": -390.31640625,
      "loss": 0.4277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6676023602485657,
      "rewards/margins": 0.9804490804672241,
      "rewards/rejected": -0.31284675002098083,
      "step": 6507
    },
    {
      "epoch": 0.8516234562852703,
      "grad_norm": 11.824488639831543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0622420571744442,
      "logits/rejected": -0.1385754942893982,
      "logps/chosen": -321.93084716796875,
      "logps/rejected": -330.637939453125,
      "loss": 0.9456,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.252987265586853,
      "rewards/margins": 0.20087876915931702,
      "rewards/rejected": -0.45386603474617004,
      "step": 6508
    },
    {
      "epoch": 0.851754314222622,
      "grad_norm": 5.09980583190918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19286254048347473,
      "logits/rejected": -0.19370917975902557,
      "logps/chosen": -554.0153198242188,
      "logps/rejected": -452.180419921875,
      "loss": 0.4257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0583043098449707,
      "rewards/margins": 1.5545902252197266,
      "rewards/rejected": -0.49628597497940063,
      "step": 6509
    },
    {
      "epoch": 0.8518851721599738,
      "grad_norm": 7.610482215881348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19004607200622559,
      "logits/rejected": -0.08562591671943665,
      "logps/chosen": -348.896728515625,
      "logps/rejected": -352.9849853515625,
      "loss": 0.5672,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8313609957695007,
      "rewards/margins": 0.9236344695091248,
      "rewards/rejected": -0.09227350354194641,
      "step": 6510
    },
    {
      "epoch": 0.8520160300973256,
      "grad_norm": 8.110445022583008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13012346625328064,
      "logits/rejected": -0.21781045198440552,
      "logps/chosen": -530.7567138671875,
      "logps/rejected": -434.9620361328125,
      "loss": 0.4975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3118702173233032,
      "rewards/margins": 1.1692345142364502,
      "rewards/rejected": -1.4811046123504639,
      "step": 6511
    },
    {
      "epoch": 0.8521468880346773,
      "grad_norm": 3.5082318782806396,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01361340843141079,
      "logits/rejected": -0.0960884541273117,
      "logps/chosen": -218.56849670410156,
      "logps/rejected": -223.62261962890625,
      "loss": 0.2667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.729400098323822,
      "rewards/margins": 1.7650130987167358,
      "rewards/rejected": -1.0356130599975586,
      "step": 6512
    },
    {
      "epoch": 0.8522777459720291,
      "grad_norm": 5.291627407073975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1843118667602539,
      "logits/rejected": 0.046906668692827225,
      "logps/chosen": -531.9466552734375,
      "logps/rejected": -395.95458984375,
      "loss": 0.253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9883979558944702,
      "rewards/margins": 2.2156741619110107,
      "rewards/rejected": -1.2272762060165405,
      "step": 6513
    },
    {
      "epoch": 0.8524086039093809,
      "grad_norm": 8.645584106445312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07358792424201965,
      "logits/rejected": 0.04103504866361618,
      "logps/chosen": -695.9225463867188,
      "logps/rejected": -405.8782958984375,
      "loss": 0.7448,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08572234213352203,
      "rewards/margins": 0.362287700176239,
      "rewards/rejected": -0.2765653729438782,
      "step": 6514
    },
    {
      "epoch": 0.8525394618467327,
      "grad_norm": 10.80269718170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2003287374973297,
      "logits/rejected": -0.17068639397621155,
      "logps/chosen": -562.4889526367188,
      "logps/rejected": -519.98291015625,
      "loss": 0.7183,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21938304603099823,
      "rewards/margins": 0.30891531705856323,
      "rewards/rejected": -0.0895322859287262,
      "step": 6515
    },
    {
      "epoch": 0.8526703197840844,
      "grad_norm": 4.147034645080566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1186506599187851,
      "logits/rejected": -0.15344715118408203,
      "logps/chosen": -382.02044677734375,
      "logps/rejected": -392.1033630371094,
      "loss": 0.2581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0496872663497925,
      "rewards/margins": 1.4663779735565186,
      "rewards/rejected": -0.4166906476020813,
      "step": 6516
    },
    {
      "epoch": 0.8528011777214362,
      "grad_norm": 5.913909435272217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3399140238761902,
      "logits/rejected": 0.2434850037097931,
      "logps/chosen": -374.0019836425781,
      "logps/rejected": -447.4395751953125,
      "loss": 0.4192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19882714748382568,
      "rewards/margins": 1.0334150791168213,
      "rewards/rejected": -0.8345878720283508,
      "step": 6517
    },
    {
      "epoch": 0.852932035658788,
      "grad_norm": 5.115767478942871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1708359718322754,
      "logits/rejected": -0.38672205805778503,
      "logps/chosen": -450.4578857421875,
      "logps/rejected": -453.6424560546875,
      "loss": 0.3524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09732409566640854,
      "rewards/margins": 1.2693302631378174,
      "rewards/rejected": -1.1720062494277954,
      "step": 6518
    },
    {
      "epoch": 0.8530628935961397,
      "grad_norm": 5.411318302154541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08994657546281815,
      "logits/rejected": 0.011531811207532883,
      "logps/chosen": -601.7662963867188,
      "logps/rejected": -370.260498046875,
      "loss": 0.373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9613572359085083,
      "rewards/margins": 1.219980001449585,
      "rewards/rejected": -0.2586227357387543,
      "step": 6519
    },
    {
      "epoch": 0.8531937515334914,
      "grad_norm": 5.570030689239502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04920411854982376,
      "logits/rejected": -0.34175360202789307,
      "logps/chosen": -482.3526611328125,
      "logps/rejected": -493.895751953125,
      "loss": 0.3249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21432320773601532,
      "rewards/margins": 1.6259719133377075,
      "rewards/rejected": -1.4116485118865967,
      "step": 6520
    },
    {
      "epoch": 0.8533246094708432,
      "grad_norm": 8.15761947631836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23053723573684692,
      "logits/rejected": 0.10666363686323166,
      "logps/chosen": -391.8455810546875,
      "logps/rejected": -378.6745300292969,
      "loss": 0.581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2313343584537506,
      "rewards/margins": 0.5866459608078003,
      "rewards/rejected": -0.8179802894592285,
      "step": 6521
    },
    {
      "epoch": 0.853455467408195,
      "grad_norm": 4.903372287750244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06091113016009331,
      "logits/rejected": -0.27234911918640137,
      "logps/chosen": -484.77166748046875,
      "logps/rejected": -798.8602905273438,
      "loss": 0.1617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8161627054214478,
      "rewards/margins": 2.525214910507202,
      "rewards/rejected": -1.7090520858764648,
      "step": 6522
    },
    {
      "epoch": 0.8535863253455468,
      "grad_norm": 4.305377006530762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04360838606953621,
      "logits/rejected": 0.011973443441092968,
      "logps/chosen": -438.867431640625,
      "logps/rejected": -537.373779296875,
      "loss": 0.2796,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0733368396759033,
      "rewards/margins": 1.4329614639282227,
      "rewards/rejected": -0.3596246838569641,
      "step": 6523
    },
    {
      "epoch": 0.8537171832828985,
      "grad_norm": 12.276664733886719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23176486790180206,
      "logits/rejected": -0.17012731730937958,
      "logps/chosen": -579.9803466796875,
      "logps/rejected": -548.3973388671875,
      "loss": 0.8833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2750038206577301,
      "rewards/margins": 0.29521843791007996,
      "rewards/rejected": -0.020214654505252838,
      "step": 6524
    },
    {
      "epoch": 0.8538480412202503,
      "grad_norm": 6.309317588806152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1532803177833557,
      "logits/rejected": 0.018200252205133438,
      "logps/chosen": -511.83160400390625,
      "logps/rejected": -424.1801452636719,
      "loss": 0.4121,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.121774673461914,
      "rewards/margins": 1.3321788311004639,
      "rewards/rejected": -0.21040421724319458,
      "step": 6525
    },
    {
      "epoch": 0.8539788991576021,
      "grad_norm": 7.6188130378723145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29685699939727783,
      "logits/rejected": -0.12063401192426682,
      "logps/chosen": -367.3642578125,
      "logps/rejected": -379.486328125,
      "loss": 0.5505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.014239147305488586,
      "rewards/margins": 0.7765552997589111,
      "rewards/rejected": -0.7907944321632385,
      "step": 6526
    },
    {
      "epoch": 0.8541097570949537,
      "grad_norm": 4.988614082336426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3696291148662567,
      "logits/rejected": -0.07060092687606812,
      "logps/chosen": -362.06024169921875,
      "logps/rejected": -405.2297668457031,
      "loss": 0.3158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04056034982204437,
      "rewards/margins": 1.7168810367584229,
      "rewards/rejected": -1.6763207912445068,
      "step": 6527
    },
    {
      "epoch": 0.8542406150323055,
      "grad_norm": 7.277618408203125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12205161899328232,
      "logits/rejected": 0.013574808835983276,
      "logps/chosen": -368.4404296875,
      "logps/rejected": -456.4996337890625,
      "loss": 0.6265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.303610622882843,
      "rewards/margins": 0.9441896677017212,
      "rewards/rejected": -1.2478002309799194,
      "step": 6528
    },
    {
      "epoch": 0.8543714729696573,
      "grad_norm": 5.134092330932617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15924330055713654,
      "logits/rejected": 0.11883910000324249,
      "logps/chosen": -326.76708984375,
      "logps/rejected": -359.7420349121094,
      "loss": 0.421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.033961296081543,
      "rewards/margins": 0.9144671559333801,
      "rewards/rejected": 0.1194942444562912,
      "step": 6529
    },
    {
      "epoch": 0.8545023309070091,
      "grad_norm": 6.845214366912842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16680556535720825,
      "logits/rejected": -0.07622191309928894,
      "logps/chosen": -547.6713256835938,
      "logps/rejected": -633.0877685546875,
      "loss": 0.5224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36835843324661255,
      "rewards/margins": 1.0794087648391724,
      "rewards/rejected": -0.7110503315925598,
      "step": 6530
    },
    {
      "epoch": 0.8546331888443608,
      "grad_norm": 7.316551685333252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1992870718240738,
      "logits/rejected": -0.1417873203754425,
      "logps/chosen": -527.8394775390625,
      "logps/rejected": -622.3529052734375,
      "loss": 0.485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4174354672431946,
      "rewards/margins": 1.1556812524795532,
      "rewards/rejected": -0.7382457852363586,
      "step": 6531
    },
    {
      "epoch": 0.8547640467817126,
      "grad_norm": 12.088775634765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20118020474910736,
      "logits/rejected": 0.12166787683963776,
      "logps/chosen": -415.6307067871094,
      "logps/rejected": -429.6498718261719,
      "loss": 1.0593,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1363229900598526,
      "rewards/margins": 0.1690814346075058,
      "rewards/rejected": -0.03275832533836365,
      "step": 6532
    },
    {
      "epoch": 0.8548949047190644,
      "grad_norm": 7.382150650024414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11347708851099014,
      "logits/rejected": 0.19530454277992249,
      "logps/chosen": -527.8772583007812,
      "logps/rejected": -559.1626586914062,
      "loss": 0.4352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08055323362350464,
      "rewards/margins": 1.4487477540969849,
      "rewards/rejected": -1.368194580078125,
      "step": 6533
    },
    {
      "epoch": 0.8550257626564162,
      "grad_norm": 3.8691861629486084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048874590545892715,
      "logits/rejected": -0.007124772295355797,
      "logps/chosen": -566.691162109375,
      "logps/rejected": -391.6935119628906,
      "loss": 0.2191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6322864294052124,
      "rewards/margins": 1.667623519897461,
      "rewards/rejected": -1.035336971282959,
      "step": 6534
    },
    {
      "epoch": 0.8551566205937678,
      "grad_norm": 6.381247520446777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49801817536354065,
      "logits/rejected": -0.50697922706604,
      "logps/chosen": -729.1351318359375,
      "logps/rejected": -515.5390014648438,
      "loss": 0.4037,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5591175556182861,
      "rewards/margins": 1.1940383911132812,
      "rewards/rejected": -1.7531558275222778,
      "step": 6535
    },
    {
      "epoch": 0.8552874785311196,
      "grad_norm": 9.494747161865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4201011061668396,
      "logits/rejected": -0.37594476342201233,
      "logps/chosen": -681.833740234375,
      "logps/rejected": -609.6273803710938,
      "loss": 0.5934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1720340996980667,
      "rewards/margins": 1.2221931219100952,
      "rewards/rejected": -1.0501590967178345,
      "step": 6536
    },
    {
      "epoch": 0.8554183364684714,
      "grad_norm": 7.674055576324463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.032607704401016235,
      "logits/rejected": -0.003269962966442108,
      "logps/chosen": -710.1715698242188,
      "logps/rejected": -552.5111083984375,
      "loss": 0.567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3735998272895813,
      "rewards/margins": 0.6124475002288818,
      "rewards/rejected": -0.2388477325439453,
      "step": 6537
    },
    {
      "epoch": 0.8555491944058232,
      "grad_norm": 15.735458374023438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11183977127075195,
      "logits/rejected": 0.16853827238082886,
      "logps/chosen": -380.16680908203125,
      "logps/rejected": -315.8497619628906,
      "loss": 0.9154,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0693622827529907,
      "rewards/margins": 0.07385921478271484,
      "rewards/rejected": -1.1432214975357056,
      "step": 6538
    },
    {
      "epoch": 0.8556800523431749,
      "grad_norm": 5.19061279296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017556609585881233,
      "logits/rejected": 0.0365813747048378,
      "logps/chosen": -426.82855224609375,
      "logps/rejected": -625.5364379882812,
      "loss": 0.313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21948623657226562,
      "rewards/margins": 1.3831143379211426,
      "rewards/rejected": -1.6026004552841187,
      "step": 6539
    },
    {
      "epoch": 0.8558109102805267,
      "grad_norm": 6.379745006561279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17715668678283691,
      "logits/rejected": -0.16710901260375977,
      "logps/chosen": -559.6805419921875,
      "logps/rejected": -547.445068359375,
      "loss": 0.3945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05852164328098297,
      "rewards/margins": 1.2893339395523071,
      "rewards/rejected": -1.2308123111724854,
      "step": 6540
    },
    {
      "epoch": 0.8559417682178785,
      "grad_norm": 5.627618789672852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1005273386836052,
      "logits/rejected": -0.3193380832672119,
      "logps/chosen": -517.3330078125,
      "logps/rejected": -573.95556640625,
      "loss": 0.3302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5469956994056702,
      "rewards/margins": 1.7525478601455688,
      "rewards/rejected": -2.2995431423187256,
      "step": 6541
    },
    {
      "epoch": 0.8560726261552303,
      "grad_norm": 9.04504680633545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09163076430559158,
      "logits/rejected": -0.12159746885299683,
      "logps/chosen": -556.2251586914062,
      "logps/rejected": -704.7646484375,
      "loss": 0.6563,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.8086089491844177,
      "rewards/margins": 1.1238517761230469,
      "rewards/rejected": -0.31524282693862915,
      "step": 6542
    },
    {
      "epoch": 0.856203484092582,
      "grad_norm": 8.241960525512695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06952910125255585,
      "logits/rejected": -0.03986234962940216,
      "logps/chosen": -480.13079833984375,
      "logps/rejected": -405.1174621582031,
      "loss": 0.6336,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6357309818267822,
      "rewards/margins": 0.853253960609436,
      "rewards/rejected": -1.4889849424362183,
      "step": 6543
    },
    {
      "epoch": 0.8563343420299337,
      "grad_norm": 7.11470890045166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23225179314613342,
      "logits/rejected": -0.07939304411411285,
      "logps/chosen": -480.6100769042969,
      "logps/rejected": -507.5693359375,
      "loss": 0.5865,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5199606418609619,
      "rewards/margins": 0.7885518670082092,
      "rewards/rejected": -1.308512568473816,
      "step": 6544
    },
    {
      "epoch": 0.8564651999672855,
      "grad_norm": 8.72246265411377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0003856942057609558,
      "logits/rejected": -0.09798318892717361,
      "logps/chosen": -475.35211181640625,
      "logps/rejected": -435.42047119140625,
      "loss": 0.5545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6896005868911743,
      "rewards/margins": 0.5899244546890259,
      "rewards/rejected": -1.2795250415802002,
      "step": 6545
    },
    {
      "epoch": 0.8565960579046373,
      "grad_norm": 5.103560447692871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02540261670947075,
      "logits/rejected": -0.007293350994586945,
      "logps/chosen": -335.84625244140625,
      "logps/rejected": -683.24169921875,
      "loss": 0.3496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5635170340538025,
      "rewards/margins": 2.257901191711426,
      "rewards/rejected": -1.6943843364715576,
      "step": 6546
    },
    {
      "epoch": 0.856726915841989,
      "grad_norm": 10.483899116516113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27993303537368774,
      "logits/rejected": -0.1284887045621872,
      "logps/chosen": -453.40545654296875,
      "logps/rejected": -576.9739990234375,
      "loss": 0.7627,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5543453097343445,
      "rewards/margins": 0.4065008759498596,
      "rewards/rejected": -0.9608461856842041,
      "step": 6547
    },
    {
      "epoch": 0.8568577737793408,
      "grad_norm": 7.469995021820068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04692002385854721,
      "logits/rejected": -0.008883036673069,
      "logps/chosen": -600.3095092773438,
      "logps/rejected": -462.9750671386719,
      "loss": 0.4418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.41266652941703796,
      "rewards/margins": 1.2858823537826538,
      "rewards/rejected": -0.8732156753540039,
      "step": 6548
    },
    {
      "epoch": 0.8569886317166926,
      "grad_norm": 3.7339303493499756,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16397131979465485,
      "logits/rejected": 0.04524470120668411,
      "logps/chosen": -507.932373046875,
      "logps/rejected": -473.5653991699219,
      "loss": 0.1978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3052048087120056,
      "rewards/margins": 1.7106643915176392,
      "rewards/rejected": -1.4054596424102783,
      "step": 6549
    },
    {
      "epoch": 0.8571194896540443,
      "grad_norm": 5.498486518859863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.042716216295957565,
      "logits/rejected": 0.05992671102285385,
      "logps/chosen": -540.11962890625,
      "logps/rejected": -406.6139831542969,
      "loss": 0.3947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28161829710006714,
      "rewards/margins": 1.4733810424804688,
      "rewards/rejected": -1.7549993991851807,
      "step": 6550
    },
    {
      "epoch": 0.8572503475913961,
      "grad_norm": 3.3592309951782227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18671707808971405,
      "logits/rejected": -0.20772165060043335,
      "logps/chosen": -541.7028198242188,
      "logps/rejected": -532.2330932617188,
      "loss": 0.1747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6456510424613953,
      "rewards/margins": 2.406360387802124,
      "rewards/rejected": -1.760709285736084,
      "step": 6551
    },
    {
      "epoch": 0.8573812055287479,
      "grad_norm": 10.343164443969727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03433023765683174,
      "logits/rejected": 0.18602897226810455,
      "logps/chosen": -500.1060485839844,
      "logps/rejected": -457.15740966796875,
      "loss": 0.8254,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30311548709869385,
      "rewards/margins": -0.10528241842985153,
      "rewards/rejected": -0.19783306121826172,
      "step": 6552
    },
    {
      "epoch": 0.8575120634660996,
      "grad_norm": 4.23867130279541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10131700336933136,
      "logits/rejected": -0.258262574672699,
      "logps/chosen": -256.6978454589844,
      "logps/rejected": -296.0850524902344,
      "loss": 0.3517,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2202441543340683,
      "rewards/margins": 1.537805438041687,
      "rewards/rejected": -1.758049488067627,
      "step": 6553
    },
    {
      "epoch": 0.8576429214034513,
      "grad_norm": 5.704420566558838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30750781297683716,
      "logits/rejected": 0.2125599980354309,
      "logps/chosen": -471.0998229980469,
      "logps/rejected": -437.615234375,
      "loss": 0.391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22407591342926025,
      "rewards/margins": 1.0902018547058105,
      "rewards/rejected": -1.3142776489257812,
      "step": 6554
    },
    {
      "epoch": 0.8577737793408031,
      "grad_norm": 6.655417442321777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030984530225396156,
      "logits/rejected": 0.26562759280204773,
      "logps/chosen": -413.0603942871094,
      "logps/rejected": -483.1163330078125,
      "loss": 0.4032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7102810144424438,
      "rewards/margins": 0.8038622140884399,
      "rewards/rejected": -1.5141432285308838,
      "step": 6555
    },
    {
      "epoch": 0.8579046372781549,
      "grad_norm": 7.380214214324951,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20177513360977173,
      "logits/rejected": 0.19241131842136383,
      "logps/chosen": -544.8793334960938,
      "logps/rejected": -563.385009765625,
      "loss": 0.5112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8444460034370422,
      "rewards/margins": 1.3512983322143555,
      "rewards/rejected": -0.506852388381958,
      "step": 6556
    },
    {
      "epoch": 0.8580354952155067,
      "grad_norm": 8.273394584655762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015782833099365234,
      "logits/rejected": -0.041691508144140244,
      "logps/chosen": -455.5845947265625,
      "logps/rejected": -414.8072204589844,
      "loss": 0.4433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2706257104873657,
      "rewards/margins": 1.580159306526184,
      "rewards/rejected": -1.3095335960388184,
      "step": 6557
    },
    {
      "epoch": 0.8581663531528584,
      "grad_norm": 9.804829597473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2855302095413208,
      "logits/rejected": 0.16789427399635315,
      "logps/chosen": -518.0078125,
      "logps/rejected": -464.627197265625,
      "loss": 0.6344,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06080852448940277,
      "rewards/margins": 0.4256255030632019,
      "rewards/rejected": -0.3648170232772827,
      "step": 6558
    },
    {
      "epoch": 0.8582972110902102,
      "grad_norm": 5.567107200622559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16911721229553223,
      "logits/rejected": -0.38183948397636414,
      "logps/chosen": -543.5408935546875,
      "logps/rejected": -302.6457824707031,
      "loss": 0.4449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2396196722984314,
      "rewards/margins": 1.1844879388809204,
      "rewards/rejected": -1.4241076707839966,
      "step": 6559
    },
    {
      "epoch": 0.858428069027562,
      "grad_norm": 4.518715858459473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25699472427368164,
      "logits/rejected": -0.14851218461990356,
      "logps/chosen": -441.031982421875,
      "logps/rejected": -337.95672607421875,
      "loss": 0.2444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.030944442376494408,
      "rewards/margins": 2.1125648021698,
      "rewards/rejected": -2.081620216369629,
      "step": 6560
    },
    {
      "epoch": 0.8585589269649138,
      "grad_norm": 5.030114650726318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3365430533885956,
      "logits/rejected": 0.003086462616920471,
      "logps/chosen": -532.8839721679688,
      "logps/rejected": -313.3633117675781,
      "loss": 0.3287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7117700576782227,
      "rewards/margins": 1.3683981895446777,
      "rewards/rejected": -0.6566280722618103,
      "step": 6561
    },
    {
      "epoch": 0.8586897849022654,
      "grad_norm": 8.067928314208984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12997572124004364,
      "logits/rejected": 0.1710679680109024,
      "logps/chosen": -601.9165649414062,
      "logps/rejected": -556.52392578125,
      "loss": 0.4985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12511025369167328,
      "rewards/margins": 0.978398323059082,
      "rewards/rejected": -1.1035085916519165,
      "step": 6562
    },
    {
      "epoch": 0.8588206428396172,
      "grad_norm": 5.551528453826904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05024845153093338,
      "logits/rejected": -0.1420411467552185,
      "logps/chosen": -541.8630981445312,
      "logps/rejected": -512.5987548828125,
      "loss": 0.3762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4908060133457184,
      "rewards/margins": 1.2669082880020142,
      "rewards/rejected": -0.7761022448539734,
      "step": 6563
    },
    {
      "epoch": 0.858951500776969,
      "grad_norm": 8.195897102355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07108990848064423,
      "logits/rejected": -0.2227974236011505,
      "logps/chosen": -357.7538146972656,
      "logps/rejected": -418.65203857421875,
      "loss": 0.4814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10747206211090088,
      "rewards/margins": 1.464524745941162,
      "rewards/rejected": -1.571996808052063,
      "step": 6564
    },
    {
      "epoch": 0.8590823587143208,
      "grad_norm": 4.646951198577881,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12232854962348938,
      "logits/rejected": -0.04709620028734207,
      "logps/chosen": -371.6094970703125,
      "logps/rejected": -415.3877258300781,
      "loss": 0.2641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40354251861572266,
      "rewards/margins": 1.2322022914886475,
      "rewards/rejected": -0.8286596536636353,
      "step": 6565
    },
    {
      "epoch": 0.8592132166516725,
      "grad_norm": 8.85440444946289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1251073181629181,
      "logits/rejected": -0.02519327402114868,
      "logps/chosen": -412.2090759277344,
      "logps/rejected": -359.8838195800781,
      "loss": 0.627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01631355658173561,
      "rewards/margins": 0.2458614706993103,
      "rewards/rejected": -0.2621749937534332,
      "step": 6566
    },
    {
      "epoch": 0.8593440745890243,
      "grad_norm": 5.328228950500488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17134137451648712,
      "logits/rejected": -0.20393560826778412,
      "logps/chosen": -340.60748291015625,
      "logps/rejected": -376.18310546875,
      "loss": 0.2698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09375858306884766,
      "rewards/margins": 1.4936082363128662,
      "rewards/rejected": -1.3998496532440186,
      "step": 6567
    },
    {
      "epoch": 0.8594749325263761,
      "grad_norm": 10.859956741333008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1266985833644867,
      "logits/rejected": -0.052642546594142914,
      "logps/chosen": -500.18121337890625,
      "logps/rejected": -310.2236022949219,
      "loss": 0.8016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7393088340759277,
      "rewards/margins": 0.12884864211082458,
      "rewards/rejected": -0.8681575059890747,
      "step": 6568
    },
    {
      "epoch": 0.8596057904637279,
      "grad_norm": 4.7670512199401855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17448100447654724,
      "logits/rejected": 0.24338530004024506,
      "logps/chosen": -346.0850524902344,
      "logps/rejected": -383.9318542480469,
      "loss": 0.3459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47648337483406067,
      "rewards/margins": 1.3123568296432495,
      "rewards/rejected": -0.8358734846115112,
      "step": 6569
    },
    {
      "epoch": 0.8597366484010796,
      "grad_norm": 5.994866847991943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16366751492023468,
      "logits/rejected": 0.04551875963807106,
      "logps/chosen": -389.1066589355469,
      "logps/rejected": -462.35479736328125,
      "loss": 0.413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3049529790878296,
      "rewards/margins": 1.0434939861297607,
      "rewards/rejected": -1.3484470844268799,
      "step": 6570
    },
    {
      "epoch": 0.8598675063384313,
      "grad_norm": 7.202552318572998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23482440412044525,
      "logits/rejected": 0.033425427973270416,
      "logps/chosen": -501.5541687011719,
      "logps/rejected": -595.8798828125,
      "loss": 0.4143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4650025963783264,
      "rewards/margins": 1.300666093826294,
      "rewards/rejected": -0.8356634974479675,
      "step": 6571
    },
    {
      "epoch": 0.8599983642757831,
      "grad_norm": 8.06840705871582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27573344111442566,
      "logits/rejected": 0.026475831866264343,
      "logps/chosen": -502.5715637207031,
      "logps/rejected": -623.312255859375,
      "loss": 0.4962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8998783826828003,
      "rewards/margins": 0.6879028677940369,
      "rewards/rejected": -1.5877810716629028,
      "step": 6572
    },
    {
      "epoch": 0.8601292222131348,
      "grad_norm": 12.360618591308594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037725597620010376,
      "logits/rejected": -0.09007886052131653,
      "logps/chosen": -427.9316101074219,
      "logps/rejected": -308.9281005859375,
      "loss": 0.5594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0668586939573288,
      "rewards/margins": 0.7202016711235046,
      "rewards/rejected": -0.7870603799819946,
      "step": 6573
    },
    {
      "epoch": 0.8602600801504866,
      "grad_norm": 5.322518825531006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07386220991611481,
      "logits/rejected": -0.250438928604126,
      "logps/chosen": -374.4330749511719,
      "logps/rejected": -399.526611328125,
      "loss": 0.3723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12892207503318787,
      "rewards/margins": 1.2703605890274048,
      "rewards/rejected": -1.399282693862915,
      "step": 6574
    },
    {
      "epoch": 0.8603909380878384,
      "grad_norm": 4.915608882904053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19422410428524017,
      "logits/rejected": 0.020530492067337036,
      "logps/chosen": -567.6910400390625,
      "logps/rejected": -507.9195556640625,
      "loss": 0.4548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11105839908123016,
      "rewards/margins": 1.7816917896270752,
      "rewards/rejected": -1.6706335544586182,
      "step": 6575
    },
    {
      "epoch": 0.8605217960251902,
      "grad_norm": 7.968474864959717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14774170517921448,
      "logits/rejected": -0.017097700387239456,
      "logps/chosen": -441.86676025390625,
      "logps/rejected": -434.95758056640625,
      "loss": 0.5602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9576497673988342,
      "rewards/margins": 0.8224157094955444,
      "rewards/rejected": -1.7800655364990234,
      "step": 6576
    },
    {
      "epoch": 0.8606526539625419,
      "grad_norm": 11.67325496673584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03048209473490715,
      "logits/rejected": 0.0032954253256320953,
      "logps/chosen": -383.6852111816406,
      "logps/rejected": -447.88641357421875,
      "loss": 0.9575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7538626194000244,
      "rewards/margins": 0.02820989489555359,
      "rewards/rejected": -0.7820724248886108,
      "step": 6577
    },
    {
      "epoch": 0.8607835118998937,
      "grad_norm": 5.286107063293457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10480041056871414,
      "logits/rejected": -0.09415988624095917,
      "logps/chosen": -490.6902770996094,
      "logps/rejected": -441.7845153808594,
      "loss": 0.289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10838697850704193,
      "rewards/margins": 1.8538457155227661,
      "rewards/rejected": -1.9622328281402588,
      "step": 6578
    },
    {
      "epoch": 0.8609143698372455,
      "grad_norm": 16.134113311767578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20080506801605225,
      "logits/rejected": -0.03819546103477478,
      "logps/chosen": -662.595458984375,
      "logps/rejected": -385.31854248046875,
      "loss": 1.1237,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3474195599555969,
      "rewards/margins": 0.28020259737968445,
      "rewards/rejected": -0.6276222467422485,
      "step": 6579
    },
    {
      "epoch": 0.8610452277745972,
      "grad_norm": 8.064016342163086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020908549427986145,
      "logits/rejected": 0.1934865415096283,
      "logps/chosen": -491.6338806152344,
      "logps/rejected": -439.6961669921875,
      "loss": 0.7392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7642072439193726,
      "rewards/margins": 0.22882291674613953,
      "rewards/rejected": -0.9930302500724792,
      "step": 6580
    },
    {
      "epoch": 0.8611760857119489,
      "grad_norm": 6.489531517028809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28342166543006897,
      "logits/rejected": -0.30637747049331665,
      "logps/chosen": -466.506591796875,
      "logps/rejected": -415.6796569824219,
      "loss": 0.3988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47609251737594604,
      "rewards/margins": 1.2493102550506592,
      "rewards/rejected": -0.7732177972793579,
      "step": 6581
    },
    {
      "epoch": 0.8613069436493007,
      "grad_norm": 8.307204246520996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24160325527191162,
      "logits/rejected": -0.06407218426465988,
      "logps/chosen": -374.19561767578125,
      "logps/rejected": -427.0985412597656,
      "loss": 0.7118,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.02583179622888565,
      "rewards/margins": 0.08410093188285828,
      "rewards/rejected": -0.058269135653972626,
      "step": 6582
    },
    {
      "epoch": 0.8614378015866525,
      "grad_norm": 5.739060401916504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03186619281768799,
      "logits/rejected": -0.03013443946838379,
      "logps/chosen": -359.0570983886719,
      "logps/rejected": -259.14459228515625,
      "loss": 0.5515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10974717140197754,
      "rewards/margins": 0.7131890058517456,
      "rewards/rejected": -0.6034418344497681,
      "step": 6583
    },
    {
      "epoch": 0.8615686595240043,
      "grad_norm": 10.199909210205078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13788506388664246,
      "logits/rejected": -0.15005196630954742,
      "logps/chosen": -403.433837890625,
      "logps/rejected": -410.05987548828125,
      "loss": 0.7583,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.33353862166404724,
      "rewards/margins": 0.20270098745822906,
      "rewards/rejected": -0.5362396240234375,
      "step": 6584
    },
    {
      "epoch": 0.861699517461356,
      "grad_norm": 7.319482326507568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16612905263900757,
      "logits/rejected": -0.24271436035633087,
      "logps/chosen": -643.097900390625,
      "logps/rejected": -475.9394226074219,
      "loss": 0.3078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6551309823989868,
      "rewards/margins": 1.390721082687378,
      "rewards/rejected": -0.7355902194976807,
      "step": 6585
    },
    {
      "epoch": 0.8618303753987078,
      "grad_norm": 6.186734199523926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14706163108348846,
      "logits/rejected": 0.11800447106361389,
      "logps/chosen": -498.2858581542969,
      "logps/rejected": -523.4546508789062,
      "loss": 0.4327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4054117202758789,
      "rewards/margins": 1.058212161064148,
      "rewards/rejected": -0.6528003811836243,
      "step": 6586
    },
    {
      "epoch": 0.8619612333360596,
      "grad_norm": 9.47866439819336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2078285962343216,
      "logits/rejected": -0.15598563849925995,
      "logps/chosen": -412.706787109375,
      "logps/rejected": -463.0263366699219,
      "loss": 0.7871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31756460666656494,
      "rewards/margins": 0.36167603731155396,
      "rewards/rejected": -0.6792406439781189,
      "step": 6587
    },
    {
      "epoch": 0.8620920912734114,
      "grad_norm": 3.256237506866455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1552865207195282,
      "logits/rejected": 0.12694090604782104,
      "logps/chosen": -458.7359313964844,
      "logps/rejected": -384.99432373046875,
      "loss": 0.2466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7658913135528564,
      "rewards/margins": 1.7971267700195312,
      "rewards/rejected": -1.0312353372573853,
      "step": 6588
    },
    {
      "epoch": 0.862222949210763,
      "grad_norm": 3.8983747959136963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15007230639457703,
      "logits/rejected": -0.11875872313976288,
      "logps/chosen": -344.6056823730469,
      "logps/rejected": -394.8201904296875,
      "loss": 0.3204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01972055435180664,
      "rewards/margins": 1.9426685571670532,
      "rewards/rejected": -1.962389349937439,
      "step": 6589
    },
    {
      "epoch": 0.8623538071481148,
      "grad_norm": 5.59372091293335,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05932120978832245,
      "logits/rejected": -0.07959118485450745,
      "logps/chosen": -411.4917297363281,
      "logps/rejected": -497.152587890625,
      "loss": 0.2967,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0440069437026978,
      "rewards/margins": 1.6941332817077637,
      "rewards/rejected": -2.738140344619751,
      "step": 6590
    },
    {
      "epoch": 0.8624846650854666,
      "grad_norm": 7.195614814758301,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1948593407869339,
      "logits/rejected": -0.0006326003931462765,
      "logps/chosen": -381.8884582519531,
      "logps/rejected": -424.3997802734375,
      "loss": 0.5955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28555870056152344,
      "rewards/margins": 0.6240859031677246,
      "rewards/rejected": -0.9096447229385376,
      "step": 6591
    },
    {
      "epoch": 0.8626155230228183,
      "grad_norm": 7.725411891937256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1643999218940735,
      "logits/rejected": -0.02372690849006176,
      "logps/chosen": -405.74591064453125,
      "logps/rejected": -397.34912109375,
      "loss": 0.534,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2046356350183487,
      "rewards/margins": 1.0659089088439941,
      "rewards/rejected": -1.2705445289611816,
      "step": 6592
    },
    {
      "epoch": 0.8627463809601701,
      "grad_norm": 6.436064720153809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10354197025299072,
      "logits/rejected": -0.011903139762580395,
      "logps/chosen": -442.82025146484375,
      "logps/rejected": -377.4232177734375,
      "loss": 0.3869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0013017654418945,
      "rewards/margins": 1.4454338550567627,
      "rewards/rejected": -2.4467356204986572,
      "step": 6593
    },
    {
      "epoch": 0.8628772388975219,
      "grad_norm": 4.30103874206543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11381883174180984,
      "logits/rejected": -0.13753525912761688,
      "logps/chosen": -568.14599609375,
      "logps/rejected": -518.80126953125,
      "loss": 0.2587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3158947229385376,
      "rewards/margins": 1.6252683401107788,
      "rewards/rejected": -1.3093734979629517,
      "step": 6594
    },
    {
      "epoch": 0.8630080968348737,
      "grad_norm": 10.333880424499512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004554171115159988,
      "logits/rejected": 0.040979597717523575,
      "logps/chosen": -505.07159423828125,
      "logps/rejected": -488.57147216796875,
      "loss": 0.708,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.44057750701904297,
      "rewards/margins": 0.2509623169898987,
      "rewards/rejected": -0.6915397644042969,
      "step": 6595
    },
    {
      "epoch": 0.8631389547722254,
      "grad_norm": 3.0853376388549805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08554314076900482,
      "logits/rejected": -0.3505439758300781,
      "logps/chosen": -380.90521240234375,
      "logps/rejected": -380.5644226074219,
      "loss": 0.1798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39316102862358093,
      "rewards/margins": 2.4486732482910156,
      "rewards/rejected": -2.0555121898651123,
      "step": 6596
    },
    {
      "epoch": 0.8632698127095771,
      "grad_norm": 5.508382320404053,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13540759682655334,
      "logits/rejected": -0.0990663468837738,
      "logps/chosen": -589.7745361328125,
      "logps/rejected": -391.27935791015625,
      "loss": 0.2501,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19393140077590942,
      "rewards/margins": 1.9661493301391602,
      "rewards/rejected": -2.1600804328918457,
      "step": 6597
    },
    {
      "epoch": 0.8634006706469289,
      "grad_norm": 6.195279598236084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10528777539730072,
      "logits/rejected": -0.10692526400089264,
      "logps/chosen": -312.5063781738281,
      "logps/rejected": -340.46649169921875,
      "loss": 0.4223,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16019269824028015,
      "rewards/margins": 1.1639066934585571,
      "rewards/rejected": -1.3240994215011597,
      "step": 6598
    },
    {
      "epoch": 0.8635315285842807,
      "grad_norm": 8.360343933105469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06467459350824356,
      "logits/rejected": 0.027197346091270447,
      "logps/chosen": -441.5699462890625,
      "logps/rejected": -468.2239990234375,
      "loss": 0.5855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4490361213684082,
      "rewards/margins": 0.7805315256118774,
      "rewards/rejected": -2.229567527770996,
      "step": 6599
    },
    {
      "epoch": 0.8636623865216324,
      "grad_norm": 8.313883781433105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011575907468795776,
      "logits/rejected": -4.634261131286621e-06,
      "logps/chosen": -469.38665771484375,
      "logps/rejected": -345.5371398925781,
      "loss": 0.5693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08920900523662567,
      "rewards/margins": 1.0830551385879517,
      "rewards/rejected": -1.1722640991210938,
      "step": 6600
    },
    {
      "epoch": 0.8637932444589842,
      "grad_norm": 6.662261009216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40923845767974854,
      "logits/rejected": -0.36797329783439636,
      "logps/chosen": -362.9193115234375,
      "logps/rejected": -517.6757202148438,
      "loss": 0.2663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21815072000026703,
      "rewards/margins": 1.6769530773162842,
      "rewards/rejected": -1.4588024616241455,
      "step": 6601
    },
    {
      "epoch": 0.863924102396336,
      "grad_norm": 8.040765762329102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2407236546278,
      "logits/rejected": 0.06738924980163574,
      "logps/chosen": -770.7850341796875,
      "logps/rejected": -582.8549194335938,
      "loss": 0.5242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6915697455406189,
      "rewards/margins": 1.1716362237930298,
      "rewards/rejected": -0.4800665080547333,
      "step": 6602
    },
    {
      "epoch": 0.8640549603336878,
      "grad_norm": 4.527759075164795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07964282482862473,
      "logits/rejected": -0.15289521217346191,
      "logps/chosen": -504.09564208984375,
      "logps/rejected": -391.537841796875,
      "loss": 0.3684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4876464903354645,
      "rewards/margins": 1.1219054460525513,
      "rewards/rejected": -1.6095519065856934,
      "step": 6603
    },
    {
      "epoch": 0.8641858182710395,
      "grad_norm": 8.679171562194824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13235530257225037,
      "logits/rejected": 0.008762212470173836,
      "logps/chosen": -578.675537109375,
      "logps/rejected": -645.9649047851562,
      "loss": 0.4391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23716622591018677,
      "rewards/margins": 1.29637610912323,
      "rewards/rejected": -1.0592098236083984,
      "step": 6604
    },
    {
      "epoch": 0.8643166762083913,
      "grad_norm": 7.40635347366333,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007823440246284008,
      "logits/rejected": 0.10425623506307602,
      "logps/chosen": -453.3674621582031,
      "logps/rejected": -354.7033386230469,
      "loss": 0.4994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1533471941947937,
      "rewards/margins": 1.0839433670043945,
      "rewards/rejected": -0.9305961728096008,
      "step": 6605
    },
    {
      "epoch": 0.864447534145743,
      "grad_norm": 8.263897895812988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00250842422246933,
      "logits/rejected": 0.15092350542545319,
      "logps/chosen": -426.7980651855469,
      "logps/rejected": -362.1112976074219,
      "loss": 0.4776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3525138795375824,
      "rewards/margins": 1.1207342147827148,
      "rewards/rejected": -1.47324800491333,
      "step": 6606
    },
    {
      "epoch": 0.8645783920830948,
      "grad_norm": 8.137027740478516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2271052598953247,
      "logits/rejected": 0.17721010744571686,
      "logps/chosen": -448.9148254394531,
      "logps/rejected": -383.8692932128906,
      "loss": 0.4976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6250886917114258,
      "rewards/margins": 0.7375032901763916,
      "rewards/rejected": -1.3625919818878174,
      "step": 6607
    },
    {
      "epoch": 0.8647092500204465,
      "grad_norm": 7.725497245788574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00998910516500473,
      "logits/rejected": -0.03215968608856201,
      "logps/chosen": -613.9893798828125,
      "logps/rejected": -541.560791015625,
      "loss": 0.3468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09443360567092896,
      "rewards/margins": 1.9751125574111938,
      "rewards/rejected": -1.8806791305541992,
      "step": 6608
    },
    {
      "epoch": 0.8648401079577983,
      "grad_norm": 10.592538833618164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15581662952899933,
      "logits/rejected": 0.08964397758245468,
      "logps/chosen": -472.560791015625,
      "logps/rejected": -519.5765380859375,
      "loss": 0.9048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5230059027671814,
      "rewards/margins": -0.01930619776248932,
      "rewards/rejected": -0.503699779510498,
      "step": 6609
    },
    {
      "epoch": 0.8649709658951501,
      "grad_norm": 6.52210807800293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1686917245388031,
      "logits/rejected": -0.0679827481508255,
      "logps/chosen": -271.7300109863281,
      "logps/rejected": -317.0512390136719,
      "loss": 0.4257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.639130175113678,
      "rewards/margins": 0.916822612285614,
      "rewards/rejected": -0.27769243717193604,
      "step": 6610
    },
    {
      "epoch": 0.8651018238325019,
      "grad_norm": 7.1176228523254395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2255735844373703,
      "logits/rejected": -0.31023868918418884,
      "logps/chosen": -271.93145751953125,
      "logps/rejected": -282.5607604980469,
      "loss": 0.5937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10572107136249542,
      "rewards/margins": 0.9148039817810059,
      "rewards/rejected": -1.0205249786376953,
      "step": 6611
    },
    {
      "epoch": 0.8652326817698536,
      "grad_norm": 7.9274773597717285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16085386276245117,
      "logits/rejected": 0.2383953481912613,
      "logps/chosen": -446.08984375,
      "logps/rejected": -440.26251220703125,
      "loss": 0.7116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17497482895851135,
      "rewards/margins": 0.46610885858535767,
      "rewards/rejected": -0.6410837769508362,
      "step": 6612
    },
    {
      "epoch": 0.8653635397072054,
      "grad_norm": 9.093490600585938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12070830166339874,
      "logits/rejected": -0.033548563718795776,
      "logps/chosen": -367.20538330078125,
      "logps/rejected": -360.0263977050781,
      "loss": 0.5289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0268789529800415,
      "rewards/margins": 0.533186674118042,
      "rewards/rejected": -1.5600656270980835,
      "step": 6613
    },
    {
      "epoch": 0.8654943976445572,
      "grad_norm": 8.537830352783203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07393184304237366,
      "logits/rejected": 0.001874449197202921,
      "logps/chosen": -519.8941650390625,
      "logps/rejected": -571.0575561523438,
      "loss": 0.5325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6300762891769409,
      "rewards/margins": 0.5623594522476196,
      "rewards/rejected": 0.06771677732467651,
      "step": 6614
    },
    {
      "epoch": 0.8656252555819088,
      "grad_norm": 8.672403335571289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1461467444896698,
      "logits/rejected": -0.2273191213607788,
      "logps/chosen": -435.4129638671875,
      "logps/rejected": -494.57879638671875,
      "loss": 0.5849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1545814424753189,
      "rewards/margins": 1.0084490776062012,
      "rewards/rejected": -0.8538677096366882,
      "step": 6615
    },
    {
      "epoch": 0.8657561135192606,
      "grad_norm": 4.785681247711182,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3812626600265503,
      "logits/rejected": 0.05024499446153641,
      "logps/chosen": -348.4804992675781,
      "logps/rejected": -346.5142822265625,
      "loss": 0.3628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16233983635902405,
      "rewards/margins": 0.9613245129585266,
      "rewards/rejected": -0.7989847660064697,
      "step": 6616
    },
    {
      "epoch": 0.8658869714566124,
      "grad_norm": 8.878763198852539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05104193091392517,
      "logits/rejected": 0.16148503124713898,
      "logps/chosen": -572.81005859375,
      "logps/rejected": -493.55450439453125,
      "loss": 0.5231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4820396602153778,
      "rewards/margins": 0.8928828835487366,
      "rewards/rejected": -1.3749226331710815,
      "step": 6617
    },
    {
      "epoch": 0.8660178293939642,
      "grad_norm": 8.91905689239502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14899516105651855,
      "logits/rejected": -0.04928058385848999,
      "logps/chosen": -428.01507568359375,
      "logps/rejected": -318.9626159667969,
      "loss": 0.5181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007899880409240723,
      "rewards/margins": 1.6714192628860474,
      "rewards/rejected": -1.6793190240859985,
      "step": 6618
    },
    {
      "epoch": 0.8661486873313159,
      "grad_norm": 6.483880996704102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38959747552871704,
      "logits/rejected": 0.02338726818561554,
      "logps/chosen": -336.0242919921875,
      "logps/rejected": -362.055908203125,
      "loss": 0.4901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0921953022480011,
      "rewards/margins": 0.8761411905288696,
      "rewards/rejected": -0.7839458584785461,
      "step": 6619
    },
    {
      "epoch": 0.8662795452686677,
      "grad_norm": 5.5905985832214355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2632561922073364,
      "logits/rejected": 0.18665343523025513,
      "logps/chosen": -637.046630859375,
      "logps/rejected": -546.6641235351562,
      "loss": 0.2669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12111780047416687,
      "rewards/margins": 1.6839308738708496,
      "rewards/rejected": -1.8050484657287598,
      "step": 6620
    },
    {
      "epoch": 0.8664104032060195,
      "grad_norm": 4.444977283477783,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016521096229553223,
      "logits/rejected": -0.09558380395174026,
      "logps/chosen": -440.03057861328125,
      "logps/rejected": -387.7886962890625,
      "loss": 0.3687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3002195358276367,
      "rewards/margins": 1.4262197017669678,
      "rewards/rejected": -1.1260000467300415,
      "step": 6621
    },
    {
      "epoch": 0.8665412611433713,
      "grad_norm": 5.661877155303955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17554624378681183,
      "logits/rejected": -0.015305042266845703,
      "logps/chosen": -360.8156433105469,
      "logps/rejected": -324.9444885253906,
      "loss": 0.3663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8692852258682251,
      "rewards/margins": 1.6246744394302368,
      "rewards/rejected": -2.493959665298462,
      "step": 6622
    },
    {
      "epoch": 0.866672119080723,
      "grad_norm": 10.533273696899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24883806705474854,
      "logits/rejected": -0.33713045716285706,
      "logps/chosen": -385.6632385253906,
      "logps/rejected": -409.1431884765625,
      "loss": 0.8038,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.06946660578250885,
      "rewards/margins": 0.08649121969938278,
      "rewards/rejected": -0.15595778822898865,
      "step": 6623
    },
    {
      "epoch": 0.8668029770180747,
      "grad_norm": 7.0940470695495605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16067972779273987,
      "logits/rejected": 0.38802945613861084,
      "logps/chosen": -235.11471557617188,
      "logps/rejected": -326.2969055175781,
      "loss": 0.5901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10541513562202454,
      "rewards/margins": 0.567233681678772,
      "rewards/rejected": -0.46181851625442505,
      "step": 6624
    },
    {
      "epoch": 0.8669338349554265,
      "grad_norm": 11.671278953552246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13706904649734497,
      "logits/rejected": -0.044722817838191986,
      "logps/chosen": -478.6031188964844,
      "logps/rejected": -533.3047485351562,
      "loss": 0.6625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8314554691314697,
      "rewards/margins": 0.922959566116333,
      "rewards/rejected": -1.7544149160385132,
      "step": 6625
    },
    {
      "epoch": 0.8670646928927783,
      "grad_norm": 5.5963826179504395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27959340810775757,
      "logits/rejected": 0.2416110336780548,
      "logps/chosen": -322.5544128417969,
      "logps/rejected": -348.1593322753906,
      "loss": 0.407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17311973869800568,
      "rewards/margins": 1.7925115823745728,
      "rewards/rejected": -1.965631365776062,
      "step": 6626
    },
    {
      "epoch": 0.86719555083013,
      "grad_norm": 5.5771379470825195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09455715119838715,
      "logits/rejected": 0.12180481851100922,
      "logps/chosen": -282.81915283203125,
      "logps/rejected": -346.170654296875,
      "loss": 0.2823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08295714855194092,
      "rewards/margins": 1.7533477544784546,
      "rewards/rejected": -1.6703906059265137,
      "step": 6627
    },
    {
      "epoch": 0.8673264087674818,
      "grad_norm": 7.484945297241211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20932462811470032,
      "logits/rejected": 0.4459884762763977,
      "logps/chosen": -314.9692077636719,
      "logps/rejected": -435.03656005859375,
      "loss": 0.6036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1074132919311523,
      "rewards/margins": 0.3779584765434265,
      "rewards/rejected": -1.485371708869934,
      "step": 6628
    },
    {
      "epoch": 0.8674572667048336,
      "grad_norm": 9.706646919250488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02468874119222164,
      "logits/rejected": -0.1898670643568039,
      "logps/chosen": -642.635986328125,
      "logps/rejected": -554.93408203125,
      "loss": 0.6151,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12254180014133453,
      "rewards/margins": 0.5809848308563232,
      "rewards/rejected": -0.7035266757011414,
      "step": 6629
    },
    {
      "epoch": 0.8675881246421854,
      "grad_norm": 7.172521591186523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03009321168065071,
      "logits/rejected": -0.22516608238220215,
      "logps/chosen": -400.98284912109375,
      "logps/rejected": -320.5275573730469,
      "loss": 0.6497,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.23457276821136475,
      "rewards/margins": 0.9449655413627625,
      "rewards/rejected": -0.7103927731513977,
      "step": 6630
    },
    {
      "epoch": 0.8677189825795371,
      "grad_norm": 7.318801403045654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20675086975097656,
      "logits/rejected": 0.1905815303325653,
      "logps/chosen": -544.8861083984375,
      "logps/rejected": -558.057373046875,
      "loss": 0.5809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7637336254119873,
      "rewards/margins": 1.0237994194030762,
      "rewards/rejected": -1.787532925605774,
      "step": 6631
    },
    {
      "epoch": 0.8678498405168888,
      "grad_norm": 7.062782287597656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18351218104362488,
      "logits/rejected": 0.1691429764032364,
      "logps/chosen": -442.0851745605469,
      "logps/rejected": -600.6480712890625,
      "loss": 0.4305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017080433666706085,
      "rewards/margins": 2.0400874614715576,
      "rewards/rejected": -2.0230069160461426,
      "step": 6632
    },
    {
      "epoch": 0.8679806984542406,
      "grad_norm": 8.102410316467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023206360638141632,
      "logits/rejected": -0.05455944687128067,
      "logps/chosen": -338.5003356933594,
      "logps/rejected": -407.6734924316406,
      "loss": 0.5357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16969658434391022,
      "rewards/margins": 1.291959285736084,
      "rewards/rejected": -1.461655855178833,
      "step": 6633
    },
    {
      "epoch": 0.8681115563915924,
      "grad_norm": 7.57091760635376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05794443190097809,
      "logits/rejected": 0.020400406792759895,
      "logps/chosen": -452.9442138671875,
      "logps/rejected": -504.03790283203125,
      "loss": 0.5351,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.47626417875289917,
      "rewards/margins": 1.1755766868591309,
      "rewards/rejected": -0.6993125081062317,
      "step": 6634
    },
    {
      "epoch": 0.8682424143289441,
      "grad_norm": 4.580071926116943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11665283143520355,
      "logits/rejected": 0.07462692260742188,
      "logps/chosen": -398.4197998046875,
      "logps/rejected": -393.0242004394531,
      "loss": 0.3153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3184950053691864,
      "rewards/margins": 1.5366055965423584,
      "rewards/rejected": -1.2181106805801392,
      "step": 6635
    },
    {
      "epoch": 0.8683732722662959,
      "grad_norm": 8.75481128692627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39253735542297363,
      "logits/rejected": -0.20009450614452362,
      "logps/chosen": -394.04766845703125,
      "logps/rejected": -266.2557373046875,
      "loss": 0.4799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2717776894569397,
      "rewards/margins": 1.6709232330322266,
      "rewards/rejected": -1.9427008628845215,
      "step": 6636
    },
    {
      "epoch": 0.8685041302036477,
      "grad_norm": 5.616189002990723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26380109786987305,
      "logits/rejected": -0.363687664270401,
      "logps/chosen": -376.78875732421875,
      "logps/rejected": -499.99151611328125,
      "loss": 0.2681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1625538170337677,
      "rewards/margins": 1.983148217201233,
      "rewards/rejected": -2.1457018852233887,
      "step": 6637
    },
    {
      "epoch": 0.8686349881409994,
      "grad_norm": 9.123156547546387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05546087771654129,
      "logits/rejected": 0.09502197057008743,
      "logps/chosen": -329.3822021484375,
      "logps/rejected": -359.2530517578125,
      "loss": 0.8484,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.029236435890197754,
      "rewards/margins": 0.21054109930992126,
      "rewards/rejected": -0.2397775650024414,
      "step": 6638
    },
    {
      "epoch": 0.8687658460783512,
      "grad_norm": 6.005764961242676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2605932652950287,
      "logits/rejected": -0.08602087199687958,
      "logps/chosen": -344.6005554199219,
      "logps/rejected": -364.981201171875,
      "loss": 0.3946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6248171329498291,
      "rewards/margins": 1.2233595848083496,
      "rewards/rejected": -0.5985424518585205,
      "step": 6639
    },
    {
      "epoch": 0.868896704015703,
      "grad_norm": 6.266007423400879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1529729664325714,
      "logits/rejected": 0.038604333996772766,
      "logps/chosen": -296.67376708984375,
      "logps/rejected": -284.1100769042969,
      "loss": 0.5793,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8126531839370728,
      "rewards/margins": 0.6973307728767395,
      "rewards/rejected": -1.509983777999878,
      "step": 6640
    },
    {
      "epoch": 0.8690275619530548,
      "grad_norm": 11.143736839294434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14842058718204498,
      "logits/rejected": -0.11733053624629974,
      "logps/chosen": -387.4766845703125,
      "logps/rejected": -434.10064697265625,
      "loss": 0.7731,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12089155614376068,
      "rewards/margins": 0.4632048010826111,
      "rewards/rejected": -0.5840963125228882,
      "step": 6641
    },
    {
      "epoch": 0.8691584198904064,
      "grad_norm": 11.81489086151123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26885902881622314,
      "logits/rejected": -0.17695200443267822,
      "logps/chosen": -482.6180419921875,
      "logps/rejected": -240.80807495117188,
      "loss": 1.1913,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3549833297729492,
      "rewards/margins": -0.3355783224105835,
      "rewards/rejected": -1.0194050073623657,
      "step": 6642
    },
    {
      "epoch": 0.8692892778277582,
      "grad_norm": 10.278631210327148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21937616169452667,
      "logits/rejected": -0.46863362193107605,
      "logps/chosen": -331.84423828125,
      "logps/rejected": -334.0384521484375,
      "loss": 0.7167,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8079308271408081,
      "rewards/margins": 0.8356972336769104,
      "rewards/rejected": -1.6436280012130737,
      "step": 6643
    },
    {
      "epoch": 0.86942013576511,
      "grad_norm": 10.182050704956055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27018100023269653,
      "logits/rejected": -0.3141070306301117,
      "logps/chosen": -460.71630859375,
      "logps/rejected": -588.744873046875,
      "loss": 0.6974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1297783851623535,
      "rewards/margins": 0.7044696807861328,
      "rewards/rejected": -1.8342481851577759,
      "step": 6644
    },
    {
      "epoch": 0.8695509937024618,
      "grad_norm": 10.668427467346191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3808438181877136,
      "logits/rejected": -0.15090525150299072,
      "logps/chosen": -357.6278991699219,
      "logps/rejected": -511.5790710449219,
      "loss": 0.6669,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7465420961380005,
      "rewards/margins": 0.25312572717666626,
      "rewards/rejected": -1.999667763710022,
      "step": 6645
    },
    {
      "epoch": 0.8696818516398135,
      "grad_norm": 4.653808116912842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03380941227078438,
      "logits/rejected": 0.06620477885007858,
      "logps/chosen": -456.146240234375,
      "logps/rejected": -354.0712890625,
      "loss": 0.3502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29061445593833923,
      "rewards/margins": 1.4519518613815308,
      "rewards/rejected": -1.1613372564315796,
      "step": 6646
    },
    {
      "epoch": 0.8698127095771653,
      "grad_norm": 12.465167999267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005820073187351227,
      "logits/rejected": -0.11376267671585083,
      "logps/chosen": -502.46319580078125,
      "logps/rejected": -348.9638977050781,
      "loss": 0.9162,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010926440358161926,
      "rewards/margins": -0.23645266890525818,
      "rewards/rejected": 0.22552624344825745,
      "step": 6647
    },
    {
      "epoch": 0.8699435675145171,
      "grad_norm": 5.74603271484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34977462887763977,
      "logits/rejected": -0.6311936974525452,
      "logps/chosen": -437.1768798828125,
      "logps/rejected": -761.93896484375,
      "loss": 0.3285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9160463213920593,
      "rewards/margins": 1.5411120653152466,
      "rewards/rejected": -2.4571585655212402,
      "step": 6648
    },
    {
      "epoch": 0.8700744254518689,
      "grad_norm": 15.293039321899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23564495146274567,
      "logits/rejected": -0.3512762784957886,
      "logps/chosen": -495.4804382324219,
      "logps/rejected": -606.8939208984375,
      "loss": 1.1921,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.209208369255066,
      "rewards/margins": -0.044943273067474365,
      "rewards/rejected": -1.1642651557922363,
      "step": 6649
    },
    {
      "epoch": 0.8702052833892205,
      "grad_norm": 8.647747993469238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41915613412857056,
      "logits/rejected": -0.3798118233680725,
      "logps/chosen": -440.5627136230469,
      "logps/rejected": -386.3132019042969,
      "loss": 0.4969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8268963098526001,
      "rewards/margins": 0.601706326007843,
      "rewards/rejected": -1.4286025762557983,
      "step": 6650
    },
    {
      "epoch": 0.8703361413265723,
      "grad_norm": 3.914332628250122,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43365082144737244,
      "logits/rejected": -0.04030431807041168,
      "logps/chosen": -439.66680908203125,
      "logps/rejected": -422.7169494628906,
      "loss": 0.159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5698442459106445,
      "rewards/margins": 2.6719093322753906,
      "rewards/rejected": -2.102065086364746,
      "step": 6651
    },
    {
      "epoch": 0.8704669992639241,
      "grad_norm": 8.95981216430664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1852477788925171,
      "logits/rejected": -0.164127916097641,
      "logps/chosen": -560.6814575195312,
      "logps/rejected": -514.1161499023438,
      "loss": 0.5551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07165983319282532,
      "rewards/margins": 1.3131526708602905,
      "rewards/rejected": -1.3848124742507935,
      "step": 6652
    },
    {
      "epoch": 0.8705978572012759,
      "grad_norm": 8.95002269744873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39459577202796936,
      "logits/rejected": 0.05462557077407837,
      "logps/chosen": -509.3055114746094,
      "logps/rejected": -467.39410400390625,
      "loss": 0.5243,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2426014095544815,
      "rewards/margins": 1.060862421989441,
      "rewards/rejected": -0.8182609677314758,
      "step": 6653
    },
    {
      "epoch": 0.8707287151386276,
      "grad_norm": 5.189688682556152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09483110904693604,
      "logits/rejected": 0.07629406452178955,
      "logps/chosen": -530.81201171875,
      "logps/rejected": -356.2255859375,
      "loss": 0.4589,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01710621267557144,
      "rewards/margins": 1.4126288890838623,
      "rewards/rejected": -1.3955227136611938,
      "step": 6654
    },
    {
      "epoch": 0.8708595730759794,
      "grad_norm": 8.538753509521484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17119231820106506,
      "logits/rejected": -0.26076027750968933,
      "logps/chosen": -363.6105651855469,
      "logps/rejected": -452.318359375,
      "loss": 0.6566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10415878891944885,
      "rewards/margins": 0.7795687913894653,
      "rewards/rejected": -0.8837276101112366,
      "step": 6655
    },
    {
      "epoch": 0.8709904310133312,
      "grad_norm": 8.861159324645996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026084795594215393,
      "logits/rejected": -0.09411276876926422,
      "logps/chosen": -414.43621826171875,
      "logps/rejected": -377.6202087402344,
      "loss": 0.6234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.32426920533180237,
      "rewards/margins": 1.5100373029708862,
      "rewards/rejected": -1.1857681274414062,
      "step": 6656
    },
    {
      "epoch": 0.8711212889506829,
      "grad_norm": 7.18064022064209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17251722514629364,
      "logits/rejected": -0.1837393045425415,
      "logps/chosen": -495.2456359863281,
      "logps/rejected": -491.3533935546875,
      "loss": 0.4701,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.39110201597213745,
      "rewards/margins": 0.9748445153236389,
      "rewards/rejected": -0.5837424993515015,
      "step": 6657
    },
    {
      "epoch": 0.8712521468880347,
      "grad_norm": 11.630473136901855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15578022599220276,
      "logits/rejected": -0.22772394120693207,
      "logps/chosen": -781.9883422851562,
      "logps/rejected": -558.243896484375,
      "loss": 0.8699,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4323664903640747,
      "rewards/margins": 0.347613662481308,
      "rewards/rejected": -1.7799803018569946,
      "step": 6658
    },
    {
      "epoch": 0.8713830048253864,
      "grad_norm": 12.558928489685059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01390589028596878,
      "logits/rejected": -0.09008558839559555,
      "logps/chosen": -598.6783447265625,
      "logps/rejected": -544.9229125976562,
      "loss": 1.1635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6107667684555054,
      "rewards/margins": 0.16258999705314636,
      "rewards/rejected": -0.7733568549156189,
      "step": 6659
    },
    {
      "epoch": 0.8715138627627382,
      "grad_norm": 7.491934299468994,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06610739976167679,
      "logits/rejected": -0.11615315079689026,
      "logps/chosen": -608.3106079101562,
      "logps/rejected": -544.3322143554688,
      "loss": 0.3793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39336490631103516,
      "rewards/margins": 1.617021083831787,
      "rewards/rejected": -1.2236562967300415,
      "step": 6660
    },
    {
      "epoch": 0.8716447207000899,
      "grad_norm": 6.246074199676514,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03583008423447609,
      "logits/rejected": -0.20759375393390656,
      "logps/chosen": -473.7868957519531,
      "logps/rejected": -370.76580810546875,
      "loss": 0.4252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.053306348621845245,
      "rewards/margins": 1.1458704471588135,
      "rewards/rejected": -1.0925642251968384,
      "step": 6661
    },
    {
      "epoch": 0.8717755786374417,
      "grad_norm": 6.42143440246582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1785299926996231,
      "logits/rejected": -0.07726090401411057,
      "logps/chosen": -406.7529602050781,
      "logps/rejected": -311.8708801269531,
      "loss": 0.4217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18642710149288177,
      "rewards/margins": 1.1760611534118652,
      "rewards/rejected": -1.3624882698059082,
      "step": 6662
    },
    {
      "epoch": 0.8719064365747935,
      "grad_norm": 8.497443199157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18858812749385834,
      "logits/rejected": -0.17183056473731995,
      "logps/chosen": -517.5791625976562,
      "logps/rejected": -395.5021057128906,
      "loss": 0.6816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04110376536846161,
      "rewards/margins": 1.2601993083953857,
      "rewards/rejected": -1.2190955877304077,
      "step": 6663
    },
    {
      "epoch": 0.8720372945121453,
      "grad_norm": 6.670744895935059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10802260041236877,
      "logits/rejected": -0.19702669978141785,
      "logps/chosen": -482.30743408203125,
      "logps/rejected": -500.2209167480469,
      "loss": 0.413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6971797943115234,
      "rewards/margins": 2.2558302879333496,
      "rewards/rejected": -1.558650255203247,
      "step": 6664
    },
    {
      "epoch": 0.872168152449497,
      "grad_norm": 8.003388404846191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018003009259700775,
      "logits/rejected": -0.16282764077186584,
      "logps/chosen": -740.3186645507812,
      "logps/rejected": -525.8613891601562,
      "loss": 0.4993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3027504086494446,
      "rewards/margins": 0.8349632620811462,
      "rewards/rejected": -0.5322128534317017,
      "step": 6665
    },
    {
      "epoch": 0.8722990103868488,
      "grad_norm": 6.291310787200928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18975670635700226,
      "logits/rejected": 0.011415254324674606,
      "logps/chosen": -369.7122802734375,
      "logps/rejected": -354.8719177246094,
      "loss": 0.4417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0028388798236846924,
      "rewards/margins": 0.971289873123169,
      "rewards/rejected": -0.9684509634971619,
      "step": 6666
    },
    {
      "epoch": 0.8724298683242006,
      "grad_norm": 12.245706558227539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.059099938720464706,
      "logits/rejected": -0.058080486953258514,
      "logps/chosen": -528.4782104492188,
      "logps/rejected": -488.6627197265625,
      "loss": 1.3419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12108269333839417,
      "rewards/margins": 0.24310509860515594,
      "rewards/rejected": -0.12202240526676178,
      "step": 6667
    },
    {
      "epoch": 0.8725607262615523,
      "grad_norm": 8.918930053710938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03914140909910202,
      "logits/rejected": 0.041622065007686615,
      "logps/chosen": -588.8441772460938,
      "logps/rejected": -672.27294921875,
      "loss": 0.7037,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8583356142044067,
      "rewards/margins": 0.7291181087493896,
      "rewards/rejected": 0.12921756505966187,
      "step": 6668
    },
    {
      "epoch": 0.872691584198904,
      "grad_norm": 7.745192050933838,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09117528796195984,
      "logits/rejected": -0.10991083830595016,
      "logps/chosen": -344.177734375,
      "logps/rejected": -412.17437744140625,
      "loss": 0.5049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.430424302816391,
      "rewards/margins": 0.7915281057357788,
      "rewards/rejected": -0.3611038625240326,
      "step": 6669
    },
    {
      "epoch": 0.8728224421362558,
      "grad_norm": 7.991396427154541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14592580497264862,
      "logits/rejected": 0.1536031812429428,
      "logps/chosen": -461.82562255859375,
      "logps/rejected": -357.284912109375,
      "loss": 0.5408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20488472282886505,
      "rewards/margins": 0.9905862212181091,
      "rewards/rejected": -1.195470929145813,
      "step": 6670
    },
    {
      "epoch": 0.8729533000736076,
      "grad_norm": 5.0263824462890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18161411583423615,
      "logits/rejected": -0.2542802691459656,
      "logps/chosen": -393.1965637207031,
      "logps/rejected": -375.08856201171875,
      "loss": 0.2492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18010330200195312,
      "rewards/margins": 1.5855389833450317,
      "rewards/rejected": -1.4054358005523682,
      "step": 6671
    },
    {
      "epoch": 0.8730841580109594,
      "grad_norm": 9.55589771270752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3503933548927307,
      "logits/rejected": -0.20300602912902832,
      "logps/chosen": -430.5884094238281,
      "logps/rejected": -498.4834899902344,
      "loss": 0.8291,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.37127628922462463,
      "rewards/margins": -0.03989581763744354,
      "rewards/rejected": 0.411172091960907,
      "step": 6672
    },
    {
      "epoch": 0.8732150159483111,
      "grad_norm": 7.3976545333862305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2755528688430786,
      "logits/rejected": -0.19042189419269562,
      "logps/chosen": -411.10345458984375,
      "logps/rejected": -497.67718505859375,
      "loss": 0.7216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20016993582248688,
      "rewards/margins": 0.8309009671211243,
      "rewards/rejected": -0.6307310461997986,
      "step": 6673
    },
    {
      "epoch": 0.8733458738856629,
      "grad_norm": 6.601901531219482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08259478211402893,
      "logits/rejected": -0.1237221211194992,
      "logps/chosen": -422.9015808105469,
      "logps/rejected": -384.0293273925781,
      "loss": 0.523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9816588163375854,
      "rewards/margins": 0.6580944061279297,
      "rewards/rejected": -1.6397532224655151,
      "step": 6674
    },
    {
      "epoch": 0.8734767318230147,
      "grad_norm": 6.109951496124268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07666027545928955,
      "logits/rejected": 0.09019330888986588,
      "logps/chosen": -328.2116394042969,
      "logps/rejected": -360.015869140625,
      "loss": 0.2099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11268842220306396,
      "rewards/margins": 2.1675829887390137,
      "rewards/rejected": -2.05489444732666,
      "step": 6675
    },
    {
      "epoch": 0.8736075897603665,
      "grad_norm": 8.050698280334473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03763584792613983,
      "logits/rejected": -0.09914372861385345,
      "logps/chosen": -347.07476806640625,
      "logps/rejected": -388.61077880859375,
      "loss": 0.6422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22232173383235931,
      "rewards/margins": 0.8835645914077759,
      "rewards/rejected": -1.1058863401412964,
      "step": 6676
    },
    {
      "epoch": 0.8737384476977181,
      "grad_norm": 6.057464122772217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08163455128669739,
      "logits/rejected": 0.018459420651197433,
      "logps/chosen": -398.5172119140625,
      "logps/rejected": -423.7695617675781,
      "loss": 0.4267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1465623676776886,
      "rewards/margins": 0.7981852889060974,
      "rewards/rejected": -0.651623010635376,
      "step": 6677
    },
    {
      "epoch": 0.8738693056350699,
      "grad_norm": 5.597115993499756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2527239918708801,
      "logits/rejected": -0.40897509455680847,
      "logps/chosen": -497.0691223144531,
      "logps/rejected": -315.7016906738281,
      "loss": 0.2155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1500428318977356,
      "rewards/margins": 1.704738974571228,
      "rewards/rejected": -1.5546962022781372,
      "step": 6678
    },
    {
      "epoch": 0.8740001635724217,
      "grad_norm": 8.123790740966797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40039384365081787,
      "logits/rejected": -0.4158564507961273,
      "logps/chosen": -250.5506134033203,
      "logps/rejected": -347.79229736328125,
      "loss": 0.6003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7713104486465454,
      "rewards/margins": 0.444068968296051,
      "rewards/rejected": -1.2153793573379517,
      "step": 6679
    },
    {
      "epoch": 0.8741310215097734,
      "grad_norm": 7.859193801879883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30315205454826355,
      "logits/rejected": 0.3010677993297577,
      "logps/chosen": -466.27960205078125,
      "logps/rejected": -429.08526611328125,
      "loss": 0.6325,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09008634090423584,
      "rewards/margins": 0.4856970012187958,
      "rewards/rejected": -0.5757833123207092,
      "step": 6680
    },
    {
      "epoch": 0.8742618794471252,
      "grad_norm": 4.971790313720703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0069222524762153625,
      "logits/rejected": 0.10110410302877426,
      "logps/chosen": -357.7856140136719,
      "logps/rejected": -502.7340393066406,
      "loss": 0.3497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6102590560913086,
      "rewards/margins": 1.4276877641677856,
      "rewards/rejected": -2.0379467010498047,
      "step": 6681
    },
    {
      "epoch": 0.874392737384477,
      "grad_norm": 7.827742099761963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28213179111480713,
      "logits/rejected": 0.02347465232014656,
      "logps/chosen": -587.6814575195312,
      "logps/rejected": -554.08544921875,
      "loss": 0.4928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5850666165351868,
      "rewards/margins": 1.3154919147491455,
      "rewards/rejected": -0.7304252982139587,
      "step": 6682
    },
    {
      "epoch": 0.8745235953218288,
      "grad_norm": 6.393713474273682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08183762431144714,
      "logits/rejected": -0.0013644695281982422,
      "logps/chosen": -351.1408386230469,
      "logps/rejected": -405.29071044921875,
      "loss": 0.546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02958013117313385,
      "rewards/margins": 1.1625834703445435,
      "rewards/rejected": -1.1330034732818604,
      "step": 6683
    },
    {
      "epoch": 0.8746544532591805,
      "grad_norm": 7.358728885650635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07766886800527573,
      "logits/rejected": 0.1605239063501358,
      "logps/chosen": -683.1430053710938,
      "logps/rejected": -372.36761474609375,
      "loss": 0.3822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.836639404296875,
      "rewards/margins": 1.5269609689712524,
      "rewards/rejected": -0.6903215050697327,
      "step": 6684
    },
    {
      "epoch": 0.8747853111965322,
      "grad_norm": 7.869312763214111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09725494682788849,
      "logits/rejected": -0.020484959706664085,
      "logps/chosen": -576.238037109375,
      "logps/rejected": -325.3797607421875,
      "loss": 0.4887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6263811588287354,
      "rewards/margins": 1.4185632467269897,
      "rewards/rejected": -0.7921820878982544,
      "step": 6685
    },
    {
      "epoch": 0.874916169133884,
      "grad_norm": 9.042807579040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04819575697183609,
      "logits/rejected": -0.02181258797645569,
      "logps/chosen": -290.0597229003906,
      "logps/rejected": -366.77532958984375,
      "loss": 0.5184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3501901924610138,
      "rewards/margins": 0.8207108378410339,
      "rewards/rejected": -1.1709009408950806,
      "step": 6686
    },
    {
      "epoch": 0.8750470270712358,
      "grad_norm": 6.034404754638672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04831548407673836,
      "logits/rejected": -0.0959264263510704,
      "logps/chosen": -434.6196594238281,
      "logps/rejected": -355.37518310546875,
      "loss": 0.4331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.003562554717063904,
      "rewards/margins": 0.8789141178131104,
      "rewards/rejected": -0.8753515481948853,
      "step": 6687
    },
    {
      "epoch": 0.8751778850085875,
      "grad_norm": 9.373233795166016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10753742605447769,
      "logits/rejected": 0.057739101350307465,
      "logps/chosen": -718.3017578125,
      "logps/rejected": -597.078125,
      "loss": 0.514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42120635509490967,
      "rewards/margins": 0.7279316186904907,
      "rewards/rejected": -1.1491378545761108,
      "step": 6688
    },
    {
      "epoch": 0.8753087429459393,
      "grad_norm": 10.67190170288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1719963252544403,
      "logits/rejected": -0.22556613385677338,
      "logps/chosen": -472.2421875,
      "logps/rejected": -384.626953125,
      "loss": 0.766,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0802859216928482,
      "rewards/margins": 0.5212011933326721,
      "rewards/rejected": -0.6014871001243591,
      "step": 6689
    },
    {
      "epoch": 0.8754396008832911,
      "grad_norm": 8.013126373291016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08744204044342041,
      "logits/rejected": -0.10622835159301758,
      "logps/chosen": -385.99267578125,
      "logps/rejected": -416.4601135253906,
      "loss": 0.5754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007148727774620056,
      "rewards/margins": 1.1236492395401,
      "rewards/rejected": -1.1165006160736084,
      "step": 6690
    },
    {
      "epoch": 0.8755704588206429,
      "grad_norm": 7.4927449226379395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07153953611850739,
      "logits/rejected": 0.2039399892091751,
      "logps/chosen": -628.7652587890625,
      "logps/rejected": -546.3298950195312,
      "loss": 0.5109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003067389130592346,
      "rewards/margins": 0.566457211971283,
      "rewards/rejected": -0.5633898377418518,
      "step": 6691
    },
    {
      "epoch": 0.8757013167579946,
      "grad_norm": 6.112249851226807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22220176458358765,
      "logits/rejected": -0.10133476555347443,
      "logps/chosen": -442.4715576171875,
      "logps/rejected": -499.9109802246094,
      "loss": 0.3398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17549000680446625,
      "rewards/margins": 1.6902680397033691,
      "rewards/rejected": -1.8657582998275757,
      "step": 6692
    },
    {
      "epoch": 0.8758321746953464,
      "grad_norm": 7.549436569213867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1231791079044342,
      "logits/rejected": 0.3555489182472229,
      "logps/chosen": -513.8032836914062,
      "logps/rejected": -395.46697998046875,
      "loss": 0.576,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.264360636472702,
      "rewards/margins": 0.5971877574920654,
      "rewards/rejected": -0.33282721042633057,
      "step": 6693
    },
    {
      "epoch": 0.8759630326326981,
      "grad_norm": 13.597517967224121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09373104572296143,
      "logits/rejected": -0.06386850029230118,
      "logps/chosen": -461.57781982421875,
      "logps/rejected": -359.4788513183594,
      "loss": 0.8282,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0192937850952148,
      "rewards/margins": 0.13148115575313568,
      "rewards/rejected": -1.1507749557495117,
      "step": 6694
    },
    {
      "epoch": 0.8760938905700499,
      "grad_norm": 7.26856803894043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22669270634651184,
      "logits/rejected": -0.11571703106164932,
      "logps/chosen": -403.362548828125,
      "logps/rejected": -464.73626708984375,
      "loss": 0.575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08562411367893219,
      "rewards/margins": 0.5401437878608704,
      "rewards/rejected": -0.6257679462432861,
      "step": 6695
    },
    {
      "epoch": 0.8762247485074016,
      "grad_norm": 5.767683029174805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21701112389564514,
      "logits/rejected": 0.13652706146240234,
      "logps/chosen": -475.5365905761719,
      "logps/rejected": -353.5299377441406,
      "loss": 0.3567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2871532738208771,
      "rewards/margins": 1.4686131477355957,
      "rewards/rejected": -1.181459903717041,
      "step": 6696
    },
    {
      "epoch": 0.8763556064447534,
      "grad_norm": 8.0369291305542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12134519964456558,
      "logits/rejected": -0.1980375200510025,
      "logps/chosen": -404.70159912109375,
      "logps/rejected": -336.82305908203125,
      "loss": 0.3744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4386173486709595,
      "rewards/margins": 1.9598153829574585,
      "rewards/rejected": -1.521198034286499,
      "step": 6697
    },
    {
      "epoch": 0.8764864643821052,
      "grad_norm": 10.004914283752441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1817481368780136,
      "logits/rejected": -0.3350723385810852,
      "logps/chosen": -476.7769775390625,
      "logps/rejected": -463.02862548828125,
      "loss": 0.6498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8565753698348999,
      "rewards/margins": 0.4208053946495056,
      "rewards/rejected": -1.2773808240890503,
      "step": 6698
    },
    {
      "epoch": 0.876617322319457,
      "grad_norm": 7.2937493324279785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13682898879051208,
      "logits/rejected": -0.16567496955394745,
      "logps/chosen": -600.8638305664062,
      "logps/rejected": -536.505126953125,
      "loss": 0.2977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3853895664215088,
      "rewards/margins": 1.799232840538025,
      "rewards/rejected": -2.1846225261688232,
      "step": 6699
    },
    {
      "epoch": 0.8767481802568087,
      "grad_norm": 5.7323408126831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21846237778663635,
      "logits/rejected": -0.18147042393684387,
      "logps/chosen": -654.023681640625,
      "logps/rejected": -528.4615478515625,
      "loss": 0.2812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8380838632583618,
      "rewards/margins": 2.0720720291137695,
      "rewards/rejected": -1.2339885234832764,
      "step": 6700
    },
    {
      "epoch": 0.8768790381941605,
      "grad_norm": 9.588030815124512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0670287236571312,
      "logits/rejected": -0.08646386861801147,
      "logps/chosen": -515.4902954101562,
      "logps/rejected": -488.8836669921875,
      "loss": 0.734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3138580620288849,
      "rewards/margins": 0.26063650846481323,
      "rewards/rejected": 0.05322151631116867,
      "step": 6701
    },
    {
      "epoch": 0.8770098961315123,
      "grad_norm": 9.375978469848633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35026609897613525,
      "logits/rejected": 0.3168847858905792,
      "logps/chosen": -644.18115234375,
      "logps/rejected": -456.27178955078125,
      "loss": 0.6234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8441112637519836,
      "rewards/margins": 0.7968590259552002,
      "rewards/rejected": -1.6409703493118286,
      "step": 6702
    },
    {
      "epoch": 0.8771407540688639,
      "grad_norm": 9.900026321411133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12963595986366272,
      "logits/rejected": -0.17770950496196747,
      "logps/chosen": -713.6502075195312,
      "logps/rejected": -455.8132629394531,
      "loss": 0.7888,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04053250700235367,
      "rewards/margins": 0.27603644132614136,
      "rewards/rejected": -0.31656894087791443,
      "step": 6703
    },
    {
      "epoch": 0.8772716120062157,
      "grad_norm": 6.710133075714111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03820323944091797,
      "logits/rejected": -0.07582557946443558,
      "logps/chosen": -423.7296142578125,
      "logps/rejected": -556.3284912109375,
      "loss": 0.4893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08185139298439026,
      "rewards/margins": 0.7935304641723633,
      "rewards/rejected": -0.8753818273544312,
      "step": 6704
    },
    {
      "epoch": 0.8774024699435675,
      "grad_norm": 10.666729927062988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14956554770469666,
      "logits/rejected": -0.3311922550201416,
      "logps/chosen": -399.7547912597656,
      "logps/rejected": -369.453857421875,
      "loss": 0.9331,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06121443212032318,
      "rewards/margins": 0.09265251457691193,
      "rewards/rejected": -0.1538669764995575,
      "step": 6705
    },
    {
      "epoch": 0.8775333278809193,
      "grad_norm": 7.783671855926514,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16796085238456726,
      "logits/rejected": -0.07482252269983292,
      "logps/chosen": -354.95770263671875,
      "logps/rejected": -320.831298828125,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.42630070447921753,
      "rewards/margins": 0.41336318850517273,
      "rewards/rejected": 0.012937530875205994,
      "step": 6706
    },
    {
      "epoch": 0.877664185818271,
      "grad_norm": 4.798546314239502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2163570076227188,
      "logits/rejected": 0.1429470032453537,
      "logps/chosen": -402.429931640625,
      "logps/rejected": -339.1044616699219,
      "loss": 0.3184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2694631516933441,
      "rewards/margins": 1.8599331378936768,
      "rewards/rejected": -1.5904701948165894,
      "step": 6707
    },
    {
      "epoch": 0.8777950437556228,
      "grad_norm": 7.961528301239014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3227727711200714,
      "logits/rejected": -0.007782965898513794,
      "logps/chosen": -665.268310546875,
      "logps/rejected": -564.054931640625,
      "loss": 0.3642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6096233129501343,
      "rewards/margins": 1.485214352607727,
      "rewards/rejected": -0.8755910992622375,
      "step": 6708
    },
    {
      "epoch": 0.8779259016929746,
      "grad_norm": 10.909748077392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09627910703420639,
      "logits/rejected": -0.16960665583610535,
      "logps/chosen": -494.43475341796875,
      "logps/rejected": -443.3973083496094,
      "loss": 0.7782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.016894638538360596,
      "rewards/margins": 0.06439771503210068,
      "rewards/rejected": -0.08129234611988068,
      "step": 6709
    },
    {
      "epoch": 0.8780567596303264,
      "grad_norm": 7.928158283233643,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017408914864063263,
      "logits/rejected": 0.04359297454357147,
      "logps/chosen": -375.43316650390625,
      "logps/rejected": -391.91180419921875,
      "loss": 0.6334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9230607748031616,
      "rewards/margins": 0.5489420294761658,
      "rewards/rejected": 0.3741188049316406,
      "step": 6710
    },
    {
      "epoch": 0.878187617567678,
      "grad_norm": 11.81502628326416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16040068864822388,
      "logits/rejected": 0.2999521493911743,
      "logps/chosen": -410.5523681640625,
      "logps/rejected": -428.9062805175781,
      "loss": 0.6559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3710293173789978,
      "rewards/margins": 0.5470796823501587,
      "rewards/rejected": -0.17605039477348328,
      "step": 6711
    },
    {
      "epoch": 0.8783184755050298,
      "grad_norm": 5.195652961730957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15029610693454742,
      "logits/rejected": -0.12140853703022003,
      "logps/chosen": -526.599365234375,
      "logps/rejected": -375.2784118652344,
      "loss": 0.3206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9201267957687378,
      "rewards/margins": 1.8787835836410522,
      "rewards/rejected": -0.9586568474769592,
      "step": 6712
    },
    {
      "epoch": 0.8784493334423816,
      "grad_norm": 8.01628303527832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02208014205098152,
      "logits/rejected": -0.03940185159444809,
      "logps/chosen": -584.298828125,
      "logps/rejected": -686.6862182617188,
      "loss": 0.5111,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5470980405807495,
      "rewards/margins": 1.388366460800171,
      "rewards/rejected": -1.9354643821716309,
      "step": 6713
    },
    {
      "epoch": 0.8785801913797334,
      "grad_norm": 5.487078666687012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08128226548433304,
      "logits/rejected": 0.10613758116960526,
      "logps/chosen": -488.5958557128906,
      "logps/rejected": -518.2245483398438,
      "loss": 0.3792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8375622034072876,
      "rewards/margins": 1.0132170915603638,
      "rewards/rejected": -0.17565496265888214,
      "step": 6714
    },
    {
      "epoch": 0.8787110493170851,
      "grad_norm": 8.05069637298584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09427828341722488,
      "logits/rejected": -0.01340540125966072,
      "logps/chosen": -428.5122375488281,
      "logps/rejected": -334.0183410644531,
      "loss": 0.5427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4743292033672333,
      "rewards/margins": 1.0849032402038574,
      "rewards/rejected": -1.5592323541641235,
      "step": 6715
    },
    {
      "epoch": 0.8788419072544369,
      "grad_norm": 4.6201276779174805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03961219638586044,
      "logits/rejected": -0.2562409043312073,
      "logps/chosen": -326.891845703125,
      "logps/rejected": -419.5187072753906,
      "loss": 0.239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08811341226100922,
      "rewards/margins": 1.903212547302246,
      "rewards/rejected": -1.8150991201400757,
      "step": 6716
    },
    {
      "epoch": 0.8789727651917887,
      "grad_norm": 5.299514293670654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20745928585529327,
      "logits/rejected": 0.10353468358516693,
      "logps/chosen": -497.72698974609375,
      "logps/rejected": -502.83740234375,
      "loss": 0.3653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13809089362621307,
      "rewards/margins": 1.895103931427002,
      "rewards/rejected": -2.0331947803497314,
      "step": 6717
    },
    {
      "epoch": 0.8791036231291405,
      "grad_norm": 6.880570888519287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013772226870059967,
      "logits/rejected": -0.009636469185352325,
      "logps/chosen": -428.43701171875,
      "logps/rejected": -596.8756103515625,
      "loss": 0.3662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5763664245605469,
      "rewards/margins": 1.6223175525665283,
      "rewards/rejected": -2.198683977127075,
      "step": 6718
    },
    {
      "epoch": 0.8792344810664922,
      "grad_norm": 3.399895191192627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16257235407829285,
      "logits/rejected": -0.24220383167266846,
      "logps/chosen": -393.21295166015625,
      "logps/rejected": -330.3479919433594,
      "loss": 0.3353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20810845494270325,
      "rewards/margins": 1.2826628684997559,
      "rewards/rejected": -1.074554204940796,
      "step": 6719
    },
    {
      "epoch": 0.879365339003844,
      "grad_norm": 4.204643726348877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03941603749990463,
      "logits/rejected": 0.0024001598358154297,
      "logps/chosen": -418.62286376953125,
      "logps/rejected": -472.44439697265625,
      "loss": 0.2854,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.010356314480304718,
      "rewards/margins": 2.006227731704712,
      "rewards/rejected": -2.0165841579437256,
      "step": 6720
    },
    {
      "epoch": 0.8794961969411957,
      "grad_norm": 8.280948638916016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06395154446363449,
      "logits/rejected": -0.017847269773483276,
      "logps/chosen": -441.26519775390625,
      "logps/rejected": -440.3243408203125,
      "loss": 0.6894,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.47832778096199036,
      "rewards/margins": 0.6736217737197876,
      "rewards/rejected": -0.19529400765895844,
      "step": 6721
    },
    {
      "epoch": 0.8796270548785475,
      "grad_norm": 4.736708641052246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19960176944732666,
      "logits/rejected": -0.06429862976074219,
      "logps/chosen": -323.9644470214844,
      "logps/rejected": -379.28155517578125,
      "loss": 0.3103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20271196961402893,
      "rewards/margins": 1.2930353879928589,
      "rewards/rejected": -1.0903234481811523,
      "step": 6722
    },
    {
      "epoch": 0.8797579128158992,
      "grad_norm": 4.710851669311523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09580755978822708,
      "logits/rejected": -0.11746254563331604,
      "logps/chosen": -555.9838256835938,
      "logps/rejected": -469.1483459472656,
      "loss": 0.2434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3479669392108917,
      "rewards/margins": 1.5205491781234741,
      "rewards/rejected": -1.1725822687149048,
      "step": 6723
    },
    {
      "epoch": 0.879888770753251,
      "grad_norm": 6.452384948730469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09737849980592728,
      "logits/rejected": 0.07474013417959213,
      "logps/chosen": -521.1819458007812,
      "logps/rejected": -479.3388366699219,
      "loss": 0.3263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7119418978691101,
      "rewards/margins": 1.3347673416137695,
      "rewards/rejected": -0.6228253841400146,
      "step": 6724
    },
    {
      "epoch": 0.8800196286906028,
      "grad_norm": 6.810962200164795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02703278698027134,
      "logits/rejected": -0.1858403980731964,
      "logps/chosen": -458.06103515625,
      "logps/rejected": -477.75897216796875,
      "loss": 0.4648,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6135786771774292,
      "rewards/margins": 1.0461974143981934,
      "rewards/rejected": -0.4326187074184418,
      "step": 6725
    },
    {
      "epoch": 0.8801504866279545,
      "grad_norm": 7.257233619689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30945929884910583,
      "logits/rejected": -0.7483669519424438,
      "logps/chosen": -287.7732238769531,
      "logps/rejected": -459.328857421875,
      "loss": 0.4507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5722661018371582,
      "rewards/margins": 1.2635351419448853,
      "rewards/rejected": -1.8358012437820435,
      "step": 6726
    },
    {
      "epoch": 0.8802813445653063,
      "grad_norm": 8.5169038772583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07374584674835205,
      "logits/rejected": 0.09023693203926086,
      "logps/chosen": -314.6126403808594,
      "logps/rejected": -615.8674926757812,
      "loss": 0.757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20776358246803284,
      "rewards/margins": 0.5994910597801208,
      "rewards/rejected": -0.391727477312088,
      "step": 6727
    },
    {
      "epoch": 0.8804122025026581,
      "grad_norm": 4.0738067626953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10762310773134232,
      "logits/rejected": -0.2730204164981842,
      "logps/chosen": -435.61663818359375,
      "logps/rejected": -543.6687622070312,
      "loss": 0.2008,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08202075958251953,
      "rewards/margins": 2.2356159687042236,
      "rewards/rejected": -2.317636728286743,
      "step": 6728
    },
    {
      "epoch": 0.8805430604400099,
      "grad_norm": 8.261772155761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07662050426006317,
      "logits/rejected": -0.1078222244977951,
      "logps/chosen": -373.8915100097656,
      "logps/rejected": -485.57843017578125,
      "loss": 0.5994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6132906675338745,
      "rewards/margins": 0.7047938108444214,
      "rewards/rejected": -0.09150315821170807,
      "step": 6729
    },
    {
      "epoch": 0.8806739183773615,
      "grad_norm": 6.585116863250732,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0379282608628273,
      "logits/rejected": -0.06759700179100037,
      "logps/chosen": -412.2506408691406,
      "logps/rejected": -406.0111999511719,
      "loss": 0.4589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3254657983779907,
      "rewards/margins": 1.7343626022338867,
      "rewards/rejected": -1.408896803855896,
      "step": 6730
    },
    {
      "epoch": 0.8808047763147133,
      "grad_norm": 6.451587677001953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10722619295120239,
      "logits/rejected": 0.0395527184009552,
      "logps/chosen": -360.5552978515625,
      "logps/rejected": -376.1820068359375,
      "loss": 0.5128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4081638753414154,
      "rewards/margins": 1.2665965557098389,
      "rewards/rejected": -0.8584327697753906,
      "step": 6731
    },
    {
      "epoch": 0.8809356342520651,
      "grad_norm": 5.239136695861816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023722093552350998,
      "logits/rejected": -0.22135566174983978,
      "logps/chosen": -422.3518371582031,
      "logps/rejected": -509.76190185546875,
      "loss": 0.3072,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5330957174301147,
      "rewards/margins": 1.4053541421890259,
      "rewards/rejected": -0.8722585439682007,
      "step": 6732
    },
    {
      "epoch": 0.8810664921894169,
      "grad_norm": 4.737268924713135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06373555213212967,
      "logits/rejected": 0.04594624415040016,
      "logps/chosen": -383.5537414550781,
      "logps/rejected": -462.826904296875,
      "loss": 0.2696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6093919277191162,
      "rewards/margins": 2.4906816482543945,
      "rewards/rejected": -1.8812898397445679,
      "step": 6733
    },
    {
      "epoch": 0.8811973501267686,
      "grad_norm": 9.411341667175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30088353157043457,
      "logits/rejected": -0.21331709623336792,
      "logps/chosen": -465.2961120605469,
      "logps/rejected": -490.45257568359375,
      "loss": 1.0404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2296639382839203,
      "rewards/margins": -0.019424259662628174,
      "rewards/rejected": -0.21023958921432495,
      "step": 6734
    },
    {
      "epoch": 0.8813282080641204,
      "grad_norm": 5.161308765411377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08546047657728195,
      "logits/rejected": -0.28708145022392273,
      "logps/chosen": -317.6869812011719,
      "logps/rejected": -472.081787109375,
      "loss": 0.2485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16920757293701172,
      "rewards/margins": 1.7968416213989258,
      "rewards/rejected": -1.9660491943359375,
      "step": 6735
    },
    {
      "epoch": 0.8814590660014722,
      "grad_norm": 9.046967506408691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23278838396072388,
      "logits/rejected": 0.0611732080578804,
      "logps/chosen": -456.1458435058594,
      "logps/rejected": -519.3245849609375,
      "loss": 0.5996,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006447814404964447,
      "rewards/margins": 1.0074385404586792,
      "rewards/rejected": -1.0009907484054565,
      "step": 6736
    },
    {
      "epoch": 0.881589923938824,
      "grad_norm": 9.855947494506836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09794189780950546,
      "logits/rejected": 0.04982181638479233,
      "logps/chosen": -464.1042785644531,
      "logps/rejected": -268.88616943359375,
      "loss": 0.7953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.4489072859287262,
      "rewards/margins": 0.5047999620437622,
      "rewards/rejected": -0.055892642587423325,
      "step": 6737
    },
    {
      "epoch": 0.8817207818761756,
      "grad_norm": 6.616552352905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05746597424149513,
      "logits/rejected": -0.1928216814994812,
      "logps/chosen": -599.1949462890625,
      "logps/rejected": -538.3401489257812,
      "loss": 0.3033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3163926899433136,
      "rewards/margins": 2.1036510467529297,
      "rewards/rejected": -1.787258267402649,
      "step": 6738
    },
    {
      "epoch": 0.8818516398135274,
      "grad_norm": 10.377387046813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0895274430513382,
      "logits/rejected": -0.05207423120737076,
      "logps/chosen": -340.31353759765625,
      "logps/rejected": -354.660888671875,
      "loss": 0.7031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12498930841684341,
      "rewards/margins": 0.8592065572738647,
      "rewards/rejected": -0.9841959476470947,
      "step": 6739
    },
    {
      "epoch": 0.8819824977508792,
      "grad_norm": 8.151520729064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4443374574184418,
      "logits/rejected": -0.39141789078712463,
      "logps/chosen": -374.99346923828125,
      "logps/rejected": -385.9270324707031,
      "loss": 0.8191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04967420548200607,
      "rewards/margins": 0.10613460838794708,
      "rewards/rejected": -0.05646036937832832,
      "step": 6740
    },
    {
      "epoch": 0.882113355688231,
      "grad_norm": 6.70379638671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11564826220273972,
      "logits/rejected": -0.08884033560752869,
      "logps/chosen": -408.9693603515625,
      "logps/rejected": -448.980712890625,
      "loss": 0.7286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004692830145359039,
      "rewards/margins": 0.5363434553146362,
      "rewards/rejected": -0.5316505432128906,
      "step": 6741
    },
    {
      "epoch": 0.8822442136255827,
      "grad_norm": 15.424226760864258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18504345417022705,
      "logits/rejected": -0.1926417350769043,
      "logps/chosen": -418.66607666015625,
      "logps/rejected": -589.9893798828125,
      "loss": 1.075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0208449363708496,
      "rewards/margins": -0.18353445827960968,
      "rewards/rejected": -0.8373103737831116,
      "step": 6742
    },
    {
      "epoch": 0.8823750715629345,
      "grad_norm": 8.717207908630371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2863040566444397,
      "logits/rejected": -0.27588436007499695,
      "logps/chosen": -397.62939453125,
      "logps/rejected": -477.8755798339844,
      "loss": 0.4982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23678059875965118,
      "rewards/margins": 0.8677900433540344,
      "rewards/rejected": -1.1045706272125244,
      "step": 6743
    },
    {
      "epoch": 0.8825059295002863,
      "grad_norm": 6.388675212860107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27556413412094116,
      "logits/rejected": -0.0367426797747612,
      "logps/chosen": -348.0599060058594,
      "logps/rejected": -438.1184997558594,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4541515111923218,
      "rewards/margins": 1.5783778429031372,
      "rewards/rejected": -1.1242263317108154,
      "step": 6744
    },
    {
      "epoch": 0.882636787437638,
      "grad_norm": 5.079624652862549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4328656792640686,
      "logits/rejected": -0.3557019829750061,
      "logps/chosen": -393.5820007324219,
      "logps/rejected": -465.767822265625,
      "loss": 0.4472,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15005092322826385,
      "rewards/margins": 1.28743577003479,
      "rewards/rejected": -1.1373847723007202,
      "step": 6745
    },
    {
      "epoch": 0.8827676453749898,
      "grad_norm": 9.392302513122559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1566002070903778,
      "logits/rejected": -0.19443176686763763,
      "logps/chosen": -416.5146484375,
      "logps/rejected": -391.1705322265625,
      "loss": 0.6955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14271871745586395,
      "rewards/margins": 0.7042651772499084,
      "rewards/rejected": -0.846983790397644,
      "step": 6746
    },
    {
      "epoch": 0.8828985033123415,
      "grad_norm": 9.292380332946777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0401875264942646,
      "logits/rejected": 0.03430280461907387,
      "logps/chosen": -481.23974609375,
      "logps/rejected": -403.846923828125,
      "loss": 0.758,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4302072525024414,
      "rewards/margins": 0.1582176238298416,
      "rewards/rejected": -0.5884249210357666,
      "step": 6747
    },
    {
      "epoch": 0.8830293612496933,
      "grad_norm": 7.271967887878418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06654459238052368,
      "logits/rejected": 0.02559637650847435,
      "logps/chosen": -372.8897705078125,
      "logps/rejected": -357.611328125,
      "loss": 0.5162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4321924149990082,
      "rewards/margins": 1.001534104347229,
      "rewards/rejected": -0.5693417191505432,
      "step": 6748
    },
    {
      "epoch": 0.883160219187045,
      "grad_norm": 8.097167015075684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2385147362947464,
      "logits/rejected": -0.2683723568916321,
      "logps/chosen": -418.27618408203125,
      "logps/rejected": -453.0159606933594,
      "loss": 0.5061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.00023406744003295898,
      "rewards/margins": 1.1564685106277466,
      "rewards/rejected": -1.1567026376724243,
      "step": 6749
    },
    {
      "epoch": 0.8832910771243968,
      "grad_norm": 5.749520301818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3391178548336029,
      "logits/rejected": -0.31889259815216064,
      "logps/chosen": -568.4703979492188,
      "logps/rejected": -477.6084289550781,
      "loss": 0.2903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6166958212852478,
      "rewards/margins": 1.357843041419983,
      "rewards/rejected": -0.7411472201347351,
      "step": 6750
    },
    {
      "epoch": 0.8834219350617486,
      "grad_norm": 9.736617088317871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06636133790016174,
      "logits/rejected": -0.07635176926851273,
      "logps/chosen": -597.3402099609375,
      "logps/rejected": -556.7255859375,
      "loss": 0.8974,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.169932559132576,
      "rewards/margins": 0.38153594732284546,
      "rewards/rejected": -0.21160337328910828,
      "step": 6751
    },
    {
      "epoch": 0.8835527929991004,
      "grad_norm": 8.266599655151367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.41792482137680054,
      "logits/rejected": -0.04106806591153145,
      "logps/chosen": -325.6527099609375,
      "logps/rejected": -432.6340637207031,
      "loss": 0.5165,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16883204877376556,
      "rewards/margins": 0.9789009094238281,
      "rewards/rejected": -1.1477328538894653,
      "step": 6752
    },
    {
      "epoch": 0.8836836509364521,
      "grad_norm": 8.339580535888672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16270290315151215,
      "logits/rejected": -0.08362658321857452,
      "logps/chosen": -440.911865234375,
      "logps/rejected": -399.7510986328125,
      "loss": 0.5076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10645599663257599,
      "rewards/margins": 0.896891713142395,
      "rewards/rejected": -0.7904356718063354,
      "step": 6753
    },
    {
      "epoch": 0.8838145088738039,
      "grad_norm": 10.264310836791992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0006235651671886444,
      "logits/rejected": -0.02137615904211998,
      "logps/chosen": -448.1322326660156,
      "logps/rejected": -561.578369140625,
      "loss": 0.5413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1467457115650177,
      "rewards/margins": 0.7165646553039551,
      "rewards/rejected": -0.8633103370666504,
      "step": 6754
    },
    {
      "epoch": 0.8839453668111557,
      "grad_norm": 9.37773323059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06916426122188568,
      "logits/rejected": -0.24753689765930176,
      "logps/chosen": -373.85247802734375,
      "logps/rejected": -423.2302551269531,
      "loss": 0.603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22804084420204163,
      "rewards/margins": 0.7557715177536011,
      "rewards/rejected": -0.9838124513626099,
      "step": 6755
    },
    {
      "epoch": 0.8840762247485074,
      "grad_norm": 12.386274337768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6212892532348633,
      "logits/rejected": -0.41170287132263184,
      "logps/chosen": -542.3509521484375,
      "logps/rejected": -578.123291015625,
      "loss": 0.797,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8978906273841858,
      "rewards/margins": -0.03396167978644371,
      "rewards/rejected": -0.863929033279419,
      "step": 6756
    },
    {
      "epoch": 0.8842070826858591,
      "grad_norm": 3.759986400604248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14514513313770294,
      "logits/rejected": -0.17961758375167847,
      "logps/chosen": -520.3065185546875,
      "logps/rejected": -419.13653564453125,
      "loss": 0.2506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.032233476638794,
      "rewards/margins": 1.8353168964385986,
      "rewards/rejected": -0.8030834197998047,
      "step": 6757
    },
    {
      "epoch": 0.8843379406232109,
      "grad_norm": 9.32077407836914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06417889893054962,
      "logits/rejected": 0.027389878407120705,
      "logps/chosen": -603.0693969726562,
      "logps/rejected": -369.55584716796875,
      "loss": 0.7521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7642881274223328,
      "rewards/margins": 1.0441877841949463,
      "rewards/rejected": -1.8084759712219238,
      "step": 6758
    },
    {
      "epoch": 0.8844687985605627,
      "grad_norm": 10.418763160705566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11655941605567932,
      "logits/rejected": 0.15629181265830994,
      "logps/chosen": -544.9605102539062,
      "logps/rejected": -640.2096557617188,
      "loss": 0.8637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15705528855323792,
      "rewards/margins": 0.6626606583595276,
      "rewards/rejected": -0.5056052207946777,
      "step": 6759
    },
    {
      "epoch": 0.8845996564979145,
      "grad_norm": 8.748458862304688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29841119050979614,
      "logits/rejected": -0.4275282025337219,
      "logps/chosen": -365.7401123046875,
      "logps/rejected": -437.11932373046875,
      "loss": 0.5381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11021293699741364,
      "rewards/margins": 1.189836859703064,
      "rewards/rejected": -1.3000496625900269,
      "step": 6760
    },
    {
      "epoch": 0.8847305144352662,
      "grad_norm": 7.452264308929443,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15967212617397308,
      "logits/rejected": -0.059397101402282715,
      "logps/chosen": -361.9237976074219,
      "logps/rejected": -350.15997314453125,
      "loss": 0.5919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6820719242095947,
      "rewards/margins": 0.579376220703125,
      "rewards/rejected": -1.2614481449127197,
      "step": 6761
    },
    {
      "epoch": 0.884861372372618,
      "grad_norm": 6.813710689544678,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27549275755882263,
      "logits/rejected": -0.04493430256843567,
      "logps/chosen": -524.106689453125,
      "logps/rejected": -501.86016845703125,
      "loss": 0.2949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.674826443195343,
      "rewards/margins": 1.7792105674743652,
      "rewards/rejected": -1.1043840646743774,
      "step": 6762
    },
    {
      "epoch": 0.8849922303099698,
      "grad_norm": 6.481354713439941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10849003493785858,
      "logits/rejected": -0.1695377379655838,
      "logps/chosen": -331.7962646484375,
      "logps/rejected": -331.9209289550781,
      "loss": 0.471,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.716282844543457,
      "rewards/margins": 1.200911521911621,
      "rewards/rejected": -1.9171943664550781,
      "step": 6763
    },
    {
      "epoch": 0.8851230882473216,
      "grad_norm": 9.95874309539795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.036309100687503815,
      "logits/rejected": -0.10376280546188354,
      "logps/chosen": -519.04833984375,
      "logps/rejected": -391.52264404296875,
      "loss": 0.2731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4623590409755707,
      "rewards/margins": 1.4760770797729492,
      "rewards/rejected": -1.9384362697601318,
      "step": 6764
    },
    {
      "epoch": 0.8852539461846732,
      "grad_norm": 11.072258949279785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18233759701251984,
      "logits/rejected": -0.05618630349636078,
      "logps/chosen": -291.3945007324219,
      "logps/rejected": -336.3014221191406,
      "loss": 0.8455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4058233499526978,
      "rewards/margins": 0.21410465240478516,
      "rewards/rejected": -1.6199278831481934,
      "step": 6765
    },
    {
      "epoch": 0.885384804122025,
      "grad_norm": 13.406540870666504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24233776330947876,
      "logits/rejected": 0.0315525159239769,
      "logps/chosen": -555.5420532226562,
      "logps/rejected": -577.5934448242188,
      "loss": 0.7402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7522161602973938,
      "rewards/margins": 0.5988010168075562,
      "rewards/rejected": -1.3510172367095947,
      "step": 6766
    },
    {
      "epoch": 0.8855156620593768,
      "grad_norm": 11.095151901245117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.210475355386734,
      "logits/rejected": -0.03123416006565094,
      "logps/chosen": -480.0066833496094,
      "logps/rejected": -443.4893798828125,
      "loss": 0.8641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.36331361532211304,
      "rewards/margins": 0.757885217666626,
      "rewards/rejected": -1.1211987733840942,
      "step": 6767
    },
    {
      "epoch": 0.8856465199967285,
      "grad_norm": 3.7105884552001953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003801915794610977,
      "logits/rejected": -0.3021829426288605,
      "logps/chosen": -522.1409912109375,
      "logps/rejected": -495.4927978515625,
      "loss": 0.1699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9760724902153015,
      "rewards/margins": 2.3830435276031494,
      "rewards/rejected": -1.4069708585739136,
      "step": 6768
    },
    {
      "epoch": 0.8857773779340803,
      "grad_norm": 10.367825508117676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0106612304225564,
      "logits/rejected": -0.218678280711174,
      "logps/chosen": -395.32122802734375,
      "logps/rejected": -447.80267333984375,
      "loss": 0.966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3444124460220337,
      "rewards/margins": 0.14501473307609558,
      "rewards/rejected": -0.48942720890045166,
      "step": 6769
    },
    {
      "epoch": 0.8859082358714321,
      "grad_norm": 9.033722877502441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18202590942382812,
      "logits/rejected": 0.07982655614614487,
      "logps/chosen": -527.9993896484375,
      "logps/rejected": -460.6455078125,
      "loss": 0.5823,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.719258189201355,
      "rewards/margins": 0.6200136542320251,
      "rewards/rejected": -1.3392717838287354,
      "step": 6770
    },
    {
      "epoch": 0.8860390938087839,
      "grad_norm": 7.165765762329102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09360527247190475,
      "logits/rejected": -0.037834398448467255,
      "logps/chosen": -483.05224609375,
      "logps/rejected": -371.41387939453125,
      "loss": 0.512,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07264842092990875,
      "rewards/margins": 0.7906643152236938,
      "rewards/rejected": -0.8633127212524414,
      "step": 6771
    },
    {
      "epoch": 0.8861699517461356,
      "grad_norm": 10.727645874023438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22627778351306915,
      "logits/rejected": -0.217651829123497,
      "logps/chosen": -419.59100341796875,
      "logps/rejected": -502.5735168457031,
      "loss": 0.9156,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1434212625026703,
      "rewards/margins": 0.20399510860443115,
      "rewards/rejected": -0.34741634130477905,
      "step": 6772
    },
    {
      "epoch": 0.8863008096834873,
      "grad_norm": 9.852195739746094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1061038225889206,
      "logits/rejected": 0.12469124794006348,
      "logps/chosen": -641.38427734375,
      "logps/rejected": -503.0572509765625,
      "loss": 0.4399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06957628577947617,
      "rewards/margins": 0.8715999722480774,
      "rewards/rejected": -0.9411762356758118,
      "step": 6773
    },
    {
      "epoch": 0.8864316676208391,
      "grad_norm": 8.986112594604492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2368193417787552,
      "logits/rejected": -0.2640795409679413,
      "logps/chosen": -546.20947265625,
      "logps/rejected": -697.9078369140625,
      "loss": 0.4048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19117426872253418,
      "rewards/margins": 1.4345765113830566,
      "rewards/rejected": -1.6257508993148804,
      "step": 6774
    },
    {
      "epoch": 0.8865625255581909,
      "grad_norm": 7.05488395690918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12091218680143356,
      "logits/rejected": -0.29139626026153564,
      "logps/chosen": -548.6437377929688,
      "logps/rejected": -528.7216796875,
      "loss": 0.4079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33815687894821167,
      "rewards/margins": 1.296775460243225,
      "rewards/rejected": -1.6349323987960815,
      "step": 6775
    },
    {
      "epoch": 0.8866933834955426,
      "grad_norm": 5.06945276260376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12530604004859924,
      "logits/rejected": -0.031118765473365784,
      "logps/chosen": -238.92056274414062,
      "logps/rejected": -254.56201171875,
      "loss": 0.5921,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19452650845050812,
      "rewards/margins": 0.615953803062439,
      "rewards/rejected": -0.42142727971076965,
      "step": 6776
    },
    {
      "epoch": 0.8868242414328944,
      "grad_norm": 9.055964469909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2264341115951538,
      "logits/rejected": -0.2797500491142273,
      "logps/chosen": -335.0148010253906,
      "logps/rejected": -310.0069274902344,
      "loss": 0.7033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7572382092475891,
      "rewards/margins": 1.0971202850341797,
      "rewards/rejected": -1.8543585538864136,
      "step": 6777
    },
    {
      "epoch": 0.8869550993702462,
      "grad_norm": 9.58580493927002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08065017312765121,
      "logits/rejected": 0.1349826157093048,
      "logps/chosen": -499.3275146484375,
      "logps/rejected": -539.0054931640625,
      "loss": 0.764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.672066867351532,
      "rewards/margins": 0.3009418249130249,
      "rewards/rejected": 0.3711250424385071,
      "step": 6778
    },
    {
      "epoch": 0.887085957307598,
      "grad_norm": 6.657781600952148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24598899483680725,
      "logits/rejected": -0.07825927436351776,
      "logps/chosen": -308.5664367675781,
      "logps/rejected": -574.22119140625,
      "loss": 0.4568,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7699002623558044,
      "rewards/margins": 1.3381767272949219,
      "rewards/rejected": -2.108077049255371,
      "step": 6779
    },
    {
      "epoch": 0.8872168152449497,
      "grad_norm": 11.689667701721191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20236676931381226,
      "logits/rejected": -0.21467269957065582,
      "logps/chosen": -395.31219482421875,
      "logps/rejected": -344.140869140625,
      "loss": 0.6793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08420297503471375,
      "rewards/margins": 1.0415780544281006,
      "rewards/rejected": -0.957375168800354,
      "step": 6780
    },
    {
      "epoch": 0.8873476731823015,
      "grad_norm": 7.52230167388916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17122063040733337,
      "logits/rejected": -0.15306204557418823,
      "logps/chosen": -643.2161865234375,
      "logps/rejected": -531.5259399414062,
      "loss": 0.4323,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2905149459838867,
      "rewards/margins": 1.4655792713165283,
      "rewards/rejected": -1.7560943365097046,
      "step": 6781
    },
    {
      "epoch": 0.8874785311196532,
      "grad_norm": 10.23429012298584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1631874293088913,
      "logits/rejected": 0.006952181458473206,
      "logps/chosen": -833.9763793945312,
      "logps/rejected": -661.0856323242188,
      "loss": 0.5742,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.469755619764328,
      "rewards/margins": 1.2709282636642456,
      "rewards/rejected": -0.80117267370224,
      "step": 6782
    },
    {
      "epoch": 0.887609389057005,
      "grad_norm": 7.7897820472717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23126161098480225,
      "logits/rejected": -0.05535402148962021,
      "logps/chosen": -338.6534423828125,
      "logps/rejected": -477.8101806640625,
      "loss": 0.6159,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3076309561729431,
      "rewards/margins": 1.1225281953811646,
      "rewards/rejected": -0.814897358417511,
      "step": 6783
    },
    {
      "epoch": 0.8877402469943567,
      "grad_norm": 10.388618469238281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04899759963154793,
      "logits/rejected": 0.02535773068666458,
      "logps/chosen": -635.9241333007812,
      "logps/rejected": -603.9942016601562,
      "loss": 0.6434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4872201681137085,
      "rewards/margins": 0.7225761413574219,
      "rewards/rejected": -1.20979642868042,
      "step": 6784
    },
    {
      "epoch": 0.8878711049317085,
      "grad_norm": 5.160669803619385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16815944015979767,
      "logits/rejected": -0.013044480234384537,
      "logps/chosen": -389.0990295410156,
      "logps/rejected": -356.22857666015625,
      "loss": 0.3836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5673106908798218,
      "rewards/margins": 1.140081763267517,
      "rewards/rejected": -0.5727710127830505,
      "step": 6785
    },
    {
      "epoch": 0.8880019628690603,
      "grad_norm": 6.676093578338623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3508489727973938,
      "logits/rejected": -0.23618699610233307,
      "logps/chosen": -316.8875732421875,
      "logps/rejected": -425.95904541015625,
      "loss": 0.3393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7120146751403809,
      "rewards/margins": 1.6459928750991821,
      "rewards/rejected": -2.3580074310302734,
      "step": 6786
    },
    {
      "epoch": 0.8881328208064121,
      "grad_norm": 8.956792831420898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.44016456604003906,
      "logits/rejected": 0.2556915879249573,
      "logps/chosen": -349.7540588378906,
      "logps/rejected": -380.14581298828125,
      "loss": 0.7738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5815733075141907,
      "rewards/margins": 0.889431357383728,
      "rewards/rejected": -1.471004605293274,
      "step": 6787
    },
    {
      "epoch": 0.8882636787437638,
      "grad_norm": 6.302381992340088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24758024513721466,
      "logits/rejected": -0.28608396649360657,
      "logps/chosen": -476.04266357421875,
      "logps/rejected": -529.0311279296875,
      "loss": 0.4053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16057932376861572,
      "rewards/margins": 1.3223092555999756,
      "rewards/rejected": -1.4828884601593018,
      "step": 6788
    },
    {
      "epoch": 0.8883945366811156,
      "grad_norm": 4.5133185386657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06021555885672569,
      "logits/rejected": -0.1977035105228424,
      "logps/chosen": -690.06298828125,
      "logps/rejected": -366.0321044921875,
      "loss": 0.2893,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.788049042224884,
      "rewards/margins": 1.4639211893081665,
      "rewards/rejected": -0.6758720278739929,
      "step": 6789
    },
    {
      "epoch": 0.8885253946184674,
      "grad_norm": 6.119810581207275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06481383740901947,
      "logits/rejected": 0.19191932678222656,
      "logps/chosen": -388.1300964355469,
      "logps/rejected": -358.8489685058594,
      "loss": 0.4557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26685523986816406,
      "rewards/margins": 1.1013199090957642,
      "rewards/rejected": -0.8344646096229553,
      "step": 6790
    },
    {
      "epoch": 0.888656252555819,
      "grad_norm": 6.330267429351807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024521350860595703,
      "logits/rejected": -0.20933762192726135,
      "logps/chosen": -513.3026733398438,
      "logps/rejected": -546.5389404296875,
      "loss": 0.3357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12648524343967438,
      "rewards/margins": 2.2627062797546387,
      "rewards/rejected": -2.3891916275024414,
      "step": 6791
    },
    {
      "epoch": 0.8887871104931708,
      "grad_norm": 8.403197288513184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05468476191163063,
      "logits/rejected": 0.05252593755722046,
      "logps/chosen": -518.4592895507812,
      "logps/rejected": -675.65380859375,
      "loss": 0.3096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2066301703453064,
      "rewards/margins": 2.0854272842407227,
      "rewards/rejected": -1.8787972927093506,
      "step": 6792
    },
    {
      "epoch": 0.8889179684305226,
      "grad_norm": 7.3558030128479,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06192263215780258,
      "logits/rejected": -0.2190871238708496,
      "logps/chosen": -500.34600830078125,
      "logps/rejected": -499.64263916015625,
      "loss": 0.5948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.53248131275177,
      "rewards/margins": 0.9437259435653687,
      "rewards/rejected": -1.4762073755264282,
      "step": 6793
    },
    {
      "epoch": 0.8890488263678744,
      "grad_norm": 6.792753219604492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32268932461738586,
      "logits/rejected": -0.3483903408050537,
      "logps/chosen": -414.771240234375,
      "logps/rejected": -406.8270263671875,
      "loss": 0.4758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1296718120574951,
      "rewards/margins": 1.3323454856872559,
      "rewards/rejected": -2.462017297744751,
      "step": 6794
    },
    {
      "epoch": 0.8891796843052261,
      "grad_norm": 4.893947601318359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032292306423187256,
      "logits/rejected": -0.08588074892759323,
      "logps/chosen": -348.11328125,
      "logps/rejected": -483.92425537109375,
      "loss": 0.2616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2655979096889496,
      "rewards/margins": 1.5570902824401855,
      "rewards/rejected": -1.8226882219314575,
      "step": 6795
    },
    {
      "epoch": 0.8893105422425779,
      "grad_norm": 9.585395812988281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07137122005224228,
      "logits/rejected": -0.013808352872729301,
      "logps/chosen": -592.5440063476562,
      "logps/rejected": -460.2550048828125,
      "loss": 0.4814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.40220123529434204,
      "rewards/margins": 1.2072317600250244,
      "rewards/rejected": -0.8050304651260376,
      "step": 6796
    },
    {
      "epoch": 0.8894414001799297,
      "grad_norm": 7.9320454597473145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05858302488923073,
      "logits/rejected": -0.12751364707946777,
      "logps/chosen": -467.8831787109375,
      "logps/rejected": -406.9200439453125,
      "loss": 0.5712,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05185261368751526,
      "rewards/margins": 0.688515305519104,
      "rewards/rejected": -0.6366626620292664,
      "step": 6797
    },
    {
      "epoch": 0.8895722581172815,
      "grad_norm": 6.658123970031738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11763882637023926,
      "logits/rejected": 0.09999856352806091,
      "logps/chosen": -384.8275146484375,
      "logps/rejected": -431.583251953125,
      "loss": 0.5555,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4549017548561096,
      "rewards/margins": 0.38942795991897583,
      "rewards/rejected": -0.8443296551704407,
      "step": 6798
    },
    {
      "epoch": 0.8897031160546331,
      "grad_norm": 8.259483337402344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14448107779026031,
      "logits/rejected": 0.26025184988975525,
      "logps/chosen": -518.39306640625,
      "logps/rejected": -517.442138671875,
      "loss": 0.5171,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1520233005285263,
      "rewards/margins": 0.81927090883255,
      "rewards/rejected": -0.6672475337982178,
      "step": 6799
    },
    {
      "epoch": 0.8898339739919849,
      "grad_norm": 7.841831684112549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2860907316207886,
      "logits/rejected": 0.35683509707450867,
      "logps/chosen": -335.9966735839844,
      "logps/rejected": -342.07598876953125,
      "loss": 0.7558,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3784223794937134,
      "rewards/margins": 0.2692151665687561,
      "rewards/rejected": -0.6476375460624695,
      "step": 6800
    },
    {
      "epoch": 0.8899648319293367,
      "grad_norm": 6.46565580368042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3838609755039215,
      "logits/rejected": -0.37497517466545105,
      "logps/chosen": -397.3448181152344,
      "logps/rejected": -418.9011535644531,
      "loss": 0.3221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.257941484451294,
      "rewards/margins": 1.1916284561157227,
      "rewards/rejected": -2.4495699405670166,
      "step": 6801
    },
    {
      "epoch": 0.8900956898666885,
      "grad_norm": 9.085612297058105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007177241146564484,
      "logits/rejected": -0.029490087181329727,
      "logps/chosen": -441.7445983886719,
      "logps/rejected": -368.3287658691406,
      "loss": 0.7365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29476624727249146,
      "rewards/margins": 0.4091176986694336,
      "rewards/rejected": -0.11435145139694214,
      "step": 6802
    },
    {
      "epoch": 0.8902265478040402,
      "grad_norm": 5.699561595916748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1831670105457306,
      "logits/rejected": -0.1899033635854721,
      "logps/chosen": -309.7316589355469,
      "logps/rejected": -403.9065246582031,
      "loss": 0.3965,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9816111326217651,
      "rewards/margins": 1.347068190574646,
      "rewards/rejected": -2.328679084777832,
      "step": 6803
    },
    {
      "epoch": 0.890357405741392,
      "grad_norm": 5.63954496383667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17493630945682526,
      "logits/rejected": -0.39351680874824524,
      "logps/chosen": -398.64593505859375,
      "logps/rejected": -393.8355712890625,
      "loss": 0.3124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8405231237411499,
      "rewards/margins": 1.2346590757369995,
      "rewards/rejected": -2.0751821994781494,
      "step": 6804
    },
    {
      "epoch": 0.8904882636787438,
      "grad_norm": 7.336556911468506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06891407072544098,
      "logits/rejected": -0.06261157989501953,
      "logps/chosen": -575.2521362304688,
      "logps/rejected": -867.12255859375,
      "loss": 0.5075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5253515243530273,
      "rewards/margins": 1.3411391973495483,
      "rewards/rejected": -0.815787672996521,
      "step": 6805
    },
    {
      "epoch": 0.8906191216160956,
      "grad_norm": 9.605602264404297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06836500018835068,
      "logits/rejected": 0.19768795371055603,
      "logps/chosen": -612.8626098632812,
      "logps/rejected": -620.3506469726562,
      "loss": 0.7158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009136952459812164,
      "rewards/margins": 0.30145740509033203,
      "rewards/rejected": -0.3105943500995636,
      "step": 6806
    },
    {
      "epoch": 0.8907499795534473,
      "grad_norm": 6.616172790527344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1600043922662735,
      "logits/rejected": -0.40034374594688416,
      "logps/chosen": -379.3610534667969,
      "logps/rejected": -374.7632141113281,
      "loss": 0.5456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21713504195213318,
      "rewards/margins": 0.5644699931144714,
      "rewards/rejected": -0.7816049456596375,
      "step": 6807
    },
    {
      "epoch": 0.890880837490799,
      "grad_norm": 8.003332138061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30905479192733765,
      "logits/rejected": -0.430629700422287,
      "logps/chosen": -568.2869262695312,
      "logps/rejected": -419.63043212890625,
      "loss": 0.5267,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1485111266374588,
      "rewards/margins": 0.7462301254272461,
      "rewards/rejected": -0.5977190732955933,
      "step": 6808
    },
    {
      "epoch": 0.8910116954281508,
      "grad_norm": 8.720979690551758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0191575326025486,
      "logits/rejected": -0.21683791279792786,
      "logps/chosen": -635.8076171875,
      "logps/rejected": -474.46405029296875,
      "loss": 0.7203,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06763458251953125,
      "rewards/margins": 0.9099524617195129,
      "rewards/rejected": -0.8423178195953369,
      "step": 6809
    },
    {
      "epoch": 0.8911425533655025,
      "grad_norm": 5.652780532836914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12840056419372559,
      "logits/rejected": -0.08615504950284958,
      "logps/chosen": -523.8764038085938,
      "logps/rejected": -501.391845703125,
      "loss": 0.5265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04252377152442932,
      "rewards/margins": 1.2665128707885742,
      "rewards/rejected": -1.3090367317199707,
      "step": 6810
    },
    {
      "epoch": 0.8912734113028543,
      "grad_norm": 6.226598739624023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1833573579788208,
      "logits/rejected": 0.08182960003614426,
      "logps/chosen": -567.7821044921875,
      "logps/rejected": -524.6937866210938,
      "loss": 0.4919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.59383225440979,
      "rewards/margins": 0.8359867930412292,
      "rewards/rejected": -0.24215452373027802,
      "step": 6811
    },
    {
      "epoch": 0.8914042692402061,
      "grad_norm": 9.17705249786377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23898190259933472,
      "logits/rejected": -0.47004181146621704,
      "logps/chosen": -537.7889404296875,
      "logps/rejected": -472.1108093261719,
      "loss": 0.6979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.067832350730896,
      "rewards/margins": 0.7145804166793823,
      "rewards/rejected": -1.7824126482009888,
      "step": 6812
    },
    {
      "epoch": 0.8915351271775579,
      "grad_norm": 11.266826629638672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08822779357433319,
      "logits/rejected": 0.14891161024570465,
      "logps/chosen": -522.9220581054688,
      "logps/rejected": -582.3182983398438,
      "loss": 0.8024,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008660905063152313,
      "rewards/margins": 0.5814563035964966,
      "rewards/rejected": -0.5901172161102295,
      "step": 6813
    },
    {
      "epoch": 0.8916659851149096,
      "grad_norm": 8.563942909240723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6964993476867676,
      "logits/rejected": -0.5750181674957275,
      "logps/chosen": -474.3201904296875,
      "logps/rejected": -511.027099609375,
      "loss": 0.4267,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8049936890602112,
      "rewards/margins": 1.205052137374878,
      "rewards/rejected": -2.0100455284118652,
      "step": 6814
    },
    {
      "epoch": 0.8917968430522614,
      "grad_norm": 12.82522201538086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12646090984344482,
      "logits/rejected": -0.1934547871351242,
      "logps/chosen": -393.273193359375,
      "logps/rejected": -405.253662109375,
      "loss": 1.3116,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.2968904972076416,
      "rewards/margins": -0.8609399795532227,
      "rewards/rejected": -0.43595051765441895,
      "step": 6815
    },
    {
      "epoch": 0.8919277009896132,
      "grad_norm": 5.03122615814209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37771645188331604,
      "logits/rejected": -0.3622968792915344,
      "logps/chosen": -454.9351806640625,
      "logps/rejected": -443.4789733886719,
      "loss": 0.4668,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17863905429840088,
      "rewards/margins": 1.1927746534347534,
      "rewards/rejected": -1.0141355991363525,
      "step": 6816
    },
    {
      "epoch": 0.892058558926965,
      "grad_norm": 5.709699630737305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16219592094421387,
      "logits/rejected": 0.10322505235671997,
      "logps/chosen": -504.80694580078125,
      "logps/rejected": -532.2430419921875,
      "loss": 0.3007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3200756311416626,
      "rewards/margins": 1.832401990890503,
      "rewards/rejected": -1.5123264789581299,
      "step": 6817
    },
    {
      "epoch": 0.8921894168643166,
      "grad_norm": 5.242959022521973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006224468350410461,
      "logits/rejected": -0.1562693566083908,
      "logps/chosen": -382.30731201171875,
      "logps/rejected": -315.49261474609375,
      "loss": 0.4109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2551315426826477,
      "rewards/margins": 1.2452301979064941,
      "rewards/rejected": -0.9900985956192017,
      "step": 6818
    },
    {
      "epoch": 0.8923202748016684,
      "grad_norm": 6.063178539276123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15613609552383423,
      "logits/rejected": -0.06501011550426483,
      "logps/chosen": -517.938720703125,
      "logps/rejected": -456.90185546875,
      "loss": 0.4161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.82820725440979,
      "rewards/margins": 1.1259946823120117,
      "rewards/rejected": -0.29778751730918884,
      "step": 6819
    },
    {
      "epoch": 0.8924511327390202,
      "grad_norm": 8.315559387207031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5161476135253906,
      "logits/rejected": -0.22226247191429138,
      "logps/chosen": -470.168212890625,
      "logps/rejected": -553.2781982421875,
      "loss": 0.5423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3832361102104187,
      "rewards/margins": 0.8559346199035645,
      "rewards/rejected": -0.4726986289024353,
      "step": 6820
    },
    {
      "epoch": 0.892581990676372,
      "grad_norm": 6.108231067657471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009160332381725311,
      "logits/rejected": -0.06474357843399048,
      "logps/chosen": -462.49664306640625,
      "logps/rejected": -457.0100402832031,
      "loss": 0.3489,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12568911910057068,
      "rewards/margins": 1.5854930877685547,
      "rewards/rejected": -1.4598039388656616,
      "step": 6821
    },
    {
      "epoch": 0.8927128486137237,
      "grad_norm": 4.967262268066406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2690867781639099,
      "logits/rejected": 0.07210712879896164,
      "logps/chosen": -295.0253601074219,
      "logps/rejected": -457.9196472167969,
      "loss": 0.3725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16833201050758362,
      "rewards/margins": 0.9241066575050354,
      "rewards/rejected": -0.7557746171951294,
      "step": 6822
    },
    {
      "epoch": 0.8928437065510755,
      "grad_norm": 6.677704811096191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3491940498352051,
      "logits/rejected": 0.0912126824259758,
      "logps/chosen": -560.8865356445312,
      "logps/rejected": -513.3466186523438,
      "loss": 0.4837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1595333069562912,
      "rewards/margins": 0.5640321969985962,
      "rewards/rejected": -0.7235654592514038,
      "step": 6823
    },
    {
      "epoch": 0.8929745644884273,
      "grad_norm": 5.603144645690918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21251598000526428,
      "logits/rejected": -0.2345552146434784,
      "logps/chosen": -325.7992248535156,
      "logps/rejected": -497.7560729980469,
      "loss": 0.2671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5352867841720581,
      "rewards/margins": 2.017944812774658,
      "rewards/rejected": -2.5532312393188477,
      "step": 6824
    },
    {
      "epoch": 0.8931054224257791,
      "grad_norm": 9.817425727844238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.094589963555336,
      "logits/rejected": -0.10379700362682343,
      "logps/chosen": -490.29559326171875,
      "logps/rejected": -479.7583312988281,
      "loss": 0.7657,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2938762903213501,
      "rewards/margins": 0.17553648352622986,
      "rewards/rejected": -0.46941274404525757,
      "step": 6825
    },
    {
      "epoch": 0.8932362803631307,
      "grad_norm": 5.137843608856201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00639900378882885,
      "logits/rejected": -0.004026656039059162,
      "logps/chosen": -549.1688232421875,
      "logps/rejected": -518.6400756835938,
      "loss": 0.4216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8030153512954712,
      "rewards/margins": 1.0682159662246704,
      "rewards/rejected": -0.26520055532455444,
      "step": 6826
    },
    {
      "epoch": 0.8933671383004825,
      "grad_norm": 5.937886714935303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4302882254123688,
      "logits/rejected": -0.5416412353515625,
      "logps/chosen": -373.7713623046875,
      "logps/rejected": -415.87799072265625,
      "loss": 0.398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.790667712688446,
      "rewards/margins": 1.6131960153579712,
      "rewards/rejected": -2.4038636684417725,
      "step": 6827
    },
    {
      "epoch": 0.8934979962378343,
      "grad_norm": 8.76630687713623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5111494064331055,
      "logits/rejected": 0.36160755157470703,
      "logps/chosen": -423.56964111328125,
      "logps/rejected": -347.86090087890625,
      "loss": 0.8137,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04654863476753235,
      "rewards/margins": 0.9366890788078308,
      "rewards/rejected": -0.9832375645637512,
      "step": 6828
    },
    {
      "epoch": 0.8936288541751861,
      "grad_norm": 5.254186630249023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20974422991275787,
      "logits/rejected": 0.14877849817276,
      "logps/chosen": -382.2137756347656,
      "logps/rejected": -475.79449462890625,
      "loss": 0.3002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07496492564678192,
      "rewards/margins": 1.460116982460022,
      "rewards/rejected": -1.5350818634033203,
      "step": 6829
    },
    {
      "epoch": 0.8937597121125378,
      "grad_norm": 10.112221717834473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2546590268611908,
      "logits/rejected": -0.15256446599960327,
      "logps/chosen": -576.9203491210938,
      "logps/rejected": -358.709228515625,
      "loss": 0.625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19916993379592896,
      "rewards/margins": 0.9588414430618286,
      "rewards/rejected": -1.1580114364624023,
      "step": 6830
    },
    {
      "epoch": 0.8938905700498896,
      "grad_norm": 6.704100131988525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22800765931606293,
      "logits/rejected": -0.19029128551483154,
      "logps/chosen": -386.1057434082031,
      "logps/rejected": -445.2825927734375,
      "loss": 0.3349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1011989414691925,
      "rewards/margins": 1.7126152515411377,
      "rewards/rejected": -1.8138142824172974,
      "step": 6831
    },
    {
      "epoch": 0.8940214279872414,
      "grad_norm": 8.598694801330566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23520639538764954,
      "logits/rejected": 0.21925678849220276,
      "logps/chosen": -399.793701171875,
      "logps/rejected": -473.7272033691406,
      "loss": 0.7043,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.040139585733413696,
      "rewards/margins": 0.42449262738227844,
      "rewards/rejected": -0.4646322429180145,
      "step": 6832
    },
    {
      "epoch": 0.8941522859245931,
      "grad_norm": 9.033418655395508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.057295061647892,
      "logits/rejected": 0.15960077941417694,
      "logps/chosen": -510.63641357421875,
      "logps/rejected": -564.4417724609375,
      "loss": 0.6337,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16933536529541016,
      "rewards/margins": 0.5543001294136047,
      "rewards/rejected": -0.3849647641181946,
      "step": 6833
    },
    {
      "epoch": 0.8942831438619449,
      "grad_norm": 7.114795684814453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20218917727470398,
      "logits/rejected": -0.1847008764743805,
      "logps/chosen": -556.7138671875,
      "logps/rejected": -548.2197875976562,
      "loss": 0.4725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12400016188621521,
      "rewards/margins": 0.8243343234062195,
      "rewards/rejected": -0.7003341913223267,
      "step": 6834
    },
    {
      "epoch": 0.8944140017992966,
      "grad_norm": 6.217244625091553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025341741740703583,
      "logits/rejected": -0.01589447259902954,
      "logps/chosen": -537.140380859375,
      "logps/rejected": -525.7676391601562,
      "loss": 0.4069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17279395461082458,
      "rewards/margins": 0.7874549627304077,
      "rewards/rejected": -0.9602490663528442,
      "step": 6835
    },
    {
      "epoch": 0.8945448597366484,
      "grad_norm": 2.7108147144317627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08566766232252121,
      "logits/rejected": 0.025022979825735092,
      "logps/chosen": -380.2165832519531,
      "logps/rejected": -437.25238037109375,
      "loss": 0.1567,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1762305200099945,
      "rewards/margins": 2.0687389373779297,
      "rewards/rejected": -2.244969606399536,
      "step": 6836
    },
    {
      "epoch": 0.8946757176740001,
      "grad_norm": 7.201125144958496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29626503586769104,
      "logits/rejected": -0.39138150215148926,
      "logps/chosen": -495.1724548339844,
      "logps/rejected": -577.8925170898438,
      "loss": 0.5732,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.712262749671936,
      "rewards/margins": 0.6433553695678711,
      "rewards/rejected": -1.3556181192398071,
      "step": 6837
    },
    {
      "epoch": 0.8948065756113519,
      "grad_norm": 7.375085830688477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22721533477306366,
      "logits/rejected": 0.16866502165794373,
      "logps/chosen": -424.9231262207031,
      "logps/rejected": -407.6615905761719,
      "loss": 0.4913,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.00415019690990448,
      "rewards/margins": 0.9809156656265259,
      "rewards/rejected": -0.9850658178329468,
      "step": 6838
    },
    {
      "epoch": 0.8949374335487037,
      "grad_norm": 9.98381519317627,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1244991198182106,
      "logits/rejected": -0.0172636229544878,
      "logps/chosen": -454.2362365722656,
      "logps/rejected": -504.0753173828125,
      "loss": 0.9963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.21858663856983185,
      "rewards/margins": 0.2573074996471405,
      "rewards/rejected": -0.03872091323137283,
      "step": 6839
    },
    {
      "epoch": 0.8950682914860555,
      "grad_norm": 7.528152942657471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06273005902767181,
      "logits/rejected": 0.14802466332912445,
      "logps/chosen": -536.51220703125,
      "logps/rejected": -407.62420654296875,
      "loss": 0.3977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43017083406448364,
      "rewards/margins": 1.401563286781311,
      "rewards/rejected": -0.9713923931121826,
      "step": 6840
    },
    {
      "epoch": 0.8951991494234072,
      "grad_norm": 8.803707122802734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08942651748657227,
      "logits/rejected": -0.0938800796866417,
      "logps/chosen": -346.7807922363281,
      "logps/rejected": -358.5844421386719,
      "loss": 0.4601,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7027736902236938,
      "rewards/margins": 1.2300705909729004,
      "rewards/rejected": -1.9328441619873047,
      "step": 6841
    },
    {
      "epoch": 0.895330007360759,
      "grad_norm": 7.9383039474487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0787678211927414,
      "logits/rejected": -0.09229959547519684,
      "logps/chosen": -662.5433959960938,
      "logps/rejected": -594.5057983398438,
      "loss": 0.4586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5911709070205688,
      "rewards/margins": 1.1053447723388672,
      "rewards/rejected": -0.5141739845275879,
      "step": 6842
    },
    {
      "epoch": 0.8954608652981108,
      "grad_norm": 2.9960520267486572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1080472469329834,
      "logits/rejected": 0.03838887810707092,
      "logps/chosen": -335.8775634765625,
      "logps/rejected": -489.89447021484375,
      "loss": 0.2007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3323601484298706,
      "rewards/margins": 1.9452626705169678,
      "rewards/rejected": -2.277622699737549,
      "step": 6843
    },
    {
      "epoch": 0.8955917232354625,
      "grad_norm": 5.10624361038208,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04887865483760834,
      "logits/rejected": -0.04781223461031914,
      "logps/chosen": -515.9964599609375,
      "logps/rejected": -363.366943359375,
      "loss": 0.4318,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.3759427070617676,
      "rewards/margins": 1.3775428533554077,
      "rewards/rejected": -0.0016000419855117798,
      "step": 6844
    },
    {
      "epoch": 0.8957225811728142,
      "grad_norm": 10.564436912536621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009537309408187866,
      "logits/rejected": -0.0016496680909767747,
      "logps/chosen": -434.35931396484375,
      "logps/rejected": -398.868896484375,
      "loss": 0.6392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23083609342575073,
      "rewards/margins": 0.8165674209594727,
      "rewards/rejected": -0.5857313275337219,
      "step": 6845
    },
    {
      "epoch": 0.895853439110166,
      "grad_norm": 8.60001277923584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4993569552898407,
      "logits/rejected": -0.42496854066848755,
      "logps/chosen": -438.5654602050781,
      "logps/rejected": -603.3118286132812,
      "loss": 0.6448,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4321487545967102,
      "rewards/margins": 0.9605036973953247,
      "rewards/rejected": -1.3926526308059692,
      "step": 6846
    },
    {
      "epoch": 0.8959842970475178,
      "grad_norm": 4.809382438659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002218291163444519,
      "logits/rejected": -0.1969211995601654,
      "logps/chosen": -488.9491882324219,
      "logps/rejected": -483.1358642578125,
      "loss": 0.2381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0934303998947144,
      "rewards/margins": 1.6219943761825562,
      "rewards/rejected": -0.5285638570785522,
      "step": 6847
    },
    {
      "epoch": 0.8961151549848696,
      "grad_norm": 5.536828517913818,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0351402647793293,
      "logits/rejected": -0.1516004502773285,
      "logps/chosen": -389.91943359375,
      "logps/rejected": -334.34674072265625,
      "loss": 0.4601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3826878070831299,
      "rewards/margins": 0.7845459580421448,
      "rewards/rejected": -0.40185824036598206,
      "step": 6848
    },
    {
      "epoch": 0.8962460129222213,
      "grad_norm": 3.631925106048584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014230147004127502,
      "logits/rejected": 0.1448291540145874,
      "logps/chosen": -604.6177368164062,
      "logps/rejected": -559.4713745117188,
      "loss": 0.1521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9245864748954773,
      "rewards/margins": 2.1105263233184814,
      "rewards/rejected": -1.1859397888183594,
      "step": 6849
    },
    {
      "epoch": 0.8963768708595731,
      "grad_norm": 5.404338836669922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01732175052165985,
      "logits/rejected": -0.20391586422920227,
      "logps/chosen": -507.774658203125,
      "logps/rejected": -407.1669616699219,
      "loss": 0.2632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23094499111175537,
      "rewards/margins": 1.3807048797607422,
      "rewards/rejected": -1.6116498708724976,
      "step": 6850
    },
    {
      "epoch": 0.8965077287969249,
      "grad_norm": 6.843123912811279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14882832765579224,
      "logits/rejected": -0.07003460824489594,
      "logps/chosen": -407.2384033203125,
      "logps/rejected": -413.3135986328125,
      "loss": 0.5661,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4361134171485901,
      "rewards/margins": 0.5598064064979553,
      "rewards/rejected": -0.9959198236465454,
      "step": 6851
    },
    {
      "epoch": 0.8966385867342767,
      "grad_norm": 6.78715181350708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11054693907499313,
      "logits/rejected": -0.04660937935113907,
      "logps/chosen": -487.84039306640625,
      "logps/rejected": -374.1177673339844,
      "loss": 0.3931,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3713523745536804,
      "rewards/margins": 0.8401632905006409,
      "rewards/rejected": -1.2115156650543213,
      "step": 6852
    },
    {
      "epoch": 0.8967694446716283,
      "grad_norm": 4.76909875869751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04519286006689072,
      "logits/rejected": -0.3180650770664215,
      "logps/chosen": -355.4525451660156,
      "logps/rejected": -449.6929626464844,
      "loss": 0.2699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09427099674940109,
      "rewards/margins": 1.9853447675704956,
      "rewards/rejected": -1.8910737037658691,
      "step": 6853
    },
    {
      "epoch": 0.8969003026089801,
      "grad_norm": 6.75443172454834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021053781732916832,
      "logits/rejected": -0.1389060765504837,
      "logps/chosen": -408.7658386230469,
      "logps/rejected": -390.583740234375,
      "loss": 0.4043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23355597257614136,
      "rewards/margins": 1.4297916889190674,
      "rewards/rejected": -1.6633477210998535,
      "step": 6854
    },
    {
      "epoch": 0.8970311605463319,
      "grad_norm": 7.788966178894043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1290319859981537,
      "logits/rejected": -0.10133837163448334,
      "logps/chosen": -468.55999755859375,
      "logps/rejected": -363.5875549316406,
      "loss": 0.5478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.038374509662389755,
      "rewards/margins": 1.177753210067749,
      "rewards/rejected": -1.1393787860870361,
      "step": 6855
    },
    {
      "epoch": 0.8971620184836836,
      "grad_norm": 5.647534370422363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05693986266851425,
      "logits/rejected": -0.034701474010944366,
      "logps/chosen": -600.37548828125,
      "logps/rejected": -447.10546875,
      "loss": 0.3138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5885776281356812,
      "rewards/margins": 1.1754218339920044,
      "rewards/rejected": -0.5868442058563232,
      "step": 6856
    },
    {
      "epoch": 0.8972928764210354,
      "grad_norm": 10.144125938415527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1851026564836502,
      "logits/rejected": 0.08691040426492691,
      "logps/chosen": -438.7098083496094,
      "logps/rejected": -500.1932678222656,
      "loss": 0.8214,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6829246282577515,
      "rewards/margins": 0.3400169312953949,
      "rewards/rejected": -1.0229415893554688,
      "step": 6857
    },
    {
      "epoch": 0.8974237343583872,
      "grad_norm": 7.414768695831299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2273489534854889,
      "logits/rejected": -0.28216952085494995,
      "logps/chosen": -394.2911376953125,
      "logps/rejected": -336.9400939941406,
      "loss": 0.4934,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.297956109046936,
      "rewards/margins": 0.7008634209632874,
      "rewards/rejected": -1.9988195896148682,
      "step": 6858
    },
    {
      "epoch": 0.897554592295739,
      "grad_norm": 18.64371109008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04438653588294983,
      "logits/rejected": -0.1949368268251419,
      "logps/chosen": -587.2369384765625,
      "logps/rejected": -467.45880126953125,
      "loss": 1.1029,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0310672521591187,
      "rewards/margins": 0.5919981002807617,
      "rewards/rejected": -1.6230653524398804,
      "step": 6859
    },
    {
      "epoch": 0.8976854502330907,
      "grad_norm": 8.060914993286133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044872235506772995,
      "logits/rejected": -0.08650758862495422,
      "logps/chosen": -486.92218017578125,
      "logps/rejected": -441.4599914550781,
      "loss": 0.4705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021121598780155182,
      "rewards/margins": 1.0489619970321655,
      "rewards/rejected": -1.027840495109558,
      "step": 6860
    },
    {
      "epoch": 0.8978163081704424,
      "grad_norm": 7.210089206695557,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09389255940914154,
      "logits/rejected": 0.16402260959148407,
      "logps/chosen": -337.0416259765625,
      "logps/rejected": -454.56646728515625,
      "loss": 0.423,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18556518852710724,
      "rewards/margins": 1.5377179384231567,
      "rewards/rejected": -1.352152705192566,
      "step": 6861
    },
    {
      "epoch": 0.8979471661077942,
      "grad_norm": 7.241957187652588,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12633241713047028,
      "logits/rejected": -0.18332763016223907,
      "logps/chosen": -423.31011962890625,
      "logps/rejected": -329.4774169921875,
      "loss": 0.6828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19945409893989563,
      "rewards/margins": 0.9353991746902466,
      "rewards/rejected": -1.1348532438278198,
      "step": 6862
    },
    {
      "epoch": 0.898078024045146,
      "grad_norm": 11.85054874420166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2637656629085541,
      "logits/rejected": 0.17232303321361542,
      "logps/chosen": -547.3040161132812,
      "logps/rejected": -436.8518371582031,
      "loss": 0.5808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2153356373310089,
      "rewards/margins": 2.2025949954986572,
      "rewards/rejected": -1.9872595071792603,
      "step": 6863
    },
    {
      "epoch": 0.8982088819824977,
      "grad_norm": 4.312586784362793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12390085309743881,
      "logits/rejected": -0.10840029269456863,
      "logps/chosen": -440.64959716796875,
      "logps/rejected": -373.55511474609375,
      "loss": 0.2359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37482398748397827,
      "rewards/margins": 2.0859556198120117,
      "rewards/rejected": -1.7111315727233887,
      "step": 6864
    },
    {
      "epoch": 0.8983397399198495,
      "grad_norm": 7.761487007141113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024112723767757416,
      "logits/rejected": -0.07621556520462036,
      "logps/chosen": -373.9292907714844,
      "logps/rejected": -371.86041259765625,
      "loss": 0.4497,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1890115588903427,
      "rewards/margins": 0.9910732507705688,
      "rewards/rejected": -1.1800847053527832,
      "step": 6865
    },
    {
      "epoch": 0.8984705978572013,
      "grad_norm": 7.978231430053711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0518454946577549,
      "logits/rejected": -0.07839858531951904,
      "logps/chosen": -585.2743530273438,
      "logps/rejected": -512.781982421875,
      "loss": 0.3928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6663926839828491,
      "rewards/margins": 1.888883113861084,
      "rewards/rejected": -1.2224905490875244,
      "step": 6866
    },
    {
      "epoch": 0.8986014557945531,
      "grad_norm": 6.339234352111816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25803300738334656,
      "logits/rejected": 0.22500355541706085,
      "logps/chosen": -495.43798828125,
      "logps/rejected": -474.59393310546875,
      "loss": 0.3224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18954810500144958,
      "rewards/margins": 1.4797801971435547,
      "rewards/rejected": -1.2902318239212036,
      "step": 6867
    },
    {
      "epoch": 0.8987323137319048,
      "grad_norm": 7.862349033355713,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07729299366474152,
      "logits/rejected": 0.24819259345531464,
      "logps/chosen": -372.48883056640625,
      "logps/rejected": -514.5118408203125,
      "loss": 0.3816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23242540657520294,
      "rewards/margins": 1.2576841115951538,
      "rewards/rejected": -1.4901094436645508,
      "step": 6868
    },
    {
      "epoch": 0.8988631716692566,
      "grad_norm": 6.299263954162598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10167708992958069,
      "logits/rejected": 0.14145442843437195,
      "logps/chosen": -491.5315856933594,
      "logps/rejected": -544.9269409179688,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14806421101093292,
      "rewards/margins": 1.1722506284713745,
      "rewards/rejected": -1.3203147649765015,
      "step": 6869
    },
    {
      "epoch": 0.8989940296066083,
      "grad_norm": 8.903888702392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0370856449007988,
      "logits/rejected": -0.11694451421499252,
      "logps/chosen": -450.81048583984375,
      "logps/rejected": -618.1388549804688,
      "loss": 0.5177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0493828058242798,
      "rewards/margins": 1.242612361907959,
      "rewards/rejected": -2.2919952869415283,
      "step": 6870
    },
    {
      "epoch": 0.8991248875439601,
      "grad_norm": 7.925394535064697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05712662637233734,
      "logits/rejected": 0.12497372180223465,
      "logps/chosen": -562.8829345703125,
      "logps/rejected": -540.0154418945312,
      "loss": 0.4824,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21873357892036438,
      "rewards/margins": 1.2157363891601562,
      "rewards/rejected": -1.4344699382781982,
      "step": 6871
    },
    {
      "epoch": 0.8992557454813118,
      "grad_norm": 3.876634359359741,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04639234393835068,
      "logits/rejected": 0.012060888111591339,
      "logps/chosen": -473.0127868652344,
      "logps/rejected": -439.2259216308594,
      "loss": 0.1857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.51772540807724,
      "rewards/margins": 2.399178981781006,
      "rewards/rejected": -1.881453514099121,
      "step": 6872
    },
    {
      "epoch": 0.8993866034186636,
      "grad_norm": 10.274747848510742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0871981829404831,
      "logits/rejected": -0.06720968335866928,
      "logps/chosen": -613.5320434570312,
      "logps/rejected": -493.20574951171875,
      "loss": 0.6705,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1452395915985107,
      "rewards/margins": 0.3000088036060333,
      "rewards/rejected": -1.4452483654022217,
      "step": 6873
    },
    {
      "epoch": 0.8995174613560154,
      "grad_norm": 5.002893924713135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1344146430492401,
      "logits/rejected": -0.08164900541305542,
      "logps/chosen": -354.9862060546875,
      "logps/rejected": -413.57989501953125,
      "loss": 0.2668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19715234637260437,
      "rewards/margins": 1.940993070602417,
      "rewards/rejected": -1.7438405752182007,
      "step": 6874
    },
    {
      "epoch": 0.8996483192933672,
      "grad_norm": 8.270302772521973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07771538197994232,
      "logits/rejected": 0.15988366305828094,
      "logps/chosen": -438.9187316894531,
      "logps/rejected": -432.23638916015625,
      "loss": 0.4998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.020912200212478638,
      "rewards/margins": 1.0711088180541992,
      "rewards/rejected": -1.0501967668533325,
      "step": 6875
    },
    {
      "epoch": 0.8997791772307189,
      "grad_norm": 9.452603340148926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0071936920285224915,
      "logits/rejected": -0.04542285203933716,
      "logps/chosen": -494.6370544433594,
      "logps/rejected": -541.8390502929688,
      "loss": 0.9483,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.32466641068458557,
      "rewards/margins": 0.19337067008018494,
      "rewards/rejected": -0.518036961555481,
      "step": 6876
    },
    {
      "epoch": 0.8999100351680707,
      "grad_norm": 7.376235008239746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17881272733211517,
      "logits/rejected": 0.07562148571014404,
      "logps/chosen": -508.2501220703125,
      "logps/rejected": -418.23699951171875,
      "loss": 0.468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22477340698242188,
      "rewards/margins": 0.9149519205093384,
      "rewards/rejected": -1.1397253274917603,
      "step": 6877
    },
    {
      "epoch": 0.9000408931054225,
      "grad_norm": 8.704963684082031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09249402582645416,
      "logits/rejected": 0.1286775767803192,
      "logps/chosen": -454.6300964355469,
      "logps/rejected": -493.86663818359375,
      "loss": 0.5959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8621849417686462,
      "rewards/margins": 0.8796918392181396,
      "rewards/rejected": -0.01750696450471878,
      "step": 6878
    },
    {
      "epoch": 0.9001717510427741,
      "grad_norm": 12.122474670410156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01834740862250328,
      "logits/rejected": -0.02503199875354767,
      "logps/chosen": -360.3183898925781,
      "logps/rejected": -378.80767822265625,
      "loss": 0.8225,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2883793115615845,
      "rewards/margins": 0.1598939746618271,
      "rewards/rejected": -0.44827330112457275,
      "step": 6879
    },
    {
      "epoch": 0.9003026089801259,
      "grad_norm": 6.32622766494751,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3217557966709137,
      "logits/rejected": -0.11648599803447723,
      "logps/chosen": -378.6678161621094,
      "logps/rejected": -432.1167907714844,
      "loss": 0.4328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09929123520851135,
      "rewards/margins": 1.703735113143921,
      "rewards/rejected": -1.6044437885284424,
      "step": 6880
    },
    {
      "epoch": 0.9004334669174777,
      "grad_norm": 3.9174983501434326,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07403212785720825,
      "logits/rejected": -0.2684664726257324,
      "logps/chosen": -462.65252685546875,
      "logps/rejected": -441.3894958496094,
      "loss": 0.2713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05548593029379845,
      "rewards/margins": 1.5352104902267456,
      "rewards/rejected": -1.5906962156295776,
      "step": 6881
    },
    {
      "epoch": 0.9005643248548295,
      "grad_norm": 6.566378593444824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06690874695777893,
      "logits/rejected": 0.22499224543571472,
      "logps/chosen": -546.0556640625,
      "logps/rejected": -472.6371154785156,
      "loss": 0.4224,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05086401849985123,
      "rewards/margins": 1.0903607606887817,
      "rewards/rejected": -1.1412246227264404,
      "step": 6882
    },
    {
      "epoch": 0.9006951827921812,
      "grad_norm": 5.681412696838379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03482745587825775,
      "logits/rejected": -0.1573473960161209,
      "logps/chosen": -377.60546875,
      "logps/rejected": -361.08416748046875,
      "loss": 0.3295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06145534664392471,
      "rewards/margins": 1.4072351455688477,
      "rewards/rejected": -1.3457797765731812,
      "step": 6883
    },
    {
      "epoch": 0.900826040729533,
      "grad_norm": 6.822521209716797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037342190742492676,
      "logits/rejected": -0.006931141018867493,
      "logps/chosen": -418.2914123535156,
      "logps/rejected": -417.7764892578125,
      "loss": 0.478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3460373282432556,
      "rewards/margins": 0.6911823153495789,
      "rewards/rejected": -1.037219762802124,
      "step": 6884
    },
    {
      "epoch": 0.9009568986668848,
      "grad_norm": 6.076841354370117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29147443175315857,
      "logits/rejected": 0.03654235973954201,
      "logps/chosen": -394.13525390625,
      "logps/rejected": -388.30859375,
      "loss": 0.5686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05710391700267792,
      "rewards/margins": 0.9946198463439941,
      "rewards/rejected": -0.9375157952308655,
      "step": 6885
    },
    {
      "epoch": 0.9010877566042366,
      "grad_norm": 4.695486545562744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22766122221946716,
      "logits/rejected": 0.11462251096963882,
      "logps/chosen": -426.75836181640625,
      "logps/rejected": -441.23114013671875,
      "loss": 0.2444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03147432208061218,
      "rewards/margins": 1.890873908996582,
      "rewards/rejected": -1.9223483800888062,
      "step": 6886
    },
    {
      "epoch": 0.9012186145415882,
      "grad_norm": 4.847795009613037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26113051176071167,
      "logits/rejected": -0.04914223030209541,
      "logps/chosen": -489.5348815917969,
      "logps/rejected": -459.6596374511719,
      "loss": 0.2421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8004951477050781,
      "rewards/margins": 1.718936800956726,
      "rewards/rejected": -0.918441653251648,
      "step": 6887
    },
    {
      "epoch": 0.90134947247894,
      "grad_norm": 7.654916286468506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2764868140220642,
      "logits/rejected": 0.12307513505220413,
      "logps/chosen": -396.58758544921875,
      "logps/rejected": -424.3458251953125,
      "loss": 0.5934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7222909927368164,
      "rewards/margins": 0.583560049533844,
      "rewards/rejected": -1.305850863456726,
      "step": 6888
    },
    {
      "epoch": 0.9014803304162918,
      "grad_norm": 12.06692886352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035143397748470306,
      "logits/rejected": -0.2125251442193985,
      "logps/chosen": -593.65087890625,
      "logps/rejected": -550.3850708007812,
      "loss": 0.8999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8576227426528931,
      "rewards/margins": 0.284282386302948,
      "rewards/rejected": -1.1419050693511963,
      "step": 6889
    },
    {
      "epoch": 0.9016111883536436,
      "grad_norm": 8.142721176147461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17967429757118225,
      "logits/rejected": -0.02421267330646515,
      "logps/chosen": -443.9934997558594,
      "logps/rejected": -518.1536254882812,
      "loss": 0.5523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37098848819732666,
      "rewards/margins": 1.0504971742630005,
      "rewards/rejected": -1.4214856624603271,
      "step": 6890
    },
    {
      "epoch": 0.9017420462909953,
      "grad_norm": 5.266330242156982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1015758216381073,
      "logits/rejected": 0.08295589685440063,
      "logps/chosen": -400.55157470703125,
      "logps/rejected": -297.9524230957031,
      "loss": 0.4416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3282661437988281,
      "rewards/margins": 0.7935127019882202,
      "rewards/rejected": -1.121778964996338,
      "step": 6891
    },
    {
      "epoch": 0.9018729042283471,
      "grad_norm": 4.664444923400879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01962471194565296,
      "logits/rejected": -0.2556886374950409,
      "logps/chosen": -539.021484375,
      "logps/rejected": -469.43719482421875,
      "loss": 0.2473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0891611576080322,
      "rewards/margins": 2.3611226081848145,
      "rewards/rejected": -1.2719616889953613,
      "step": 6892
    },
    {
      "epoch": 0.9020037621656989,
      "grad_norm": 6.682895183563232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.308694064617157,
      "logits/rejected": 0.0388355553150177,
      "logps/chosen": -373.25341796875,
      "logps/rejected": -355.163330078125,
      "loss": 0.5277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22543278336524963,
      "rewards/margins": 0.762964129447937,
      "rewards/rejected": -0.9883968830108643,
      "step": 6893
    },
    {
      "epoch": 0.9021346201030507,
      "grad_norm": 8.69559097290039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1459961235523224,
      "logits/rejected": -0.12558916211128235,
      "logps/chosen": -781.3363037109375,
      "logps/rejected": -489.4350280761719,
      "loss": 0.4567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26764339208602905,
      "rewards/margins": 0.9026044011116028,
      "rewards/rejected": -1.1702477931976318,
      "step": 6894
    },
    {
      "epoch": 0.9022654780404024,
      "grad_norm": 15.309572219848633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26532864570617676,
      "logits/rejected": -0.27491453289985657,
      "logps/chosen": -569.9537963867188,
      "logps/rejected": -521.2332763671875,
      "loss": 0.8043,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3646650016307831,
      "rewards/margins": 0.2918073534965515,
      "rewards/rejected": -0.6564724445343018,
      "step": 6895
    },
    {
      "epoch": 0.9023963359777541,
      "grad_norm": 12.797497749328613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1255723536014557,
      "logits/rejected": -0.3240455389022827,
      "logps/chosen": -616.6043090820312,
      "logps/rejected": -456.90625,
      "loss": 1.1551,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.092613697052002,
      "rewards/margins": -0.4774256646633148,
      "rewards/rejected": -1.6151881217956543,
      "step": 6896
    },
    {
      "epoch": 0.9025271939151059,
      "grad_norm": 3.3599984645843506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2719631791114807,
      "logits/rejected": -0.35324376821517944,
      "logps/chosen": -478.381591796875,
      "logps/rejected": -406.5074462890625,
      "loss": 0.1632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3831436336040497,
      "rewards/margins": 2.276184558868408,
      "rewards/rejected": -1.8930410146713257,
      "step": 6897
    },
    {
      "epoch": 0.9026580518524576,
      "grad_norm": 13.343585014343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26296576857566833,
      "logits/rejected": -0.11007164418697357,
      "logps/chosen": -338.2001037597656,
      "logps/rejected": -283.5118103027344,
      "loss": 0.926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2847795486450195,
      "rewards/margins": -0.1218394935131073,
      "rewards/rejected": -1.1629400253295898,
      "step": 6898
    },
    {
      "epoch": 0.9027889097898094,
      "grad_norm": 10.227462768554688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06520333886146545,
      "logits/rejected": -0.11136665940284729,
      "logps/chosen": -507.88128662109375,
      "logps/rejected": -459.8233947753906,
      "loss": 0.5725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4357452392578125,
      "rewards/margins": 0.8804482817649841,
      "rewards/rejected": -1.316193699836731,
      "step": 6899
    },
    {
      "epoch": 0.9029197677271612,
      "grad_norm": 11.26302719116211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0925065353512764,
      "logits/rejected": -0.09118450433015823,
      "logps/chosen": -291.5415954589844,
      "logps/rejected": -328.9062194824219,
      "loss": 0.7209,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4549863934516907,
      "rewards/margins": 0.7101500034332275,
      "rewards/rejected": -1.1651362180709839,
      "step": 6900
    },
    {
      "epoch": 0.903050625664513,
      "grad_norm": 5.575297832489014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035050805658102036,
      "logits/rejected": 0.1026141420006752,
      "logps/chosen": -423.84771728515625,
      "logps/rejected": -372.0838623046875,
      "loss": 0.61,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45297491550445557,
      "rewards/margins": 0.909737229347229,
      "rewards/rejected": -1.3627121448516846,
      "step": 6901
    },
    {
      "epoch": 0.9031814836018647,
      "grad_norm": 8.894697189331055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04991241917014122,
      "logits/rejected": -0.014964044094085693,
      "logps/chosen": -357.46533203125,
      "logps/rejected": -367.4928894042969,
      "loss": 0.7576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48617443442344666,
      "rewards/margins": 0.798345148563385,
      "rewards/rejected": -1.2845195531845093,
      "step": 6902
    },
    {
      "epoch": 0.9033123415392165,
      "grad_norm": 6.67518424987793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1371285617351532,
      "logits/rejected": 0.2072860300540924,
      "logps/chosen": -435.2032470703125,
      "logps/rejected": -428.6026306152344,
      "loss": 0.5591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45837417244911194,
      "rewards/margins": 0.6146739721298218,
      "rewards/rejected": -0.15629979968070984,
      "step": 6903
    },
    {
      "epoch": 0.9034431994765683,
      "grad_norm": 7.233400821685791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09484323859214783,
      "logits/rejected": 0.09248567372560501,
      "logps/chosen": -339.44158935546875,
      "logps/rejected": -310.364501953125,
      "loss": 0.5692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5821981430053711,
      "rewards/margins": 0.6666710376739502,
      "rewards/rejected": -0.08447284251451492,
      "step": 6904
    },
    {
      "epoch": 0.90357405741392,
      "grad_norm": 6.339358806610107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1310855895280838,
      "logits/rejected": 0.13428670167922974,
      "logps/chosen": -558.4226684570312,
      "logps/rejected": -405.536376953125,
      "loss": 0.5348,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.19312110543251038,
      "rewards/margins": 0.8822899460792542,
      "rewards/rejected": -0.6891688108444214,
      "step": 6905
    },
    {
      "epoch": 0.9037049153512717,
      "grad_norm": 8.025043487548828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058959007263183594,
      "logits/rejected": 0.026029543951153755,
      "logps/chosen": -422.23931884765625,
      "logps/rejected": -458.91302490234375,
      "loss": 0.6399,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.510729968547821,
      "rewards/margins": 0.7156571745872498,
      "rewards/rejected": -0.2049272656440735,
      "step": 6906
    },
    {
      "epoch": 0.9038357732886235,
      "grad_norm": 7.258016109466553,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07811776548624039,
      "logits/rejected": -0.14156700670719147,
      "logps/chosen": -481.6187438964844,
      "logps/rejected": -391.3682861328125,
      "loss": 0.4406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30310943722724915,
      "rewards/margins": 1.4354827404022217,
      "rewards/rejected": -1.1323732137680054,
      "step": 6907
    },
    {
      "epoch": 0.9039666312259753,
      "grad_norm": 8.406805992126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15142415463924408,
      "logits/rejected": -0.12779022753238678,
      "logps/chosen": -248.03025817871094,
      "logps/rejected": -343.1889953613281,
      "loss": 0.7448,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.011524945497512817,
      "rewards/margins": 0.1953527331352234,
      "rewards/rejected": -0.18382777273654938,
      "step": 6908
    },
    {
      "epoch": 0.9040974891633271,
      "grad_norm": 14.070183753967285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05686718970537186,
      "logits/rejected": 0.03020472824573517,
      "logps/chosen": -602.9252319335938,
      "logps/rejected": -517.5327758789062,
      "loss": 0.9594,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4495752155780792,
      "rewards/margins": -0.04072684049606323,
      "rewards/rejected": -0.4088484048843384,
      "step": 6909
    },
    {
      "epoch": 0.9042283471006788,
      "grad_norm": 8.884828567504883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28811150789260864,
      "logits/rejected": 0.6642885208129883,
      "logps/chosen": -450.888671875,
      "logps/rejected": -587.2566528320312,
      "loss": 0.7629,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.6625850796699524,
      "rewards/margins": 0.7339751124382019,
      "rewards/rejected": -0.07138989865779877,
      "step": 6910
    },
    {
      "epoch": 0.9043592050380306,
      "grad_norm": 6.511087417602539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1827009618282318,
      "logits/rejected": -0.0020222626626491547,
      "logps/chosen": -351.9766845703125,
      "logps/rejected": -445.02685546875,
      "loss": 0.4076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.49712350964546204,
      "rewards/margins": 1.2965888977050781,
      "rewards/rejected": -0.7994652986526489,
      "step": 6911
    },
    {
      "epoch": 0.9044900629753824,
      "grad_norm": 8.819096565246582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2184498906135559,
      "logits/rejected": -0.06419438123703003,
      "logps/chosen": -555.1651000976562,
      "logps/rejected": -514.9266357421875,
      "loss": 0.5982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3160024881362915,
      "rewards/margins": 0.7591912746429443,
      "rewards/rejected": -0.4431888461112976,
      "step": 6912
    },
    {
      "epoch": 0.9046209209127342,
      "grad_norm": 11.168663024902344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1416449248790741,
      "logits/rejected": 0.038768839091062546,
      "logps/chosen": -425.96893310546875,
      "logps/rejected": -444.1855773925781,
      "loss": 1.0456,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002751380205154419,
      "rewards/margins": 0.10044091939926147,
      "rewards/rejected": -0.10319235920906067,
      "step": 6913
    },
    {
      "epoch": 0.9047517788500858,
      "grad_norm": 9.264863967895508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1424281895160675,
      "logits/rejected": 0.02114357426762581,
      "logps/chosen": -768.6368408203125,
      "logps/rejected": -449.902587890625,
      "loss": 0.5163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10326270759105682,
      "rewards/margins": 0.8105273842811584,
      "rewards/rejected": -0.7072646617889404,
      "step": 6914
    },
    {
      "epoch": 0.9048826367874376,
      "grad_norm": 5.1497321128845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06012126803398132,
      "logits/rejected": 0.0680989921092987,
      "logps/chosen": -343.0972595214844,
      "logps/rejected": -305.7903747558594,
      "loss": 0.3778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8773267865180969,
      "rewards/margins": 1.3631834983825684,
      "rewards/rejected": -0.4858565032482147,
      "step": 6915
    },
    {
      "epoch": 0.9050134947247894,
      "grad_norm": 6.587031364440918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1456434726715088,
      "logits/rejected": 0.29748445749282837,
      "logps/chosen": -446.7286682128906,
      "logps/rejected": -555.7646484375,
      "loss": 0.6774,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18452581763267517,
      "rewards/margins": 0.9475849866867065,
      "rewards/rejected": -0.7630590796470642,
      "step": 6916
    },
    {
      "epoch": 0.9051443526621412,
      "grad_norm": 7.822843074798584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.051281340420246124,
      "logits/rejected": -0.10106095671653748,
      "logps/chosen": -463.52044677734375,
      "logps/rejected": -372.9539794921875,
      "loss": 0.5826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5597657561302185,
      "rewards/margins": 0.9611181616783142,
      "rewards/rejected": -0.4013523459434509,
      "step": 6917
    },
    {
      "epoch": 0.9052752105994929,
      "grad_norm": 12.212066650390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.541522741317749,
      "logits/rejected": -0.44542229175567627,
      "logps/chosen": -297.37701416015625,
      "logps/rejected": -404.5669250488281,
      "loss": 0.9971,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27653902769088745,
      "rewards/margins": 0.23154909908771515,
      "rewards/rejected": -0.5080881714820862,
      "step": 6918
    },
    {
      "epoch": 0.9054060685368447,
      "grad_norm": 5.157827854156494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05603909492492676,
      "logits/rejected": -0.04222370684146881,
      "logps/chosen": -492.6400146484375,
      "logps/rejected": -233.7403106689453,
      "loss": 0.2739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3048950135707855,
      "rewards/margins": 1.9801702499389648,
      "rewards/rejected": -1.6752753257751465,
      "step": 6919
    },
    {
      "epoch": 0.9055369264741965,
      "grad_norm": 11.020132064819336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21982061862945557,
      "logits/rejected": -0.03154052048921585,
      "logps/chosen": -552.8516845703125,
      "logps/rejected": -479.5027160644531,
      "loss": 0.8367,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24004194140434265,
      "rewards/margins": -0.04660242795944214,
      "rewards/rejected": -0.19343946874141693,
      "step": 6920
    },
    {
      "epoch": 0.9056677844115482,
      "grad_norm": 4.299160480499268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20540504157543182,
      "logits/rejected": 0.19692590832710266,
      "logps/chosen": -406.876708984375,
      "logps/rejected": -469.33636474609375,
      "loss": 0.2574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24500925838947296,
      "rewards/margins": 1.8325328826904297,
      "rewards/rejected": -1.5875236988067627,
      "step": 6921
    },
    {
      "epoch": 0.9057986423489,
      "grad_norm": 7.79763126373291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17440293729305267,
      "logits/rejected": 0.01917170360684395,
      "logps/chosen": -525.6943969726562,
      "logps/rejected": -396.3572998046875,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6564897894859314,
      "rewards/margins": 1.8366367816925049,
      "rewards/rejected": -1.1801470518112183,
      "step": 6922
    },
    {
      "epoch": 0.9059295002862517,
      "grad_norm": 12.605154037475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3047039806842804,
      "logits/rejected": -0.3193214237689972,
      "logps/chosen": -383.4632263183594,
      "logps/rejected": -416.46697998046875,
      "loss": 0.5499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4289127290248871,
      "rewards/margins": 1.095710039138794,
      "rewards/rejected": -1.5246227979660034,
      "step": 6923
    },
    {
      "epoch": 0.9060603582236035,
      "grad_norm": 10.912178993225098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11821827292442322,
      "logits/rejected": -0.029779909178614616,
      "logps/chosen": -468.5935363769531,
      "logps/rejected": -328.5635070800781,
      "loss": 0.8309,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0904194563627243,
      "rewards/margins": 0.9509702324867249,
      "rewards/rejected": -0.8605509400367737,
      "step": 6924
    },
    {
      "epoch": 0.9061912161609552,
      "grad_norm": 6.505436420440674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15660852193832397,
      "logits/rejected": 0.3109578490257263,
      "logps/chosen": -735.4151000976562,
      "logps/rejected": -498.337890625,
      "loss": 0.4894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7602910995483398,
      "rewards/margins": 0.6917781829833984,
      "rewards/rejected": 0.0685129389166832,
      "step": 6925
    },
    {
      "epoch": 0.906322074098307,
      "grad_norm": 7.916569232940674,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0828973799943924,
      "logits/rejected": -0.14772237837314606,
      "logps/chosen": -608.353271484375,
      "logps/rejected": -554.6419677734375,
      "loss": 0.5827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4500349164009094,
      "rewards/margins": 0.7287313938140869,
      "rewards/rejected": -0.2786964774131775,
      "step": 6926
    },
    {
      "epoch": 0.9064529320356588,
      "grad_norm": 5.010420799255371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22730335593223572,
      "logits/rejected": -0.11948567628860474,
      "logps/chosen": -390.219482421875,
      "logps/rejected": -369.0864562988281,
      "loss": 0.4162,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7788323760032654,
      "rewards/margins": 1.057141661643982,
      "rewards/rejected": -0.27830925583839417,
      "step": 6927
    },
    {
      "epoch": 0.9065837899730106,
      "grad_norm": 9.22495174407959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10832306742668152,
      "logits/rejected": 0.0706985592842102,
      "logps/chosen": -412.337646484375,
      "logps/rejected": -332.938720703125,
      "loss": 0.4553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23503269255161285,
      "rewards/margins": 0.8094654679298401,
      "rewards/rejected": -1.044498085975647,
      "step": 6928
    },
    {
      "epoch": 0.9067146479103623,
      "grad_norm": 5.263415336608887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20756427943706512,
      "logits/rejected": 0.1454426348209381,
      "logps/chosen": -384.98077392578125,
      "logps/rejected": -393.9852294921875,
      "loss": 0.3707,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8486246466636658,
      "rewards/margins": 1.3210251331329346,
      "rewards/rejected": -0.4724004864692688,
      "step": 6929
    },
    {
      "epoch": 0.9068455058477141,
      "grad_norm": 4.2528791427612305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08166898787021637,
      "logits/rejected": 0.13381990790367126,
      "logps/chosen": -479.80926513671875,
      "logps/rejected": -472.2616882324219,
      "loss": 0.2567,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.59859299659729,
      "rewards/margins": 1.4493021965026855,
      "rewards/rejected": -0.8507091403007507,
      "step": 6930
    },
    {
      "epoch": 0.9069763637850659,
      "grad_norm": 5.894028186798096,
      "learning_rate": 5e-06,
      "logits/chosen": -0.219285786151886,
      "logits/rejected": -0.0810394361615181,
      "logps/chosen": -592.9190063476562,
      "logps/rejected": -452.8955993652344,
      "loss": 0.3476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1349048912525177,
      "rewards/margins": 1.446518898010254,
      "rewards/rejected": -1.5814237594604492,
      "step": 6931
    },
    {
      "epoch": 0.9071072217224176,
      "grad_norm": 5.128249168395996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16794271767139435,
      "logits/rejected": -0.11738873273134232,
      "logps/chosen": -292.1982116699219,
      "logps/rejected": -268.0464782714844,
      "loss": 0.3988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4927810728549957,
      "rewards/margins": 1.3891239166259766,
      "rewards/rejected": -0.8963428735733032,
      "step": 6932
    },
    {
      "epoch": 0.9072380796597693,
      "grad_norm": 8.716167449951172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2865939736366272,
      "logits/rejected": -0.39605867862701416,
      "logps/chosen": -282.4171447753906,
      "logps/rejected": -423.3086853027344,
      "loss": 0.6516,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4421218931674957,
      "rewards/margins": 0.588387131690979,
      "rewards/rejected": -1.0305089950561523,
      "step": 6933
    },
    {
      "epoch": 0.9073689375971211,
      "grad_norm": 9.482239723205566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11172811686992645,
      "logits/rejected": 0.13711808621883392,
      "logps/chosen": -538.2275390625,
      "logps/rejected": -460.22662353515625,
      "loss": 0.6087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10621411353349686,
      "rewards/margins": 0.7512245178222656,
      "rewards/rejected": -0.8574386835098267,
      "step": 6934
    },
    {
      "epoch": 0.9074997955344729,
      "grad_norm": 8.85830307006836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08436383306980133,
      "logits/rejected": 0.2223089337348938,
      "logps/chosen": -624.4190673828125,
      "logps/rejected": -448.6192321777344,
      "loss": 0.7147,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.9462699890136719,
      "rewards/margins": 1.321805477142334,
      "rewards/rejected": -0.3755355775356293,
      "step": 6935
    },
    {
      "epoch": 0.9076306534718247,
      "grad_norm": 9.709619522094727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15142300724983215,
      "logits/rejected": 0.246337890625,
      "logps/chosen": -412.31573486328125,
      "logps/rejected": -410.0918884277344,
      "loss": 0.9255,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.43411749601364136,
      "rewards/margins": -0.03308311104774475,
      "rewards/rejected": 0.4672006666660309,
      "step": 6936
    },
    {
      "epoch": 0.9077615114091764,
      "grad_norm": 6.481518745422363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11179216206073761,
      "logits/rejected": 0.041217878460884094,
      "logps/chosen": -633.6771850585938,
      "logps/rejected": -718.7164306640625,
      "loss": 0.3031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09463921189308167,
      "rewards/margins": 1.1611202955245972,
      "rewards/rejected": -1.2557594776153564,
      "step": 6937
    },
    {
      "epoch": 0.9078923693465282,
      "grad_norm": 6.722721576690674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04732766002416611,
      "logits/rejected": -0.163735032081604,
      "logps/chosen": -311.4852600097656,
      "logps/rejected": -442.8880310058594,
      "loss": 0.5426,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0428987592458725,
      "rewards/margins": 0.7269687652587891,
      "rewards/rejected": -0.7698675394058228,
      "step": 6938
    },
    {
      "epoch": 0.90802322728388,
      "grad_norm": 9.969012260437012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05887458845973015,
      "logits/rejected": -0.06700205057859421,
      "logps/chosen": -431.8854675292969,
      "logps/rejected": -494.3947448730469,
      "loss": 0.8661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5686756372451782,
      "rewards/margins": 0.09666559845209122,
      "rewards/rejected": 0.47201007604599,
      "step": 6939
    },
    {
      "epoch": 0.9081540852212318,
      "grad_norm": 6.741700172424316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28881698846817017,
      "logits/rejected": 0.3039155602455139,
      "logps/chosen": -432.0645751953125,
      "logps/rejected": -439.0072021484375,
      "loss": 0.5953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3120986819267273,
      "rewards/margins": 1.0503506660461426,
      "rewards/rejected": -0.7382520437240601,
      "step": 6940
    },
    {
      "epoch": 0.9082849431585834,
      "grad_norm": 4.060944080352783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37484902143478394,
      "logits/rejected": 0.18461358547210693,
      "logps/chosen": -425.49603271484375,
      "logps/rejected": -332.36419677734375,
      "loss": 0.3607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5292986035346985,
      "rewards/margins": 1.7788736820220947,
      "rewards/rejected": -1.2495750188827515,
      "step": 6941
    },
    {
      "epoch": 0.9084158010959352,
      "grad_norm": 9.91075611114502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09041738510131836,
      "logits/rejected": -0.26359087228775024,
      "logps/chosen": -579.7152709960938,
      "logps/rejected": -541.7108764648438,
      "loss": 0.6374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0231277197599411,
      "rewards/margins": 0.578300416469574,
      "rewards/rejected": -0.6014281511306763,
      "step": 6942
    },
    {
      "epoch": 0.908546659033287,
      "grad_norm": 4.5250396728515625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3350803554058075,
      "logits/rejected": -0.4038081169128418,
      "logps/chosen": -487.81341552734375,
      "logps/rejected": -528.9441528320312,
      "loss": 0.345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32910650968551636,
      "rewards/margins": 1.1610134840011597,
      "rewards/rejected": -0.8319069147109985,
      "step": 6943
    },
    {
      "epoch": 0.9086775169706387,
      "grad_norm": 5.131070137023926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14056503772735596,
      "logits/rejected": 0.0030167922377586365,
      "logps/chosen": -436.92376708984375,
      "logps/rejected": -444.7320556640625,
      "loss": 0.3063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5371967554092407,
      "rewards/margins": 1.3574814796447754,
      "rewards/rejected": -0.8202846646308899,
      "step": 6944
    },
    {
      "epoch": 0.9088083749079905,
      "grad_norm": 8.958765029907227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12342314422130585,
      "logits/rejected": 0.12352856993675232,
      "logps/chosen": -295.0980224609375,
      "logps/rejected": -327.2019348144531,
      "loss": 0.7507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5261960625648499,
      "rewards/margins": 0.24627205729484558,
      "rewards/rejected": -0.7724682092666626,
      "step": 6945
    },
    {
      "epoch": 0.9089392328453423,
      "grad_norm": 5.887529373168945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1902090162038803,
      "logits/rejected": 0.14966264367103577,
      "logps/chosen": -432.77978515625,
      "logps/rejected": -417.90997314453125,
      "loss": 0.5915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3483324646949768,
      "rewards/margins": 0.4851396679878235,
      "rewards/rejected": -0.13680729269981384,
      "step": 6946
    },
    {
      "epoch": 0.9090700907826941,
      "grad_norm": 7.13439416885376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05902426689863205,
      "logits/rejected": -0.21341316401958466,
      "logps/chosen": -358.88232421875,
      "logps/rejected": -407.7980041503906,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22492066025733948,
      "rewards/margins": 0.6873236298561096,
      "rewards/rejected": -0.46240293979644775,
      "step": 6947
    },
    {
      "epoch": 0.9092009487200458,
      "grad_norm": 7.998818397521973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1903514713048935,
      "logits/rejected": -0.16521799564361572,
      "logps/chosen": -496.8642578125,
      "logps/rejected": -326.71209716796875,
      "loss": 0.5968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.027983814477920532,
      "rewards/margins": 0.44973224401474,
      "rewards/rejected": -0.4777160882949829,
      "step": 6948
    },
    {
      "epoch": 0.9093318066573975,
      "grad_norm": 9.021222114562988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11680891364812851,
      "logits/rejected": 0.2583005130290985,
      "logps/chosen": -485.26837158203125,
      "logps/rejected": -517.7788696289062,
      "loss": 0.6432,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29691508412361145,
      "rewards/margins": 0.5978793501853943,
      "rewards/rejected": -0.8947944641113281,
      "step": 6949
    },
    {
      "epoch": 0.9094626645947493,
      "grad_norm": 6.446780204772949,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14637039601802826,
      "logits/rejected": -0.36931008100509644,
      "logps/chosen": -415.54205322265625,
      "logps/rejected": -389.40887451171875,
      "loss": 0.438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09657593071460724,
      "rewards/margins": 0.8278368711471558,
      "rewards/rejected": -0.924412727355957,
      "step": 6950
    },
    {
      "epoch": 0.9095935225321011,
      "grad_norm": 9.920703887939453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15406078100204468,
      "logits/rejected": -0.23228198289871216,
      "logps/chosen": -572.5260620117188,
      "logps/rejected": -426.91607666015625,
      "loss": 0.8725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1863315999507904,
      "rewards/margins": 0.6912513375282288,
      "rewards/rejected": -0.8775829076766968,
      "step": 6951
    },
    {
      "epoch": 0.9097243804694528,
      "grad_norm": 9.955192565917969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0014717206358909607,
      "logits/rejected": 0.08816869556903839,
      "logps/chosen": -445.8388671875,
      "logps/rejected": -439.2248229980469,
      "loss": 0.6466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4483003318309784,
      "rewards/margins": 0.13299962878227234,
      "rewards/rejected": -0.5813000202178955,
      "step": 6952
    },
    {
      "epoch": 0.9098552384068046,
      "grad_norm": 7.684419631958008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1052885353565216,
      "logits/rejected": 0.06956750899553299,
      "logps/chosen": -491.96148681640625,
      "logps/rejected": -421.1643981933594,
      "loss": 0.5344,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03390425443649292,
      "rewards/margins": 0.9314556121826172,
      "rewards/rejected": -0.9653599858283997,
      "step": 6953
    },
    {
      "epoch": 0.9099860963441564,
      "grad_norm": 6.116199493408203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07084521651268005,
      "logits/rejected": -0.14775946736335754,
      "logps/chosen": -456.1900634765625,
      "logps/rejected": -681.653076171875,
      "loss": 0.2691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23119807243347168,
      "rewards/margins": 1.861297845840454,
      "rewards/rejected": -1.630099892616272,
      "step": 6954
    },
    {
      "epoch": 0.9101169542815082,
      "grad_norm": 5.775725841522217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17861653864383698,
      "logits/rejected": -0.15482543408870697,
      "logps/chosen": -430.9736328125,
      "logps/rejected": -525.7339477539062,
      "loss": 0.4338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1626676768064499,
      "rewards/margins": 1.309258222579956,
      "rewards/rejected": -1.471925973892212,
      "step": 6955
    },
    {
      "epoch": 0.9102478122188599,
      "grad_norm": 9.053348541259766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04106256738305092,
      "logits/rejected": -0.07305016368627548,
      "logps/chosen": -636.6292724609375,
      "logps/rejected": -562.7451171875,
      "loss": 0.5696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29854637384414673,
      "rewards/margins": 1.226844310760498,
      "rewards/rejected": -1.525390625,
      "step": 6956
    },
    {
      "epoch": 0.9103786701562117,
      "grad_norm": 5.882050037384033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17200978100299835,
      "logits/rejected": 0.08124924451112747,
      "logps/chosen": -369.2011413574219,
      "logps/rejected": -379.99530029296875,
      "loss": 0.3777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.008519381284713745,
      "rewards/margins": 1.2020561695098877,
      "rewards/rejected": -1.2105755805969238,
      "step": 6957
    },
    {
      "epoch": 0.9105095280935634,
      "grad_norm": 6.785224437713623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03773157671093941,
      "logits/rejected": 0.0019267583265900612,
      "logps/chosen": -452.6181945800781,
      "logps/rejected": -328.6466369628906,
      "loss": 0.3873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.428650826215744,
      "rewards/margins": 1.1984038352966309,
      "rewards/rejected": -0.769753098487854,
      "step": 6958
    },
    {
      "epoch": 0.9106403860309152,
      "grad_norm": 15.99449634552002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043286148458719254,
      "logits/rejected": -0.07993210852146149,
      "logps/chosen": -753.7754516601562,
      "logps/rejected": -681.110595703125,
      "loss": 0.9362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6116439700126648,
      "rewards/margins": 0.15081408619880676,
      "rewards/rejected": -0.7624580264091492,
      "step": 6959
    },
    {
      "epoch": 0.9107712439682669,
      "grad_norm": 6.8938493728637695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008978866040706635,
      "logits/rejected": -0.0633842721581459,
      "logps/chosen": -612.1255493164062,
      "logps/rejected": -693.9672241210938,
      "loss": 0.3737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4732244610786438,
      "rewards/margins": 1.3628675937652588,
      "rewards/rejected": -0.8896430730819702,
      "step": 6960
    },
    {
      "epoch": 0.9109021019056187,
      "grad_norm": 8.085563659667969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19211485981941223,
      "logits/rejected": 0.13628512620925903,
      "logps/chosen": -286.3382263183594,
      "logps/rejected": -279.1763000488281,
      "loss": 0.741,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0756072998046875,
      "rewards/margins": 0.4874531626701355,
      "rewards/rejected": -0.5630604028701782,
      "step": 6961
    },
    {
      "epoch": 0.9110329598429705,
      "grad_norm": 7.149238586425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23121005296707153,
      "logits/rejected": -0.07983013987541199,
      "logps/chosen": -649.0414428710938,
      "logps/rejected": -525.6917114257812,
      "loss": 0.6236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16088426113128662,
      "rewards/margins": 0.7830383777618408,
      "rewards/rejected": -0.9439226388931274,
      "step": 6962
    },
    {
      "epoch": 0.9111638177803222,
      "grad_norm": 10.509098052978516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13385213911533356,
      "logits/rejected": 0.16047796607017517,
      "logps/chosen": -489.2490539550781,
      "logps/rejected": -457.03863525390625,
      "loss": 1.0397,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.35197964310646057,
      "rewards/margins": -0.13944664597511292,
      "rewards/rejected": -0.21253298223018646,
      "step": 6963
    },
    {
      "epoch": 0.911294675717674,
      "grad_norm": 7.9720540046691895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2687839865684509,
      "logits/rejected": -0.35098743438720703,
      "logps/chosen": -321.633544921875,
      "logps/rejected": -374.15350341796875,
      "loss": 0.6598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31319087743759155,
      "rewards/margins": 0.7332072257995605,
      "rewards/rejected": -1.0463980436325073,
      "step": 6964
    },
    {
      "epoch": 0.9114255336550258,
      "grad_norm": 13.14410400390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018262721598148346,
      "logits/rejected": -0.1488826870918274,
      "logps/chosen": -406.342529296875,
      "logps/rejected": -337.34613037109375,
      "loss": 0.8356,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9065448641777039,
      "rewards/margins": 0.06181010603904724,
      "rewards/rejected": -0.9683549404144287,
      "step": 6965
    },
    {
      "epoch": 0.9115563915923776,
      "grad_norm": 9.820015907287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14934951066970825,
      "logits/rejected": -0.035813022404909134,
      "logps/chosen": -316.35968017578125,
      "logps/rejected": -362.4203186035156,
      "loss": 0.7879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07511177659034729,
      "rewards/margins": 0.046941936016082764,
      "rewards/rejected": 0.028169825673103333,
      "step": 6966
    },
    {
      "epoch": 0.9116872495297292,
      "grad_norm": 12.338461875915527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31021350622177124,
      "logits/rejected": -0.03286297619342804,
      "logps/chosen": -394.7286682128906,
      "logps/rejected": -433.2750244140625,
      "loss": 0.9879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6790599822998047,
      "rewards/margins": 0.07862171530723572,
      "rewards/rejected": -0.757681667804718,
      "step": 6967
    },
    {
      "epoch": 0.911818107467081,
      "grad_norm": 27.996854782104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0648287981748581,
      "logits/rejected": -0.06920170038938522,
      "logps/chosen": -563.3435668945312,
      "logps/rejected": -436.646484375,
      "loss": 0.7645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.32835853099823,
      "rewards/margins": 0.6749321222305298,
      "rewards/rejected": -2.0032904148101807,
      "step": 6968
    },
    {
      "epoch": 0.9119489654044328,
      "grad_norm": 5.95816707611084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36218902468681335,
      "logits/rejected": 0.16469702124595642,
      "logps/chosen": -477.87750244140625,
      "logps/rejected": -485.6544189453125,
      "loss": 0.3801,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3152754306793213,
      "rewards/margins": 1.2391002178192139,
      "rewards/rejected": -0.923824667930603,
      "step": 6969
    },
    {
      "epoch": 0.9120798233417846,
      "grad_norm": 6.520346641540527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0744202509522438,
      "logits/rejected": -0.00022837892174720764,
      "logps/chosen": -390.2409362792969,
      "logps/rejected": -501.899658203125,
      "loss": 0.5107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.026030391454696655,
      "rewards/margins": 0.5686649084091187,
      "rewards/rejected": -0.5946952700614929,
      "step": 6970
    },
    {
      "epoch": 0.9122106812791363,
      "grad_norm": 7.480312347412109,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13469603657722473,
      "logits/rejected": -0.2636066973209381,
      "logps/chosen": -436.57977294921875,
      "logps/rejected": -430.3969421386719,
      "loss": 0.4663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06898576021194458,
      "rewards/margins": 1.1188724040985107,
      "rewards/rejected": -1.049886703491211,
      "step": 6971
    },
    {
      "epoch": 0.9123415392164881,
      "grad_norm": 5.192076683044434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05285480245947838,
      "logits/rejected": 0.04489489272236824,
      "logps/chosen": -356.51300048828125,
      "logps/rejected": -378.6918640136719,
      "loss": 0.3787,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1479770541191101,
      "rewards/margins": 1.4907844066619873,
      "rewards/rejected": -1.342807412147522,
      "step": 6972
    },
    {
      "epoch": 0.9124723971538399,
      "grad_norm": 6.820907115936279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09558939933776855,
      "logits/rejected": -0.24161791801452637,
      "logps/chosen": -391.0024108886719,
      "logps/rejected": -370.4639892578125,
      "loss": 0.4454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.832636296749115,
      "rewards/margins": 1.2893974781036377,
      "rewards/rejected": -2.1220335960388184,
      "step": 6973
    },
    {
      "epoch": 0.9126032550911917,
      "grad_norm": 6.498626232147217,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21452495455741882,
      "logits/rejected": 0.1041254848241806,
      "logps/chosen": -374.7015380859375,
      "logps/rejected": -484.391845703125,
      "loss": 0.489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33501529693603516,
      "rewards/margins": 0.7595003247261047,
      "rewards/rejected": -1.0945156812667847,
      "step": 6974
    },
    {
      "epoch": 0.9127341130285433,
      "grad_norm": 6.93205451965332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1964675933122635,
      "logits/rejected": -0.08474098891019821,
      "logps/chosen": -292.1028137207031,
      "logps/rejected": -320.32830810546875,
      "loss": 0.8454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1791410446166992,
      "rewards/margins": 0.21219217777252197,
      "rewards/rejected": -1.3913332223892212,
      "step": 6975
    },
    {
      "epoch": 0.9128649709658951,
      "grad_norm": 7.992082595825195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11056870967149734,
      "logits/rejected": -0.21048122644424438,
      "logps/chosen": -742.5234985351562,
      "logps/rejected": -433.54443359375,
      "loss": 0.4578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19611530005931854,
      "rewards/margins": 0.8394237756729126,
      "rewards/rejected": -0.6433085203170776,
      "step": 6976
    },
    {
      "epoch": 0.9129958289032469,
      "grad_norm": 7.667474269866943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2533493638038635,
      "logits/rejected": -0.31083250045776367,
      "logps/chosen": -443.0672302246094,
      "logps/rejected": -368.33123779296875,
      "loss": 0.5473,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14760437607765198,
      "rewards/margins": 0.856971800327301,
      "rewards/rejected": -1.0045759677886963,
      "step": 6977
    },
    {
      "epoch": 0.9131266868405987,
      "grad_norm": 4.752242565155029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08278708159923553,
      "logits/rejected": 0.07560700923204422,
      "logps/chosen": -578.5138549804688,
      "logps/rejected": -470.4629821777344,
      "loss": 0.2765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16413480043411255,
      "rewards/margins": 1.2447500228881836,
      "rewards/rejected": -1.4088847637176514,
      "step": 6978
    },
    {
      "epoch": 0.9132575447779504,
      "grad_norm": 5.3014655113220215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06736928224563599,
      "logits/rejected": -0.16149777173995972,
      "logps/chosen": -523.2361450195312,
      "logps/rejected": -360.8047180175781,
      "loss": 0.3355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.859166145324707,
      "rewards/margins": 1.4339059591293335,
      "rewards/rejected": -0.5747398138046265,
      "step": 6979
    },
    {
      "epoch": 0.9133884027153022,
      "grad_norm": 8.980700492858887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013552553951740265,
      "logits/rejected": 0.07408233731985092,
      "logps/chosen": -368.0687255859375,
      "logps/rejected": -387.37335205078125,
      "loss": 0.7342,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.21128042042255402,
      "rewards/margins": 0.1690891534090042,
      "rewards/rejected": -0.38036954402923584,
      "step": 6980
    },
    {
      "epoch": 0.913519260652654,
      "grad_norm": 9.00540828704834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10869982838630676,
      "logits/rejected": -0.3773941099643707,
      "logps/chosen": -446.9008483886719,
      "logps/rejected": -420.0396423339844,
      "loss": 0.5923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3396579623222351,
      "rewards/margins": 0.4078729748725891,
      "rewards/rejected": -0.7475309371948242,
      "step": 6981
    },
    {
      "epoch": 0.9136501185900058,
      "grad_norm": 5.112074851989746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12481538206338882,
      "logits/rejected": -0.2754703164100647,
      "logps/chosen": -450.24365234375,
      "logps/rejected": -420.7392883300781,
      "loss": 0.3979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10533884167671204,
      "rewards/margins": 1.1148300170898438,
      "rewards/rejected": -1.220168948173523,
      "step": 6982
    },
    {
      "epoch": 0.9137809765273575,
      "grad_norm": 8.826776504516602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2803962826728821,
      "logits/rejected": -0.21571722626686096,
      "logps/chosen": -393.8816223144531,
      "logps/rejected": -360.51739501953125,
      "loss": 0.6669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08583133667707443,
      "rewards/margins": 0.39862796664237976,
      "rewards/rejected": -0.4844593405723572,
      "step": 6983
    },
    {
      "epoch": 0.9139118344647092,
      "grad_norm": 6.2637248039245605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07748755812644958,
      "logits/rejected": -0.022809229791164398,
      "logps/chosen": -435.6343078613281,
      "logps/rejected": -360.24835205078125,
      "loss": 0.4706,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.37153610587120056,
      "rewards/margins": 0.7084372043609619,
      "rewards/rejected": -0.33690112829208374,
      "step": 6984
    },
    {
      "epoch": 0.914042692402061,
      "grad_norm": 7.529125213623047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07481959462165833,
      "logits/rejected": -0.2638050615787506,
      "logps/chosen": -473.6709289550781,
      "logps/rejected": -482.3785705566406,
      "loss": 0.4931,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06154478341341019,
      "rewards/margins": 0.8391982913017273,
      "rewards/rejected": -0.7776535153388977,
      "step": 6985
    },
    {
      "epoch": 0.9141735503394127,
      "grad_norm": 11.75158977508545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10596562922000885,
      "logits/rejected": 0.13016173243522644,
      "logps/chosen": -546.6150512695312,
      "logps/rejected": -723.9052734375,
      "loss": 0.5725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43932896852493286,
      "rewards/margins": 0.628223717212677,
      "rewards/rejected": -0.1888946294784546,
      "step": 6986
    },
    {
      "epoch": 0.9143044082767645,
      "grad_norm": 4.647514820098877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11344335973262787,
      "logits/rejected": -0.15046902000904083,
      "logps/chosen": -341.8716125488281,
      "logps/rejected": -324.94561767578125,
      "loss": 0.3365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01605227217078209,
      "rewards/margins": 1.1009304523468018,
      "rewards/rejected": -1.1169825792312622,
      "step": 6987
    },
    {
      "epoch": 0.9144352662141163,
      "grad_norm": 8.80482292175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19351665675640106,
      "logits/rejected": -0.33990004658699036,
      "logps/chosen": -542.102783203125,
      "logps/rejected": -463.2525634765625,
      "loss": 0.623,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.9254271388053894,
      "rewards/margins": 0.28288573026657104,
      "rewards/rejected": 0.6425413489341736,
      "step": 6988
    },
    {
      "epoch": 0.9145661241514681,
      "grad_norm": 6.186544895172119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008418255485594273,
      "logits/rejected": 0.02972208335995674,
      "logps/chosen": -462.31146240234375,
      "logps/rejected": -451.956298828125,
      "loss": 0.4976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3242984712123871,
      "rewards/margins": 0.6897116303443909,
      "rewards/rejected": -1.0140100717544556,
      "step": 6989
    },
    {
      "epoch": 0.9146969820888198,
      "grad_norm": 5.796890735626221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2039279043674469,
      "logits/rejected": 0.004528101533651352,
      "logps/chosen": -410.37530517578125,
      "logps/rejected": -325.94659423828125,
      "loss": 0.3446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4579117000102997,
      "rewards/margins": 1.429948091506958,
      "rewards/rejected": -0.9720364809036255,
      "step": 6990
    },
    {
      "epoch": 0.9148278400261716,
      "grad_norm": 9.780858039855957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13170982897281647,
      "logits/rejected": -0.2511950433254242,
      "logps/chosen": -502.5692443847656,
      "logps/rejected": -378.5364685058594,
      "loss": 0.6488,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38265755772590637,
      "rewards/margins": 0.7594237923622131,
      "rewards/rejected": -1.1420812606811523,
      "step": 6991
    },
    {
      "epoch": 0.9149586979635234,
      "grad_norm": 5.3789544105529785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21865074336528778,
      "logits/rejected": -0.0768556147813797,
      "logps/chosen": -540.229736328125,
      "logps/rejected": -391.2780456542969,
      "loss": 0.3824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15988942980766296,
      "rewards/margins": 0.903650164604187,
      "rewards/rejected": -1.0635395050048828,
      "step": 6992
    },
    {
      "epoch": 0.9150895559008752,
      "grad_norm": 5.359701156616211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07023507356643677,
      "logits/rejected": -0.24145884811878204,
      "logps/chosen": -298.36187744140625,
      "logps/rejected": -310.1810607910156,
      "loss": 0.3902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07019540667533875,
      "rewards/margins": 1.1236851215362549,
      "rewards/rejected": -1.193880558013916,
      "step": 6993
    },
    {
      "epoch": 0.9152204138382268,
      "grad_norm": 6.007167816162109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19615767896175385,
      "logits/rejected": -0.171916201710701,
      "logps/chosen": -522.8271484375,
      "logps/rejected": -506.3585205078125,
      "loss": 0.3802,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29933398962020874,
      "rewards/margins": 0.9884384870529175,
      "rewards/rejected": -0.6891044974327087,
      "step": 6994
    },
    {
      "epoch": 0.9153512717755786,
      "grad_norm": 8.601293563842773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22135376930236816,
      "logits/rejected": -0.17702192068099976,
      "logps/chosen": -532.232421875,
      "logps/rejected": -392.9172058105469,
      "loss": 0.5219,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21588820219039917,
      "rewards/margins": 1.1775341033935547,
      "rewards/rejected": -1.3934223651885986,
      "step": 6995
    },
    {
      "epoch": 0.9154821297129304,
      "grad_norm": 6.958297252655029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03068457916378975,
      "logits/rejected": 0.04468909278512001,
      "logps/chosen": -408.8287353515625,
      "logps/rejected": -421.61553955078125,
      "loss": 0.5752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04617729410529137,
      "rewards/margins": 1.0281778573989868,
      "rewards/rejected": -0.9820006489753723,
      "step": 6996
    },
    {
      "epoch": 0.9156129876502822,
      "grad_norm": 6.146101951599121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023641403764486313,
      "logits/rejected": -0.12107092142105103,
      "logps/chosen": -416.2486572265625,
      "logps/rejected": -431.8576354980469,
      "loss": 0.4961,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8198752999305725,
      "rewards/margins": 0.7996131777763367,
      "rewards/rejected": -1.6194887161254883,
      "step": 6997
    },
    {
      "epoch": 0.9157438455876339,
      "grad_norm": 5.2470269203186035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1149401068687439,
      "logits/rejected": 0.08503754436969757,
      "logps/chosen": -411.4976501464844,
      "logps/rejected": -439.2388916015625,
      "loss": 0.3711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.460754930973053,
      "rewards/margins": 1.2185304164886475,
      "rewards/rejected": -0.7577753663063049,
      "step": 6998
    },
    {
      "epoch": 0.9158747035249857,
      "grad_norm": 7.981894493103027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025071360170841217,
      "logits/rejected": -0.10503743588924408,
      "logps/chosen": -366.7266845703125,
      "logps/rejected": -413.11407470703125,
      "loss": 0.5753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2077709138393402,
      "rewards/margins": 1.0215176343917847,
      "rewards/rejected": -0.8137465715408325,
      "step": 6999
    },
    {
      "epoch": 0.9160055614623375,
      "grad_norm": 9.848883628845215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019422966986894608,
      "logits/rejected": 0.016087742522358894,
      "logps/chosen": -455.53692626953125,
      "logps/rejected": -462.68804931640625,
      "loss": 0.6489,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14494480192661285,
      "rewards/margins": 1.5583010911941528,
      "rewards/rejected": -1.7032458782196045,
      "step": 7000
    },
    {
      "epoch": 0.9161364193996893,
      "grad_norm": 12.845462799072266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1475432813167572,
      "logits/rejected": 0.08846692740917206,
      "logps/chosen": -585.1605224609375,
      "logps/rejected": -750.171875,
      "loss": 0.5689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7587242722511292,
      "rewards/margins": 0.7026779055595398,
      "rewards/rejected": -1.461402177810669,
      "step": 7001
    },
    {
      "epoch": 0.9162672773370409,
      "grad_norm": 12.92348575592041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09912087768316269,
      "logits/rejected": 0.15043824911117554,
      "logps/chosen": -433.0792236328125,
      "logps/rejected": -399.29736328125,
      "loss": 0.6492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3306468725204468,
      "rewards/margins": 0.39941465854644775,
      "rewards/rejected": -1.7300615310668945,
      "step": 7002
    },
    {
      "epoch": 0.9163981352743927,
      "grad_norm": 11.980764389038086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023844771087169647,
      "logits/rejected": 0.20028769969940186,
      "logps/chosen": -501.579833984375,
      "logps/rejected": -389.5167541503906,
      "loss": 0.9295,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2515796422958374,
      "rewards/margins": -0.17797473073005676,
      "rewards/rejected": -1.0736050605773926,
      "step": 7003
    },
    {
      "epoch": 0.9165289932117445,
      "grad_norm": 10.003607749938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2453845739364624,
      "logits/rejected": -0.1295943409204483,
      "logps/chosen": -424.5870361328125,
      "logps/rejected": -473.7270202636719,
      "loss": 1.0362,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.46868014335632324,
      "rewards/margins": -0.16163423657417297,
      "rewards/rejected": -0.30704593658447266,
      "step": 7004
    },
    {
      "epoch": 0.9166598511490963,
      "grad_norm": 7.370810508728027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21915236115455627,
      "logits/rejected": -0.16569453477859497,
      "logps/chosen": -374.1992492675781,
      "logps/rejected": -381.25457763671875,
      "loss": 0.4129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20749147236347198,
      "rewards/margins": 1.5561697483062744,
      "rewards/rejected": -1.76366126537323,
      "step": 7005
    },
    {
      "epoch": 0.916790709086448,
      "grad_norm": 3.694474697113037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14813841879367828,
      "logits/rejected": -0.003342263400554657,
      "logps/chosen": -521.077880859375,
      "logps/rejected": -541.9793090820312,
      "loss": 0.1877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.010496135801076889,
      "rewards/margins": 1.925912857055664,
      "rewards/rejected": -1.9364089965820312,
      "step": 7006
    },
    {
      "epoch": 0.9169215670237998,
      "grad_norm": 4.759900093078613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2403990924358368,
      "logits/rejected": -0.2389187514781952,
      "logps/chosen": -504.7484436035156,
      "logps/rejected": -398.37506103515625,
      "loss": 0.3903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.38430100679397583,
      "rewards/margins": 1.2454915046691895,
      "rewards/rejected": -0.8611904382705688,
      "step": 7007
    },
    {
      "epoch": 0.9170524249611516,
      "grad_norm": 6.091616630554199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0898856520652771,
      "logits/rejected": -0.1933353841304779,
      "logps/chosen": -440.36041259765625,
      "logps/rejected": -443.3686828613281,
      "loss": 0.515,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49232596158981323,
      "rewards/margins": 1.146305799484253,
      "rewards/rejected": -1.638631820678711,
      "step": 7008
    },
    {
      "epoch": 0.9171832828985033,
      "grad_norm": 5.193303108215332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03824687749147415,
      "logits/rejected": 0.05452543497085571,
      "logps/chosen": -445.8388671875,
      "logps/rejected": -412.6288146972656,
      "loss": 0.3101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18163055181503296,
      "rewards/margins": 1.246766448020935,
      "rewards/rejected": -1.4283970594406128,
      "step": 7009
    },
    {
      "epoch": 0.917314140835855,
      "grad_norm": 8.034567832946777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06641369313001633,
      "logits/rejected": 0.05778728425502777,
      "logps/chosen": -419.5696105957031,
      "logps/rejected": -544.2333374023438,
      "loss": 0.6497,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4203981161117554,
      "rewards/margins": 0.7048141956329346,
      "rewards/rejected": -2.1252124309539795,
      "step": 7010
    },
    {
      "epoch": 0.9174449987732068,
      "grad_norm": 9.700201034545898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1763419210910797,
      "logits/rejected": -0.14440563321113586,
      "logps/chosen": -250.92453002929688,
      "logps/rejected": -263.3345947265625,
      "loss": 0.6238,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7457721829414368,
      "rewards/margins": 0.4385848939418793,
      "rewards/rejected": -1.1843571662902832,
      "step": 7011
    },
    {
      "epoch": 0.9175758567105586,
      "grad_norm": 7.444943428039551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013616565614938736,
      "logits/rejected": -0.08697118610143661,
      "logps/chosen": -660.9393310546875,
      "logps/rejected": -457.7856750488281,
      "loss": 0.6803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6651409268379211,
      "rewards/margins": 0.8139516711235046,
      "rewards/rejected": -0.1488107442855835,
      "step": 7012
    },
    {
      "epoch": 0.9177067146479103,
      "grad_norm": 9.292526245117188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05595370754599571,
      "logits/rejected": -0.2758907377719879,
      "logps/chosen": -490.6885681152344,
      "logps/rejected": -505.0065002441406,
      "loss": 0.4457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33114108443260193,
      "rewards/margins": 1.3764857053756714,
      "rewards/rejected": -1.7076267004013062,
      "step": 7013
    },
    {
      "epoch": 0.9178375725852621,
      "grad_norm": 7.527162075042725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31988441944122314,
      "logits/rejected": 0.03764317184686661,
      "logps/chosen": -529.682373046875,
      "logps/rejected": -486.05706787109375,
      "loss": 0.4281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07350308448076248,
      "rewards/margins": 1.1688666343688965,
      "rewards/rejected": -1.2423697710037231,
      "step": 7014
    },
    {
      "epoch": 0.9179684305226139,
      "grad_norm": 6.233441352844238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035930827260017395,
      "logits/rejected": -0.22981764376163483,
      "logps/chosen": -406.9881286621094,
      "logps/rejected": -472.6143798828125,
      "loss": 0.4488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45644259452819824,
      "rewards/margins": 1.3246166706085205,
      "rewards/rejected": -1.7810592651367188,
      "step": 7015
    },
    {
      "epoch": 0.9180992884599657,
      "grad_norm": 11.154698371887207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35532146692276,
      "logits/rejected": 0.011285930871963501,
      "logps/chosen": -404.1661376953125,
      "logps/rejected": -343.744384765625,
      "loss": 0.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3972448706626892,
      "rewards/margins": 0.48484304547309875,
      "rewards/rejected": -0.8820878267288208,
      "step": 7016
    },
    {
      "epoch": 0.9182301463973174,
      "grad_norm": 3.681330442428589,
      "learning_rate": 5e-06,
      "logits/chosen": -0.089271679520607,
      "logits/rejected": -0.042799897491931915,
      "logps/chosen": -502.0594177246094,
      "logps/rejected": -427.81793212890625,
      "loss": 0.2911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2649068236351013,
      "rewards/margins": 1.5391755104064941,
      "rewards/rejected": -1.2742687463760376,
      "step": 7017
    },
    {
      "epoch": 0.9183610043346692,
      "grad_norm": 7.464208602905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03428693860769272,
      "logits/rejected": 0.032778386026620865,
      "logps/chosen": -467.21484375,
      "logps/rejected": -382.3321533203125,
      "loss": 0.4975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7961771488189697,
      "rewards/margins": 1.393542766571045,
      "rewards/rejected": -0.5973655581474304,
      "step": 7018
    },
    {
      "epoch": 0.918491862272021,
      "grad_norm": 8.537023544311523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1319686323404312,
      "logits/rejected": 0.06197214126586914,
      "logps/chosen": -540.4354858398438,
      "logps/rejected": -530.6998291015625,
      "loss": 0.6343,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02532714605331421,
      "rewards/margins": 0.563872218132019,
      "rewards/rejected": -0.5385450720787048,
      "step": 7019
    },
    {
      "epoch": 0.9186227202093727,
      "grad_norm": 9.23570728302002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22264772653579712,
      "logits/rejected": 0.18369179964065552,
      "logps/chosen": -429.91119384765625,
      "logps/rejected": -423.9132385253906,
      "loss": 0.7134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1895359307527542,
      "rewards/margins": 0.5908091068267822,
      "rewards/rejected": -0.4012731909751892,
      "step": 7020
    },
    {
      "epoch": 0.9187535781467244,
      "grad_norm": 4.949112892150879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5409116744995117,
      "logits/rejected": 0.36922311782836914,
      "logps/chosen": -546.5108032226562,
      "logps/rejected": -382.71209716796875,
      "loss": 0.3495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4069961905479431,
      "rewards/margins": 1.364137053489685,
      "rewards/rejected": -0.9571407437324524,
      "step": 7021
    },
    {
      "epoch": 0.9188844360840762,
      "grad_norm": 7.9454803466796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003283656667917967,
      "logits/rejected": 0.0007953103631734848,
      "logps/chosen": -406.6536865234375,
      "logps/rejected": -388.4083557128906,
      "loss": 0.633,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14227858185768127,
      "rewards/margins": 0.9744240045547485,
      "rewards/rejected": -0.8321453928947449,
      "step": 7022
    },
    {
      "epoch": 0.919015294021428,
      "grad_norm": 2.7366864681243896,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06655877828598022,
      "logits/rejected": -0.2844061851501465,
      "logps/chosen": -500.4060363769531,
      "logps/rejected": -552.8845825195312,
      "loss": 0.1152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8038694262504578,
      "rewards/margins": 3.015746593475342,
      "rewards/rejected": -2.2118771076202393,
      "step": 7023
    },
    {
      "epoch": 0.9191461519587798,
      "grad_norm": 6.6197099685668945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08230678737163544,
      "logits/rejected": -0.06250403076410294,
      "logps/chosen": -616.4444580078125,
      "logps/rejected": -436.9090270996094,
      "loss": 0.4814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.34477499127388,
      "rewards/margins": 0.8503214120864868,
      "rewards/rejected": -0.5055464506149292,
      "step": 7024
    },
    {
      "epoch": 0.9192770098961315,
      "grad_norm": 9.310879707336426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00022961944341659546,
      "logits/rejected": -0.11099701374769211,
      "logps/chosen": -478.07318115234375,
      "logps/rejected": -360.46026611328125,
      "loss": 0.6564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32759663462638855,
      "rewards/margins": 0.34454768896102905,
      "rewards/rejected": -0.67214435338974,
      "step": 7025
    },
    {
      "epoch": 0.9194078678334833,
      "grad_norm": 5.739975452423096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02224506065249443,
      "logits/rejected": -0.10105864703655243,
      "logps/chosen": -650.2535400390625,
      "logps/rejected": -536.843994140625,
      "loss": 0.4249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09986495971679688,
      "rewards/margins": 0.9272093176841736,
      "rewards/rejected": -1.0270743370056152,
      "step": 7026
    },
    {
      "epoch": 0.9195387257708351,
      "grad_norm": 3.825590133666992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07175356149673462,
      "logits/rejected": 0.08710090816020966,
      "logps/chosen": -446.0893859863281,
      "logps/rejected": -318.0472412109375,
      "loss": 0.3179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.2932411432266235,
      "rewards/margins": 1.3399174213409424,
      "rewards/rejected": -0.04667630046606064,
      "step": 7027
    },
    {
      "epoch": 0.9196695837081869,
      "grad_norm": 9.458559036254883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12219350039958954,
      "logits/rejected": -0.043614327907562256,
      "logps/chosen": -534.8543701171875,
      "logps/rejected": -407.09332275390625,
      "loss": 0.6015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12386666238307953,
      "rewards/margins": 1.1652189493179321,
      "rewards/rejected": -1.0413522720336914,
      "step": 7028
    },
    {
      "epoch": 0.9198004416455385,
      "grad_norm": 3.691415309906006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14270555973052979,
      "logits/rejected": -0.21416878700256348,
      "logps/chosen": -499.2371826171875,
      "logps/rejected": -378.5295104980469,
      "loss": 0.253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9323005676269531,
      "rewards/margins": 2.1455273628234863,
      "rewards/rejected": -1.213226556777954,
      "step": 7029
    },
    {
      "epoch": 0.9199312995828903,
      "grad_norm": 8.387787818908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07755884528160095,
      "logits/rejected": -0.09637735038995743,
      "logps/chosen": -352.4937744140625,
      "logps/rejected": -357.59234619140625,
      "loss": 0.6366,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.49103447794914246,
      "rewards/margins": 1.1251182556152344,
      "rewards/rejected": -1.6161527633666992,
      "step": 7030
    },
    {
      "epoch": 0.9200621575202421,
      "grad_norm": 7.309444427490234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004687707871198654,
      "logits/rejected": -0.09596742689609528,
      "logps/chosen": -346.43988037109375,
      "logps/rejected": -510.53228759765625,
      "loss": 0.2964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16467304527759552,
      "rewards/margins": 1.7666435241699219,
      "rewards/rejected": -1.6019704341888428,
      "step": 7031
    },
    {
      "epoch": 0.9201930154575938,
      "grad_norm": 8.663500785827637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047869447618722916,
      "logits/rejected": -0.24638135731220245,
      "logps/chosen": -362.3376770019531,
      "logps/rejected": -409.8387451171875,
      "loss": 0.8085,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4146472215652466,
      "rewards/margins": 0.42870038747787476,
      "rewards/rejected": -0.8433476090431213,
      "step": 7032
    },
    {
      "epoch": 0.9203238733949456,
      "grad_norm": 7.132320880889893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15227818489074707,
      "logits/rejected": 0.05205197632312775,
      "logps/chosen": -446.063720703125,
      "logps/rejected": -426.71295166015625,
      "loss": 0.532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4732710123062134,
      "rewards/margins": 1.1502232551574707,
      "rewards/rejected": -0.6769521832466125,
      "step": 7033
    },
    {
      "epoch": 0.9204547313322974,
      "grad_norm": 5.446848392486572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22269852459430695,
      "logits/rejected": 0.26934951543807983,
      "logps/chosen": -380.4982604980469,
      "logps/rejected": -469.0868835449219,
      "loss": 0.4672,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07284872233867645,
      "rewards/margins": 0.664523184299469,
      "rewards/rejected": -0.7373718023300171,
      "step": 7034
    },
    {
      "epoch": 0.9205855892696492,
      "grad_norm": 6.929329872131348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0210317000746727,
      "logits/rejected": -0.14933611452579498,
      "logps/chosen": -304.8170166015625,
      "logps/rejected": -281.7578125,
      "loss": 0.4965,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10470083355903625,
      "rewards/margins": 0.7507579326629639,
      "rewards/rejected": -0.8554587364196777,
      "step": 7035
    },
    {
      "epoch": 0.9207164472070009,
      "grad_norm": 8.651403427124023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006339967250823975,
      "logits/rejected": -0.04646913707256317,
      "logps/chosen": -405.80615234375,
      "logps/rejected": -578.2195434570312,
      "loss": 0.5872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.362173855304718,
      "rewards/margins": 0.449501097202301,
      "rewards/rejected": -0.8116748929023743,
      "step": 7036
    },
    {
      "epoch": 0.9208473051443526,
      "grad_norm": 5.639925003051758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17118620872497559,
      "logits/rejected": 0.17408785223960876,
      "logps/chosen": -581.1372680664062,
      "logps/rejected": -430.29779052734375,
      "loss": 0.3598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4071909785270691,
      "rewards/margins": 1.4324097633361816,
      "rewards/rejected": -1.0252187252044678,
      "step": 7037
    },
    {
      "epoch": 0.9209781630817044,
      "grad_norm": 6.703723430633545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08383303880691528,
      "logits/rejected": -0.13012003898620605,
      "logps/chosen": -418.13201904296875,
      "logps/rejected": -362.40826416015625,
      "loss": 0.5047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25051459670066833,
      "rewards/margins": 0.6659815311431885,
      "rewards/rejected": -0.9164961576461792,
      "step": 7038
    },
    {
      "epoch": 0.9211090210190562,
      "grad_norm": 5.969831466674805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06010878458619118,
      "logits/rejected": 0.1551523655653,
      "logps/chosen": -489.58233642578125,
      "logps/rejected": -418.84307861328125,
      "loss": 0.4311,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20377786457538605,
      "rewards/margins": 1.212997317314148,
      "rewards/rejected": -1.0092194080352783,
      "step": 7039
    },
    {
      "epoch": 0.9212398789564079,
      "grad_norm": 9.403568267822266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2795751392841339,
      "logits/rejected": 0.4328175187110901,
      "logps/chosen": -465.7586364746094,
      "logps/rejected": -567.3553466796875,
      "loss": 0.6794,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32281073927879333,
      "rewards/margins": 0.470964252948761,
      "rewards/rejected": -0.7937750220298767,
      "step": 7040
    },
    {
      "epoch": 0.9213707368937597,
      "grad_norm": 6.177302360534668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07149136811494827,
      "logits/rejected": 0.00603615865111351,
      "logps/chosen": -368.88641357421875,
      "logps/rejected": -501.0075988769531,
      "loss": 0.3635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.041203320026397705,
      "rewards/margins": 1.213498592376709,
      "rewards/rejected": -1.1722952127456665,
      "step": 7041
    },
    {
      "epoch": 0.9215015948311115,
      "grad_norm": 8.171514511108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25687819719314575,
      "logits/rejected": -0.3239405155181885,
      "logps/chosen": -466.27276611328125,
      "logps/rejected": -302.1867370605469,
      "loss": 0.585,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4953462779521942,
      "rewards/margins": 0.3933626115322113,
      "rewards/rejected": -0.8887089490890503,
      "step": 7042
    },
    {
      "epoch": 0.9216324527684633,
      "grad_norm": 9.435441970825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002259647473692894,
      "logits/rejected": -0.24132061004638672,
      "logps/chosen": -477.2818298339844,
      "logps/rejected": -420.0097351074219,
      "loss": 0.7835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6935949325561523,
      "rewards/margins": 0.4136137068271637,
      "rewards/rejected": -1.1072086095809937,
      "step": 7043
    },
    {
      "epoch": 0.921763310705815,
      "grad_norm": 5.182117462158203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10325749218463898,
      "logits/rejected": 0.0008041355758905411,
      "logps/chosen": -557.3328247070312,
      "logps/rejected": -504.11700439453125,
      "loss": 0.3841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9164738059043884,
      "rewards/margins": 1.4400168657302856,
      "rewards/rejected": -0.5235430002212524,
      "step": 7044
    },
    {
      "epoch": 0.9218941686431668,
      "grad_norm": 11.839810371398926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21952296793460846,
      "logits/rejected": -0.5167533755302429,
      "logps/chosen": -398.15911865234375,
      "logps/rejected": -398.1653747558594,
      "loss": 0.6012,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3988065719604492,
      "rewards/margins": 1.2115240097045898,
      "rewards/rejected": -2.610330820083618,
      "step": 7045
    },
    {
      "epoch": 0.9220250265805185,
      "grad_norm": 4.603287696838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08447310328483582,
      "logits/rejected": -0.18971051275730133,
      "logps/chosen": -499.64459228515625,
      "logps/rejected": -467.6181640625,
      "loss": 0.2862,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37354087829589844,
      "rewards/margins": 1.2978756427764893,
      "rewards/rejected": -1.6714165210723877,
      "step": 7046
    },
    {
      "epoch": 0.9221558845178703,
      "grad_norm": 7.109714984893799,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28894296288490295,
      "logits/rejected": -0.28397077322006226,
      "logps/chosen": -560.0916137695312,
      "logps/rejected": -390.6625061035156,
      "loss": 0.3002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5185035467147827,
      "rewards/margins": 1.8976629972457886,
      "rewards/rejected": -1.3791594505310059,
      "step": 7047
    },
    {
      "epoch": 0.922286742455222,
      "grad_norm": 4.652278423309326,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0582193061709404,
      "logits/rejected": -0.1347617208957672,
      "logps/chosen": -444.0036926269531,
      "logps/rejected": -380.8037414550781,
      "loss": 0.3288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7298163771629333,
      "rewards/margins": 1.3393146991729736,
      "rewards/rejected": -0.6094984412193298,
      "step": 7048
    },
    {
      "epoch": 0.9224176003925738,
      "grad_norm": 5.775312423706055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08448445796966553,
      "logits/rejected": -0.19543510675430298,
      "logps/chosen": -411.79583740234375,
      "logps/rejected": -367.86248779296875,
      "loss": 0.4863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28997907042503357,
      "rewards/margins": 0.9937724471092224,
      "rewards/rejected": -1.2837514877319336,
      "step": 7049
    },
    {
      "epoch": 0.9225484583299256,
      "grad_norm": 4.330724239349365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.104903943836689,
      "logits/rejected": -0.012015506625175476,
      "logps/chosen": -636.5983276367188,
      "logps/rejected": -570.8915405273438,
      "loss": 0.2345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05028276890516281,
      "rewards/margins": 1.5515950918197632,
      "rewards/rejected": -1.6018778085708618,
      "step": 7050
    },
    {
      "epoch": 0.9226793162672773,
      "grad_norm": 6.489340782165527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15784621238708496,
      "logits/rejected": 0.034893956035375595,
      "logps/chosen": -363.60797119140625,
      "logps/rejected": -472.318603515625,
      "loss": 0.4044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23912622034549713,
      "rewards/margins": 0.9045047163963318,
      "rewards/rejected": -1.143630862236023,
      "step": 7051
    },
    {
      "epoch": 0.9228101742046291,
      "grad_norm": 2.8880221843719482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3067680895328522,
      "logits/rejected": -0.1619192361831665,
      "logps/chosen": -449.1692810058594,
      "logps/rejected": -379.6041259765625,
      "loss": 0.271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6519476175308228,
      "rewards/margins": 1.7837165594100952,
      "rewards/rejected": -1.131769061088562,
      "step": 7052
    },
    {
      "epoch": 0.9229410321419809,
      "grad_norm": 8.193734169006348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11978209763765335,
      "logits/rejected": 0.13870617747306824,
      "logps/chosen": -611.7097778320312,
      "logps/rejected": -571.0166625976562,
      "loss": 0.4821,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10720177739858627,
      "rewards/margins": 1.3735501766204834,
      "rewards/rejected": -1.2663484811782837,
      "step": 7053
    },
    {
      "epoch": 0.9230718900793327,
      "grad_norm": 4.328392505645752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.052735067903995514,
      "logits/rejected": -0.10243450105190277,
      "logps/chosen": -425.7021484375,
      "logps/rejected": -669.0328369140625,
      "loss": 0.2879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09776246547698975,
      "rewards/margins": 1.8073431253433228,
      "rewards/rejected": -1.7095807790756226,
      "step": 7054
    },
    {
      "epoch": 0.9232027480166843,
      "grad_norm": 8.573471069335938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029874153435230255,
      "logits/rejected": -0.022865835577249527,
      "logps/chosen": -587.1910400390625,
      "logps/rejected": -501.96185302734375,
      "loss": 0.751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4758051037788391,
      "rewards/margins": 0.6641985177993774,
      "rewards/rejected": -0.18839339911937714,
      "step": 7055
    },
    {
      "epoch": 0.9233336059540361,
      "grad_norm": 11.1336030960083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1886238157749176,
      "logits/rejected": 0.06991195678710938,
      "logps/chosen": -484.7040710449219,
      "logps/rejected": -485.662841796875,
      "loss": 0.7612,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06472986936569214,
      "rewards/margins": 0.25520095229148865,
      "rewards/rejected": -0.1904710829257965,
      "step": 7056
    },
    {
      "epoch": 0.9234644638913879,
      "grad_norm": 5.565988063812256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2166135013103485,
      "logits/rejected": -0.1804792582988739,
      "logps/chosen": -533.858642578125,
      "logps/rejected": -547.2298583984375,
      "loss": 0.3851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3110632300376892,
      "rewards/margins": 0.9563961029052734,
      "rewards/rejected": -1.2674592733383179,
      "step": 7057
    },
    {
      "epoch": 0.9235953218287397,
      "grad_norm": 9.104275703430176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017678195610642433,
      "logits/rejected": -0.38006317615509033,
      "logps/chosen": -535.4901123046875,
      "logps/rejected": -594.4675903320312,
      "loss": 0.5068,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17817270755767822,
      "rewards/margins": 2.404576301574707,
      "rewards/rejected": -2.582749366760254,
      "step": 7058
    },
    {
      "epoch": 0.9237261797660914,
      "grad_norm": 7.574289321899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05910022556781769,
      "logits/rejected": 0.1185344010591507,
      "logps/chosen": -444.33062744140625,
      "logps/rejected": -461.520751953125,
      "loss": 0.6427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03722648322582245,
      "rewards/margins": 0.27981165051460266,
      "rewards/rejected": -0.2425852119922638,
      "step": 7059
    },
    {
      "epoch": 0.9238570377034432,
      "grad_norm": 7.342528820037842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12525664269924164,
      "logits/rejected": 0.001865241676568985,
      "logps/chosen": -375.7597351074219,
      "logps/rejected": -414.61669921875,
      "loss": 0.5638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06495562940835953,
      "rewards/margins": 0.4637282192707062,
      "rewards/rejected": -0.39877262711524963,
      "step": 7060
    },
    {
      "epoch": 0.923987895640795,
      "grad_norm": 5.373373031616211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08373138308525085,
      "logits/rejected": 0.2140873670578003,
      "logps/chosen": -506.91729736328125,
      "logps/rejected": -443.00482177734375,
      "loss": 0.3921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05126515030860901,
      "rewards/margins": 1.3799190521240234,
      "rewards/rejected": -1.3286538124084473,
      "step": 7061
    },
    {
      "epoch": 0.9241187535781468,
      "grad_norm": 9.536437034606934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06864842772483826,
      "logits/rejected": -0.27836018800735474,
      "logps/chosen": -464.18951416015625,
      "logps/rejected": -428.5579528808594,
      "loss": 0.7451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2593500316143036,
      "rewards/margins": 0.5751851797103882,
      "rewards/rejected": -0.8345352411270142,
      "step": 7062
    },
    {
      "epoch": 0.9242496115154984,
      "grad_norm": 3.9950203895568848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15102529525756836,
      "logits/rejected": -0.13889655470848083,
      "logps/chosen": -326.6988525390625,
      "logps/rejected": -339.7774963378906,
      "loss": 0.2727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33354681730270386,
      "rewards/margins": 1.5090594291687012,
      "rewards/rejected": -1.8426064252853394,
      "step": 7063
    },
    {
      "epoch": 0.9243804694528502,
      "grad_norm": 7.825955867767334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.044767480343580246,
      "logits/rejected": -0.2347394973039627,
      "logps/chosen": -422.77777099609375,
      "logps/rejected": -423.1707458496094,
      "loss": 0.5808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4117124378681183,
      "rewards/margins": 0.35359519720077515,
      "rewards/rejected": -0.765307605266571,
      "step": 7064
    },
    {
      "epoch": 0.924511327390202,
      "grad_norm": 7.746632099151611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11520954221487045,
      "logits/rejected": -0.20610761642456055,
      "logps/chosen": -434.0116271972656,
      "logps/rejected": -458.3001403808594,
      "loss": 0.43,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1413242518901825,
      "rewards/margins": 1.6284420490264893,
      "rewards/rejected": -1.7697663307189941,
      "step": 7065
    },
    {
      "epoch": 0.9246421853275538,
      "grad_norm": 8.498573303222656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07647238671779633,
      "logits/rejected": 0.0015586151275783777,
      "logps/chosen": -447.2716369628906,
      "logps/rejected": -428.79339599609375,
      "loss": 0.5965,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2250163555145264,
      "rewards/margins": 0.33774054050445557,
      "rewards/rejected": -1.5627570152282715,
      "step": 7066
    },
    {
      "epoch": 0.9247730432649055,
      "grad_norm": 6.545565605163574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014315381646156311,
      "logits/rejected": 0.11498679220676422,
      "logps/chosen": -359.5961608886719,
      "logps/rejected": -367.926513671875,
      "loss": 0.4693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15766534209251404,
      "rewards/margins": 0.8506518602371216,
      "rewards/rejected": -1.008317232131958,
      "step": 7067
    },
    {
      "epoch": 0.9249039012022573,
      "grad_norm": 8.091156005859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14494353532791138,
      "logits/rejected": -0.24070437252521515,
      "logps/chosen": -392.6573791503906,
      "logps/rejected": -447.810791015625,
      "loss": 0.9277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8954832553863525,
      "rewards/margins": 0.7565857172012329,
      "rewards/rejected": -1.6520689725875854,
      "step": 7068
    },
    {
      "epoch": 0.9250347591396091,
      "grad_norm": 7.511063098907471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21276557445526123,
      "logits/rejected": -0.007250024005770683,
      "logps/chosen": -523.19580078125,
      "logps/rejected": -421.0508728027344,
      "loss": 0.5363,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.21808069944381714,
      "rewards/margins": 1.0562502145767212,
      "rewards/rejected": -0.8381694555282593,
      "step": 7069
    },
    {
      "epoch": 0.9251656170769609,
      "grad_norm": 5.17391300201416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43473586440086365,
      "logits/rejected": -0.6808035373687744,
      "logps/chosen": -347.49078369140625,
      "logps/rejected": -438.4422302246094,
      "loss": 0.2478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.32307732105255127,
      "rewards/margins": 2.0413718223571777,
      "rewards/rejected": -2.3644492626190186,
      "step": 7070
    },
    {
      "epoch": 0.9252964750143126,
      "grad_norm": 6.628756523132324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18835577368736267,
      "logits/rejected": 0.0005057379603385925,
      "logps/chosen": -544.2066040039062,
      "logps/rejected": -496.119873046875,
      "loss": 0.3604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2781229019165039,
      "rewards/margins": 1.3828068971633911,
      "rewards/rejected": -1.6609296798706055,
      "step": 7071
    },
    {
      "epoch": 0.9254273329516643,
      "grad_norm": 10.100265502929688,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11353149265050888,
      "logits/rejected": 0.1599818915128708,
      "logps/chosen": -391.29217529296875,
      "logps/rejected": -391.1275634765625,
      "loss": 0.8577,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3373245298862457,
      "rewards/margins": 0.014053523540496826,
      "rewards/rejected": -0.35137805342674255,
      "step": 7072
    },
    {
      "epoch": 0.9255581908890161,
      "grad_norm": 6.483187198638916,
      "learning_rate": 5e-06,
      "logits/chosen": -4.8860907554626465e-05,
      "logits/rejected": 0.04300576448440552,
      "logps/chosen": -533.8076171875,
      "logps/rejected": -432.7162170410156,
      "loss": 0.397,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7596029043197632,
      "rewards/margins": 1.251600980758667,
      "rewards/rejected": -2.0112040042877197,
      "step": 7073
    },
    {
      "epoch": 0.9256890488263678,
      "grad_norm": 7.557730674743652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12677966058254242,
      "logits/rejected": 0.03463561460375786,
      "logps/chosen": -343.18316650390625,
      "logps/rejected": -325.13232421875,
      "loss": 0.6602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6003738641738892,
      "rewards/margins": 0.46646764874458313,
      "rewards/rejected": -1.06684148311615,
      "step": 7074
    },
    {
      "epoch": 0.9258199067637196,
      "grad_norm": 6.1909589767456055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05573362857103348,
      "logits/rejected": -0.046627409756183624,
      "logps/chosen": -494.8609313964844,
      "logps/rejected": -455.0172424316406,
      "loss": 0.245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.367627888917923,
      "rewards/margins": 1.9830684661865234,
      "rewards/rejected": -2.350696325302124,
      "step": 7075
    },
    {
      "epoch": 0.9259507647010714,
      "grad_norm": 6.459169387817383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08358745276927948,
      "logits/rejected": -0.27879950404167175,
      "logps/chosen": -640.2125244140625,
      "logps/rejected": -410.68621826171875,
      "loss": 0.3873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22153642773628235,
      "rewards/margins": 1.179870843887329,
      "rewards/rejected": -0.9583343267440796,
      "step": 7076
    },
    {
      "epoch": 0.9260816226384232,
      "grad_norm": 6.422175884246826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2517533600330353,
      "logits/rejected": 0.19318781793117523,
      "logps/chosen": -591.1928100585938,
      "logps/rejected": -461.9336853027344,
      "loss": 0.3527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11863292753696442,
      "rewards/margins": 1.1053922176361084,
      "rewards/rejected": -0.9867591857910156,
      "step": 7077
    },
    {
      "epoch": 0.9262124805757749,
      "grad_norm": 6.614779472351074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14814084768295288,
      "logits/rejected": -0.03935638442635536,
      "logps/chosen": -384.8072509765625,
      "logps/rejected": -402.6439514160156,
      "loss": 0.4524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5958855748176575,
      "rewards/margins": 0.9441590309143066,
      "rewards/rejected": -1.5400447845458984,
      "step": 7078
    },
    {
      "epoch": 0.9263433385131267,
      "grad_norm": 8.714156150817871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07847851514816284,
      "logits/rejected": -0.11540628969669342,
      "logps/chosen": -421.8653259277344,
      "logps/rejected": -561.8975830078125,
      "loss": 0.4901,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7014164328575134,
      "rewards/margins": 1.01858651638031,
      "rewards/rejected": -0.3171701431274414,
      "step": 7079
    },
    {
      "epoch": 0.9264741964504785,
      "grad_norm": 5.896450519561768,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2827249765396118,
      "logits/rejected": -0.03433044254779816,
      "logps/chosen": -328.4408874511719,
      "logps/rejected": -272.5398864746094,
      "loss": 0.3771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.036567699164152145,
      "rewards/margins": 1.1997194290161133,
      "rewards/rejected": -1.236286997795105,
      "step": 7080
    },
    {
      "epoch": 0.9266050543878303,
      "grad_norm": 8.245331764221191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042414478957653046,
      "logits/rejected": -0.1306150257587433,
      "logps/chosen": -342.4326171875,
      "logps/rejected": -404.15618896484375,
      "loss": 0.5619,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09511490166187286,
      "rewards/margins": 0.6808262467384338,
      "rewards/rejected": -0.7759411334991455,
      "step": 7081
    },
    {
      "epoch": 0.9267359123251819,
      "grad_norm": 8.131669044494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13722892105579376,
      "logits/rejected": -0.0964980274438858,
      "logps/chosen": -648.6945190429688,
      "logps/rejected": -430.13006591796875,
      "loss": 0.5782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0879993587732315,
      "rewards/margins": 1.032352089881897,
      "rewards/rejected": -1.1203513145446777,
      "step": 7082
    },
    {
      "epoch": 0.9268667702625337,
      "grad_norm": 7.6696648597717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23858630657196045,
      "logits/rejected": 0.11190801113843918,
      "logps/chosen": -273.8866882324219,
      "logps/rejected": -346.2880859375,
      "loss": 0.5145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6890330910682678,
      "rewards/margins": 0.6430681943893433,
      "rewards/rejected": -1.3321011066436768,
      "step": 7083
    },
    {
      "epoch": 0.9269976281998855,
      "grad_norm": 7.502652168273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07636334747076035,
      "logits/rejected": 0.11078600585460663,
      "logps/chosen": -402.1840515136719,
      "logps/rejected": -451.2208251953125,
      "loss": 0.5601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37224826216697693,
      "rewards/margins": 0.5909878015518188,
      "rewards/rejected": -0.9632360339164734,
      "step": 7084
    },
    {
      "epoch": 0.9271284861372373,
      "grad_norm": 8.300536155700684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17335009574890137,
      "logits/rejected": -0.24244986474514008,
      "logps/chosen": -383.050537109375,
      "logps/rejected": -465.7594909667969,
      "loss": 0.416,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011075004935264587,
      "rewards/margins": 1.5588316917419434,
      "rewards/rejected": -1.5699065923690796,
      "step": 7085
    },
    {
      "epoch": 0.927259344074589,
      "grad_norm": 3.090949058532715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16998514533042908,
      "logits/rejected": 0.03870147466659546,
      "logps/chosen": -549.1732788085938,
      "logps/rejected": -457.12164306640625,
      "loss": 0.1905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3621532618999481,
      "rewards/margins": 1.8790037631988525,
      "rewards/rejected": -1.5168505907058716,
      "step": 7086
    },
    {
      "epoch": 0.9273902020119408,
      "grad_norm": 7.84586238861084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14832749962806702,
      "logits/rejected": -0.025793008506298065,
      "logps/chosen": -333.5377502441406,
      "logps/rejected": -357.5045166015625,
      "loss": 0.4733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7586116790771484,
      "rewards/margins": 0.9460217952728271,
      "rewards/rejected": -1.7046334743499756,
      "step": 7087
    },
    {
      "epoch": 0.9275210599492926,
      "grad_norm": 7.497721195220947,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14543183147907257,
      "logits/rejected": -0.30648884177207947,
      "logps/chosen": -380.8372802734375,
      "logps/rejected": -372.12628173828125,
      "loss": 0.4276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.633284330368042,
      "rewards/margins": 1.1393874883651733,
      "rewards/rejected": -1.7726718187332153,
      "step": 7088
    },
    {
      "epoch": 0.9276519178866444,
      "grad_norm": 9.716854095458984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.243729829788208,
      "logits/rejected": -0.1725672334432602,
      "logps/chosen": -441.2951354980469,
      "logps/rejected": -434.4384460449219,
      "loss": 0.4539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27569830417633057,
      "rewards/margins": 0.9830595254898071,
      "rewards/rejected": -1.2587578296661377,
      "step": 7089
    },
    {
      "epoch": 0.927782775823996,
      "grad_norm": 10.172649383544922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17667318880558014,
      "logits/rejected": 0.10016196966171265,
      "logps/chosen": -571.7037353515625,
      "logps/rejected": -622.230224609375,
      "loss": 0.637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5957454442977905,
      "rewards/margins": 0.7194923162460327,
      "rewards/rejected": -1.3152377605438232,
      "step": 7090
    },
    {
      "epoch": 0.9279136337613478,
      "grad_norm": 10.051164627075195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20557937026023865,
      "logits/rejected": -0.07401883602142334,
      "logps/chosen": -363.0359802246094,
      "logps/rejected": -385.7095642089844,
      "loss": 0.5976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20214375853538513,
      "rewards/margins": 0.5337569713592529,
      "rewards/rejected": -0.3316131830215454,
      "step": 7091
    },
    {
      "epoch": 0.9280444916986996,
      "grad_norm": 13.718245506286621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06104631721973419,
      "logits/rejected": -0.06828143447637558,
      "logps/chosen": -473.9388427734375,
      "logps/rejected": -477.99658203125,
      "loss": 1.0218,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8827340006828308,
      "rewards/margins": -0.05280579626560211,
      "rewards/rejected": -0.8299282193183899,
      "step": 7092
    },
    {
      "epoch": 0.9281753496360514,
      "grad_norm": 5.46414852142334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14120879769325256,
      "logits/rejected": -0.15071070194244385,
      "logps/chosen": -430.9884033203125,
      "logps/rejected": -298.1726379394531,
      "loss": 0.3265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6473081707954407,
      "rewards/margins": 1.3534115552902222,
      "rewards/rejected": -0.7061033248901367,
      "step": 7093
    },
    {
      "epoch": 0.9283062075734031,
      "grad_norm": 7.749556541442871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11634179204702377,
      "logits/rejected": -0.002690335735678673,
      "logps/chosen": -424.84661865234375,
      "logps/rejected": -356.9822998046875,
      "loss": 0.6003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020359806716442108,
      "rewards/margins": 0.9269847869873047,
      "rewards/rejected": -0.9066250324249268,
      "step": 7094
    },
    {
      "epoch": 0.9284370655107549,
      "grad_norm": 4.952369213104248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21915169060230255,
      "logits/rejected": -0.3778473734855652,
      "logps/chosen": -501.94842529296875,
      "logps/rejected": -317.77032470703125,
      "loss": 0.3657,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3771810531616211,
      "rewards/margins": 1.1958529949188232,
      "rewards/rejected": -1.5730340480804443,
      "step": 7095
    },
    {
      "epoch": 0.9285679234481067,
      "grad_norm": 7.854785919189453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1535591036081314,
      "logits/rejected": 0.12686385214328766,
      "logps/chosen": -590.1044311523438,
      "logps/rejected": -540.8721923828125,
      "loss": 0.4921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4599929749965668,
      "rewards/margins": 0.912274181842804,
      "rewards/rejected": -0.4522812068462372,
      "step": 7096
    },
    {
      "epoch": 0.9286987813854584,
      "grad_norm": 5.219374656677246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20163922011852264,
      "logits/rejected": -0.17217087745666504,
      "logps/chosen": -397.7331237792969,
      "logps/rejected": -398.0335388183594,
      "loss": 0.3871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17174187302589417,
      "rewards/margins": 1.128947377204895,
      "rewards/rejected": -1.300689458847046,
      "step": 7097
    },
    {
      "epoch": 0.9288296393228102,
      "grad_norm": 6.769852638244629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.038166843354701996,
      "logits/rejected": 0.04558975622057915,
      "logps/chosen": -351.36407470703125,
      "logps/rejected": -390.05474853515625,
      "loss": 0.5266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4523801803588867,
      "rewards/margins": 0.9618877172470093,
      "rewards/rejected": -0.5095075368881226,
      "step": 7098
    },
    {
      "epoch": 0.9289604972601619,
      "grad_norm": 3.271193027496338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04698554053902626,
      "logits/rejected": 0.15538424253463745,
      "logps/chosen": -291.83001708984375,
      "logps/rejected": -367.4298095703125,
      "loss": 0.3464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2226129174232483,
      "rewards/margins": 1.7594972848892212,
      "rewards/rejected": -1.5368846654891968,
      "step": 7099
    },
    {
      "epoch": 0.9290913551975137,
      "grad_norm": 5.150482177734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20917385816574097,
      "logits/rejected": -0.15944378077983856,
      "logps/chosen": -478.0003967285156,
      "logps/rejected": -552.4468994140625,
      "loss": 0.3339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07845287024974823,
      "rewards/margins": 1.9012824296951294,
      "rewards/rejected": -1.8228297233581543,
      "step": 7100
    },
    {
      "epoch": 0.9292222131348654,
      "grad_norm": 11.042797088623047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3063619136810303,
      "logits/rejected": -0.15496473014354706,
      "logps/chosen": -426.5187683105469,
      "logps/rejected": -649.857666015625,
      "loss": 0.8953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2289329469203949,
      "rewards/margins": 0.08296102285385132,
      "rewards/rejected": -0.3118939697742462,
      "step": 7101
    },
    {
      "epoch": 0.9293530710722172,
      "grad_norm": 6.130800724029541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31259238719940186,
      "logits/rejected": -0.3449954688549042,
      "logps/chosen": -337.39898681640625,
      "logps/rejected": -306.6365966796875,
      "loss": 0.4988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0855191946029663,
      "rewards/margins": 0.6265464425086975,
      "rewards/rejected": -1.712065577507019,
      "step": 7102
    },
    {
      "epoch": 0.929483929009569,
      "grad_norm": 7.392549514770508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2889481484889984,
      "logits/rejected": -0.35221853852272034,
      "logps/chosen": -334.3813171386719,
      "logps/rejected": -418.10888671875,
      "loss": 0.4686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6951249837875366,
      "rewards/margins": 1.1818516254425049,
      "rewards/rejected": -1.876976728439331,
      "step": 7103
    },
    {
      "epoch": 0.9296147869469208,
      "grad_norm": 7.685812950134277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21786026656627655,
      "logits/rejected": -0.40897366404533386,
      "logps/chosen": -425.3609313964844,
      "logps/rejected": -587.7393798828125,
      "loss": 0.4583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5824233293533325,
      "rewards/margins": 1.375844955444336,
      "rewards/rejected": -1.958268404006958,
      "step": 7104
    },
    {
      "epoch": 0.9297456448842725,
      "grad_norm": 6.2601518630981445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07885739952325821,
      "logits/rejected": -0.00983969122171402,
      "logps/chosen": -583.2115478515625,
      "logps/rejected": -542.6843872070312,
      "loss": 0.4347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10185681283473969,
      "rewards/margins": 1.2426713705062866,
      "rewards/rejected": -1.1408145427703857,
      "step": 7105
    },
    {
      "epoch": 0.9298765028216243,
      "grad_norm": 6.762479305267334,
      "learning_rate": 5e-06,
      "logits/chosen": 0.165103942155838,
      "logits/rejected": 0.11483850330114365,
      "logps/chosen": -388.4810485839844,
      "logps/rejected": -402.8464660644531,
      "loss": 0.4276,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09353676438331604,
      "rewards/margins": 1.2680788040161133,
      "rewards/rejected": -1.1745420694351196,
      "step": 7106
    },
    {
      "epoch": 0.930007360758976,
      "grad_norm": 7.301579475402832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1039874479174614,
      "logits/rejected": -0.08437095582485199,
      "logps/chosen": -424.41455078125,
      "logps/rejected": -359.3774108886719,
      "loss": 0.5872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31872209906578064,
      "rewards/margins": 0.5503553748130798,
      "rewards/rejected": -0.8690775036811829,
      "step": 7107
    },
    {
      "epoch": 0.9301382186963278,
      "grad_norm": 8.72704792022705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15814706683158875,
      "logits/rejected": -0.06234952062368393,
      "logps/chosen": -366.9362487792969,
      "logps/rejected": -363.73956298828125,
      "loss": 0.8019,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.18474212288856506,
      "rewards/margins": 0.09258712828159332,
      "rewards/rejected": 0.0921550840139389,
      "step": 7108
    },
    {
      "epoch": 0.9302690766336795,
      "grad_norm": 7.372978687286377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15652622282505035,
      "logits/rejected": -0.524655282497406,
      "logps/chosen": -579.7913818359375,
      "logps/rejected": -392.0069885253906,
      "loss": 0.5699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3902580738067627,
      "rewards/margins": 0.9827331304550171,
      "rewards/rejected": -1.3729912042617798,
      "step": 7109
    },
    {
      "epoch": 0.9303999345710313,
      "grad_norm": 5.233222961425781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1325201392173767,
      "logits/rejected": 0.20610502362251282,
      "logps/chosen": -601.1782836914062,
      "logps/rejected": -442.60687255859375,
      "loss": 0.3357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06013964116573334,
      "rewards/margins": 1.3145476579666138,
      "rewards/rejected": -1.2544078826904297,
      "step": 7110
    },
    {
      "epoch": 0.9305307925083831,
      "grad_norm": 7.1555047035217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1482296586036682,
      "logits/rejected": -0.07526902109384537,
      "logps/chosen": -454.282958984375,
      "logps/rejected": -375.1460876464844,
      "loss": 0.3361,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6643920540809631,
      "rewards/margins": 1.378444790840149,
      "rewards/rejected": -0.7140527367591858,
      "step": 7111
    },
    {
      "epoch": 0.9306616504457349,
      "grad_norm": 10.670557975769043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08444830775260925,
      "logits/rejected": -0.21637651324272156,
      "logps/chosen": -366.83172607421875,
      "logps/rejected": -345.8439025878906,
      "loss": 0.9322,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4729560911655426,
      "rewards/margins": 0.2631646692752838,
      "rewards/rejected": -0.7361208200454712,
      "step": 7112
    },
    {
      "epoch": 0.9307925083830866,
      "grad_norm": 12.796504020690918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032936908304691315,
      "logits/rejected": -0.1624310463666916,
      "logps/chosen": -466.4532470703125,
      "logps/rejected": -459.89813232421875,
      "loss": 0.8579,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08681946247816086,
      "rewards/margins": -0.03933122754096985,
      "rewards/rejected": -0.04748821258544922,
      "step": 7113
    },
    {
      "epoch": 0.9309233663204384,
      "grad_norm": 8.517739295959473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30023565888404846,
      "logits/rejected": -0.08317669481039047,
      "logps/chosen": -310.61083984375,
      "logps/rejected": -337.45123291015625,
      "loss": 0.5004,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2466952800750732,
      "rewards/margins": 0.8625404238700867,
      "rewards/rejected": -2.1092357635498047,
      "step": 7114
    },
    {
      "epoch": 0.9310542242577902,
      "grad_norm": 5.523975849151611,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11642532050609589,
      "logits/rejected": -0.013029220513999462,
      "logps/chosen": -442.0959777832031,
      "logps/rejected": -382.361328125,
      "loss": 0.4082,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4749186038970947,
      "rewards/margins": 1.1144495010375977,
      "rewards/rejected": -0.6395307779312134,
      "step": 7115
    },
    {
      "epoch": 0.9311850821951418,
      "grad_norm": 8.497209548950195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20007970929145813,
      "logits/rejected": 0.13940228521823883,
      "logps/chosen": -362.0413818359375,
      "logps/rejected": -339.2532653808594,
      "loss": 0.5566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3644168972969055,
      "rewards/margins": 0.7431705594062805,
      "rewards/rejected": -1.107587456703186,
      "step": 7116
    },
    {
      "epoch": 0.9313159401324936,
      "grad_norm": 6.287991046905518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1294565498828888,
      "logits/rejected": 0.06415355950593948,
      "logps/chosen": -349.4986572265625,
      "logps/rejected": -447.322998046875,
      "loss": 0.4603,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.529316246509552,
      "rewards/margins": 0.8404857516288757,
      "rewards/rejected": -1.3698019981384277,
      "step": 7117
    },
    {
      "epoch": 0.9314467980698454,
      "grad_norm": 6.649725914001465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16896235942840576,
      "logits/rejected": -0.17667371034622192,
      "logps/chosen": -406.5091857910156,
      "logps/rejected": -401.59783935546875,
      "loss": 0.3983,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21446724236011505,
      "rewards/margins": 1.8045686483383179,
      "rewards/rejected": -1.5901013612747192,
      "step": 7118
    },
    {
      "epoch": 0.9315776560071972,
      "grad_norm": 11.438525199890137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05445501208305359,
      "logits/rejected": -0.13616737723350525,
      "logps/chosen": -417.37091064453125,
      "logps/rejected": -343.89794921875,
      "loss": 0.8666,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5019038915634155,
      "rewards/margins": -0.2372574806213379,
      "rewards/rejected": -0.26464635133743286,
      "step": 7119
    },
    {
      "epoch": 0.9317085139445489,
      "grad_norm": 7.617286682128906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2623997926712036,
      "logits/rejected": 0.2870911955833435,
      "logps/chosen": -370.5096740722656,
      "logps/rejected": -387.65899658203125,
      "loss": 0.5782,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12357348203659058,
      "rewards/margins": 0.7069852948188782,
      "rewards/rejected": -0.5834118127822876,
      "step": 7120
    },
    {
      "epoch": 0.9318393718819007,
      "grad_norm": 8.497132301330566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.319074809551239,
      "logits/rejected": -0.14130312204360962,
      "logps/chosen": -354.36871337890625,
      "logps/rejected": -382.8909606933594,
      "loss": 0.8483,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.000970721244812,
      "rewards/margins": 0.36654818058013916,
      "rewards/rejected": -1.3675189018249512,
      "step": 7121
    },
    {
      "epoch": 0.9319702298192525,
      "grad_norm": 2.9193098545074463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21167460083961487,
      "logits/rejected": -0.22213080525398254,
      "logps/chosen": -413.12335205078125,
      "logps/rejected": -473.9832763671875,
      "loss": 0.1345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8612268567085266,
      "rewards/margins": 2.5673487186431885,
      "rewards/rejected": -1.7061216831207275,
      "step": 7122
    },
    {
      "epoch": 0.9321010877566043,
      "grad_norm": 9.431056022644043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11124421656131744,
      "logits/rejected": -0.014632009901106358,
      "logps/chosen": -450.3406677246094,
      "logps/rejected": -380.82879638671875,
      "loss": 0.4531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5185693502426147,
      "rewards/margins": 1.7008967399597168,
      "rewards/rejected": -1.1823272705078125,
      "step": 7123
    },
    {
      "epoch": 0.932231945693956,
      "grad_norm": 7.011062145233154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08305130153894424,
      "logits/rejected": 0.005441499873995781,
      "logps/chosen": -396.0393981933594,
      "logps/rejected": -429.23638916015625,
      "loss": 0.4401,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.40484699606895447,
      "rewards/margins": 1.1358529329299927,
      "rewards/rejected": -0.7310060858726501,
      "step": 7124
    },
    {
      "epoch": 0.9323628036313077,
      "grad_norm": 9.48697280883789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05101960152387619,
      "logits/rejected": 0.19970238208770752,
      "logps/chosen": -541.5418701171875,
      "logps/rejected": -659.829833984375,
      "loss": 0.6258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017120361328125,
      "rewards/margins": 0.5716848373413086,
      "rewards/rejected": -0.5545645356178284,
      "step": 7125
    },
    {
      "epoch": 0.9324936615686595,
      "grad_norm": 5.981122016906738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13280953466892242,
      "logits/rejected": -0.0673220083117485,
      "logps/chosen": -375.2659912109375,
      "logps/rejected": -380.9888916015625,
      "loss": 0.483,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47587358951568604,
      "rewards/margins": 0.6249533295631409,
      "rewards/rejected": -0.14907971024513245,
      "step": 7126
    },
    {
      "epoch": 0.9326245195060113,
      "grad_norm": 7.964061260223389,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32698243856430054,
      "logits/rejected": 0.01735818386077881,
      "logps/chosen": -338.42803955078125,
      "logps/rejected": -453.62518310546875,
      "loss": 0.5512,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2045799195766449,
      "rewards/margins": 0.7167981863021851,
      "rewards/rejected": -0.5122182965278625,
      "step": 7127
    },
    {
      "epoch": 0.932755377443363,
      "grad_norm": 6.4651994705200195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3067004978656769,
      "logits/rejected": -0.07280952483415604,
      "logps/chosen": -439.4098815917969,
      "logps/rejected": -442.4263916015625,
      "loss": 0.5433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3190484941005707,
      "rewards/margins": 0.6241140365600586,
      "rewards/rejected": -0.3050655424594879,
      "step": 7128
    },
    {
      "epoch": 0.9328862353807148,
      "grad_norm": 8.818086624145508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.046383269131183624,
      "logits/rejected": -0.004318637773394585,
      "logps/chosen": -573.10791015625,
      "logps/rejected": -332.4006042480469,
      "loss": 0.7331,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2196376472711563,
      "rewards/margins": 0.14108504354953766,
      "rewards/rejected": -0.36072275042533875,
      "step": 7129
    },
    {
      "epoch": 0.9330170933180666,
      "grad_norm": 8.1349458694458,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10513366758823395,
      "logits/rejected": -0.3030310869216919,
      "logps/chosen": -383.5807189941406,
      "logps/rejected": -372.5163269042969,
      "loss": 0.8156,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1710115522146225,
      "rewards/margins": 0.1427721232175827,
      "rewards/rejected": -0.3137836754322052,
      "step": 7130
    },
    {
      "epoch": 0.9331479512554184,
      "grad_norm": 5.7489776611328125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09085211902856827,
      "logits/rejected": -0.09955351799726486,
      "logps/chosen": -437.16851806640625,
      "logps/rejected": -550.7265625,
      "loss": 0.395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03024197556078434,
      "rewards/margins": 1.4357919692993164,
      "rewards/rejected": -1.466033935546875,
      "step": 7131
    },
    {
      "epoch": 0.9332788091927701,
      "grad_norm": 7.280843734741211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2862865924835205,
      "logits/rejected": -0.21875661611557007,
      "logps/chosen": -462.68707275390625,
      "logps/rejected": -442.0592346191406,
      "loss": 0.5322,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3364119827747345,
      "rewards/margins": 0.7531123161315918,
      "rewards/rejected": -1.0895241498947144,
      "step": 7132
    },
    {
      "epoch": 0.9334096671301219,
      "grad_norm": 4.765505790710449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09825558960437775,
      "logits/rejected": -0.23131640255451202,
      "logps/chosen": -486.16070556640625,
      "logps/rejected": -458.44781494140625,
      "loss": 0.3907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.028175920248031616,
      "rewards/margins": 1.0671143531799316,
      "rewards/rejected": -1.0389384031295776,
      "step": 7133
    },
    {
      "epoch": 0.9335405250674736,
      "grad_norm": 4.075038433074951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05018651485443115,
      "logits/rejected": -0.11252240836620331,
      "logps/chosen": -517.6317138671875,
      "logps/rejected": -409.91436767578125,
      "loss": 0.2277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6341738104820251,
      "rewards/margins": 2.018126964569092,
      "rewards/rejected": -1.3839532136917114,
      "step": 7134
    },
    {
      "epoch": 0.9336713830048254,
      "grad_norm": 10.304902076721191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05379866808652878,
      "logits/rejected": 0.15484961867332458,
      "logps/chosen": -594.4580078125,
      "logps/rejected": -656.3336791992188,
      "loss": 0.816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15828344225883484,
      "rewards/margins": 0.39080333709716797,
      "rewards/rejected": -0.23251990973949432,
      "step": 7135
    },
    {
      "epoch": 0.9338022409421771,
      "grad_norm": 9.707610130310059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1562322974205017,
      "logits/rejected": -0.1584649384021759,
      "logps/chosen": -564.562255859375,
      "logps/rejected": -482.5274963378906,
      "loss": 0.7556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7580109238624573,
      "rewards/margins": 1.0136775970458984,
      "rewards/rejected": -1.771688461303711,
      "step": 7136
    },
    {
      "epoch": 0.9339330988795289,
      "grad_norm": 8.807998657226562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1608399599790573,
      "logits/rejected": -0.029744908213615417,
      "logps/chosen": -502.5942687988281,
      "logps/rejected": -397.7942810058594,
      "loss": 0.6751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01639539748430252,
      "rewards/margins": 0.9151357412338257,
      "rewards/rejected": -0.8987401723861694,
      "step": 7137
    },
    {
      "epoch": 0.9340639568168807,
      "grad_norm": 6.82089900970459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25205692648887634,
      "logits/rejected": -0.26099762320518494,
      "logps/chosen": -430.6463623046875,
      "logps/rejected": -427.05841064453125,
      "loss": 0.5375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29056990146636963,
      "rewards/margins": 0.4782480001449585,
      "rewards/rejected": -0.7688178420066833,
      "step": 7138
    },
    {
      "epoch": 0.9341948147542324,
      "grad_norm": 11.182806968688965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08664141595363617,
      "logits/rejected": -0.1918279528617859,
      "logps/chosen": -440.2499084472656,
      "logps/rejected": -424.4719543457031,
      "loss": 1.0965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.672244668006897,
      "rewards/margins": -0.2539638578891754,
      "rewards/rejected": -0.41828078031539917,
      "step": 7139
    },
    {
      "epoch": 0.9343256726915842,
      "grad_norm": 6.39700174331665,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1715942621231079,
      "logits/rejected": 0.16255977749824524,
      "logps/chosen": -327.0493469238281,
      "logps/rejected": -290.2586975097656,
      "loss": 0.4699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.553635835647583,
      "rewards/margins": 0.7949166297912598,
      "rewards/rejected": -1.3485523462295532,
      "step": 7140
    },
    {
      "epoch": 0.934456530628936,
      "grad_norm": 6.103260517120361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0927501767873764,
      "logits/rejected": -0.08018386363983154,
      "logps/chosen": -417.23529052734375,
      "logps/rejected": -400.99267578125,
      "loss": 0.3064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7312526702880859,
      "rewards/margins": 1.26629638671875,
      "rewards/rejected": -0.5350437164306641,
      "step": 7141
    },
    {
      "epoch": 0.9345873885662878,
      "grad_norm": 6.441562175750732,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13154089450836182,
      "logits/rejected": 0.062068622559309006,
      "logps/chosen": -387.5390319824219,
      "logps/rejected": -345.7536315917969,
      "loss": 0.3708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.023308277130127,
      "rewards/margins": 1.053922176361084,
      "rewards/rejected": -2.077230453491211,
      "step": 7142
    },
    {
      "epoch": 0.9347182465036394,
      "grad_norm": 9.18117904663086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09415726363658905,
      "logits/rejected": -0.08110460638999939,
      "logps/chosen": -429.2328796386719,
      "logps/rejected": -390.99774169921875,
      "loss": 0.6702,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8000474572181702,
      "rewards/margins": 0.722973108291626,
      "rewards/rejected": -1.5230205059051514,
      "step": 7143
    },
    {
      "epoch": 0.9348491044409912,
      "grad_norm": 5.616708755493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0830414742231369,
      "logits/rejected": -0.020717725157737732,
      "logps/chosen": -571.4784545898438,
      "logps/rejected": -526.6809692382812,
      "loss": 0.4021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5672591924667358,
      "rewards/margins": 1.185267448425293,
      "rewards/rejected": -1.7525267601013184,
      "step": 7144
    },
    {
      "epoch": 0.934979962378343,
      "grad_norm": 6.217149257659912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07782711833715439,
      "logits/rejected": 0.1524209827184677,
      "logps/chosen": -487.3927001953125,
      "logps/rejected": -394.9105529785156,
      "loss": 0.4251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3752649128437042,
      "rewards/margins": 1.3313572406768799,
      "rewards/rejected": -1.7066221237182617,
      "step": 7145
    },
    {
      "epoch": 0.9351108203156948,
      "grad_norm": 4.134972095489502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0795336440205574,
      "logits/rejected": -0.043081220239400864,
      "logps/chosen": -328.10296630859375,
      "logps/rejected": -436.2614440917969,
      "loss": 0.2239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46330738067626953,
      "rewards/margins": 2.3786110877990723,
      "rewards/rejected": -2.841918468475342,
      "step": 7146
    },
    {
      "epoch": 0.9352416782530465,
      "grad_norm": 9.888809204101562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09150692820549011,
      "logits/rejected": -0.04069262742996216,
      "logps/chosen": -495.625,
      "logps/rejected": -549.3545532226562,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5164991021156311,
      "rewards/margins": 0.7979191541671753,
      "rewards/rejected": -1.3144181966781616,
      "step": 7147
    },
    {
      "epoch": 0.9353725361903983,
      "grad_norm": 7.477877616882324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11047912389039993,
      "logits/rejected": 0.04291262477636337,
      "logps/chosen": -595.0655517578125,
      "logps/rejected": -461.6526794433594,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3394477367401123,
      "rewards/margins": 1.4665672779083252,
      "rewards/rejected": -1.127119541168213,
      "step": 7148
    },
    {
      "epoch": 0.9355033941277501,
      "grad_norm": 10.579018592834473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030339766293764114,
      "logits/rejected": 0.09978380799293518,
      "logps/chosen": -380.6380615234375,
      "logps/rejected": -472.03167724609375,
      "loss": 0.966,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0036789774894714355,
      "rewards/margins": -0.10357488691806793,
      "rewards/rejected": 0.10725381970405579,
      "step": 7149
    },
    {
      "epoch": 0.9356342520651019,
      "grad_norm": 10.200475692749023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1479988545179367,
      "logits/rejected": -0.02508080005645752,
      "logps/chosen": -401.54278564453125,
      "logps/rejected": -394.5157165527344,
      "loss": 0.7309,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1415884494781494,
      "rewards/margins": 0.2530807852745056,
      "rewards/rejected": -1.3946692943572998,
      "step": 7150
    },
    {
      "epoch": 0.9357651100024535,
      "grad_norm": 7.007694721221924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22717666625976562,
      "logits/rejected": 0.15500977635383606,
      "logps/chosen": -546.6989135742188,
      "logps/rejected": -395.5942077636719,
      "loss": 0.3761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05269753932952881,
      "rewards/margins": 1.6895790100097656,
      "rewards/rejected": -1.7422765493392944,
      "step": 7151
    },
    {
      "epoch": 0.9358959679398053,
      "grad_norm": 9.12775993347168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02294011041522026,
      "logits/rejected": -0.07100401073694229,
      "logps/chosen": -636.665771484375,
      "logps/rejected": -456.6397399902344,
      "loss": 0.4511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3533638119697571,
      "rewards/margins": 1.8129289150238037,
      "rewards/rejected": -2.166292667388916,
      "step": 7152
    },
    {
      "epoch": 0.9360268258771571,
      "grad_norm": 5.9850263595581055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004095159471035004,
      "logits/rejected": 0.2056885063648224,
      "logps/chosen": -330.1338806152344,
      "logps/rejected": -401.43096923828125,
      "loss": 0.364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.032571807503700256,
      "rewards/margins": 1.5354939699172974,
      "rewards/rejected": -1.5029221773147583,
      "step": 7153
    },
    {
      "epoch": 0.9361576838145089,
      "grad_norm": 12.876465797424316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17048519849777222,
      "logits/rejected": -0.062040895223617554,
      "logps/chosen": -422.34796142578125,
      "logps/rejected": -408.96575927734375,
      "loss": 0.7875,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5224491357803345,
      "rewards/margins": 0.25277063250541687,
      "rewards/rejected": -0.7752197980880737,
      "step": 7154
    },
    {
      "epoch": 0.9362885417518606,
      "grad_norm": 9.639050483703613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.033683910965919495,
      "logits/rejected": -0.11718946695327759,
      "logps/chosen": -292.0081787109375,
      "logps/rejected": -445.6677551269531,
      "loss": 0.6398,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13790684938430786,
      "rewards/margins": 0.94586181640625,
      "rewards/rejected": -1.0837687253952026,
      "step": 7155
    },
    {
      "epoch": 0.9364193996892124,
      "grad_norm": 7.981614589691162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19090114533901215,
      "logits/rejected": 0.1903468370437622,
      "logps/chosen": -492.98101806640625,
      "logps/rejected": -473.61895751953125,
      "loss": 0.47,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43182408809661865,
      "rewards/margins": 1.0229792594909668,
      "rewards/rejected": -1.4548033475875854,
      "step": 7156
    },
    {
      "epoch": 0.9365502576265642,
      "grad_norm": 11.860200881958008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011175591498613358,
      "logits/rejected": -0.11560317128896713,
      "logps/chosen": -325.5682373046875,
      "logps/rejected": -302.3419189453125,
      "loss": 0.7555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34449291229248047,
      "rewards/margins": 0.8137799501419067,
      "rewards/rejected": -1.1582729816436768,
      "step": 7157
    },
    {
      "epoch": 0.936681115563916,
      "grad_norm": 7.767493724822998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17192715406417847,
      "logits/rejected": -0.010250946506857872,
      "logps/chosen": -593.0277709960938,
      "logps/rejected": -511.87347412109375,
      "loss": 0.7116,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3322128355503082,
      "rewards/margins": 0.9271938800811768,
      "rewards/rejected": -0.5949810147285461,
      "step": 7158
    },
    {
      "epoch": 0.9368119735012677,
      "grad_norm": 6.06743860244751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026083506643772125,
      "logits/rejected": -0.10462513566017151,
      "logps/chosen": -414.6988220214844,
      "logps/rejected": -401.32684326171875,
      "loss": 0.3608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9187479019165039,
      "rewards/margins": 1.562333106994629,
      "rewards/rejected": -2.4810807704925537,
      "step": 7159
    },
    {
      "epoch": 0.9369428314386195,
      "grad_norm": 11.667936325073242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15549787878990173,
      "logits/rejected": -0.16768549382686615,
      "logps/chosen": -392.237060546875,
      "logps/rejected": -405.43731689453125,
      "loss": 0.8365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1310676634311676,
      "rewards/margins": 0.2418418675661087,
      "rewards/rejected": -0.3729095160961151,
      "step": 7160
    },
    {
      "epoch": 0.9370736893759712,
      "grad_norm": 8.970242500305176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12257304042577744,
      "logits/rejected": -0.14809006452560425,
      "logps/chosen": -728.3932495117188,
      "logps/rejected": -570.3807373046875,
      "loss": 0.5106,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6874775290489197,
      "rewards/margins": 1.3094031810760498,
      "rewards/rejected": -0.6219257712364197,
      "step": 7161
    },
    {
      "epoch": 0.9372045473133229,
      "grad_norm": 9.332653045654297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0611400306224823,
      "logits/rejected": -0.18198971450328827,
      "logps/chosen": -310.8611755371094,
      "logps/rejected": -465.67095947265625,
      "loss": 0.535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6040630340576172,
      "rewards/margins": 2.137803792953491,
      "rewards/rejected": -2.7418668270111084,
      "step": 7162
    },
    {
      "epoch": 0.9373354052506747,
      "grad_norm": 8.454771995544434,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11389848589897156,
      "logits/rejected": 0.09546788036823273,
      "logps/chosen": -481.2637634277344,
      "logps/rejected": -383.01849365234375,
      "loss": 0.594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16960367560386658,
      "rewards/margins": 1.2108434438705444,
      "rewards/rejected": -1.041239619255066,
      "step": 7163
    },
    {
      "epoch": 0.9374662631880265,
      "grad_norm": 9.355043411254883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05835951492190361,
      "logits/rejected": 0.15423299372196198,
      "logps/chosen": -428.8182067871094,
      "logps/rejected": -459.7028503417969,
      "loss": 0.8647,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.45660802721977234,
      "rewards/margins": 0.4637589752674103,
      "rewards/rejected": -0.007151007652282715,
      "step": 7164
    },
    {
      "epoch": 0.9375971211253783,
      "grad_norm": 6.4610419273376465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05366126447916031,
      "logits/rejected": 0.0032378733158111572,
      "logps/chosen": -639.0949096679688,
      "logps/rejected": -594.7783813476562,
      "loss": 0.4228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8869279623031616,
      "rewards/margins": 1.004409909248352,
      "rewards/rejected": -0.1174820065498352,
      "step": 7165
    },
    {
      "epoch": 0.93772797906273,
      "grad_norm": 10.70666217803955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1493658721446991,
      "logits/rejected": 0.043477192521095276,
      "logps/chosen": -439.9578552246094,
      "logps/rejected": -326.5393371582031,
      "loss": 0.7681,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5315414667129517,
      "rewards/margins": 0.7771486639976501,
      "rewards/rejected": -0.24560728669166565,
      "step": 7166
    },
    {
      "epoch": 0.9378588370000818,
      "grad_norm": 7.615045547485352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011228055693209171,
      "logits/rejected": -0.14846181869506836,
      "logps/chosen": -553.59765625,
      "logps/rejected": -385.3079528808594,
      "loss": 0.6989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.27417677640914917,
      "rewards/margins": 0.7375277876853943,
      "rewards/rejected": -0.4633510112762451,
      "step": 7167
    },
    {
      "epoch": 0.9379896949374336,
      "grad_norm": 8.318903923034668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.224636510014534,
      "logits/rejected": -0.3491215705871582,
      "logps/chosen": -482.558349609375,
      "logps/rejected": -466.8769226074219,
      "loss": 0.5075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20254728198051453,
      "rewards/margins": 1.3283181190490723,
      "rewards/rejected": -1.5308656692504883,
      "step": 7168
    },
    {
      "epoch": 0.9381205528747854,
      "grad_norm": 14.699420928955078,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21471017599105835,
      "logits/rejected": 0.11883290112018585,
      "logps/chosen": -603.15234375,
      "logps/rejected": -508.9400329589844,
      "loss": 1.0536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4399217367172241,
      "rewards/margins": -0.21562212705612183,
      "rewards/rejected": -0.22429966926574707,
      "step": 7169
    },
    {
      "epoch": 0.938251410812137,
      "grad_norm": 8.610424995422363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20424188673496246,
      "logits/rejected": -0.1577751338481903,
      "logps/chosen": -469.978271484375,
      "logps/rejected": -535.19921875,
      "loss": 0.5566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1794062852859497,
      "rewards/margins": 0.7932851314544678,
      "rewards/rejected": -1.972691535949707,
      "step": 7170
    },
    {
      "epoch": 0.9383822687494888,
      "grad_norm": 5.030634880065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047161608934402466,
      "logits/rejected": -0.159581258893013,
      "logps/chosen": -382.05621337890625,
      "logps/rejected": -317.64990234375,
      "loss": 0.3066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.007196053862571716,
      "rewards/margins": 1.5898480415344238,
      "rewards/rejected": -1.5826518535614014,
      "step": 7171
    },
    {
      "epoch": 0.9385131266868406,
      "grad_norm": 6.081180572509766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04582379758358002,
      "logits/rejected": 0.012195345014333725,
      "logps/chosen": -535.6192626953125,
      "logps/rejected": -512.784912109375,
      "loss": 0.416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10091697424650192,
      "rewards/margins": 1.2558294534683228,
      "rewards/rejected": -1.3567464351654053,
      "step": 7172
    },
    {
      "epoch": 0.9386439846241924,
      "grad_norm": 6.768234729766846,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31155306100845337,
      "logits/rejected": 0.09188175201416016,
      "logps/chosen": -496.9713439941406,
      "logps/rejected": -446.0553894042969,
      "loss": 0.5368,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22200661897659302,
      "rewards/margins": 1.144529104232788,
      "rewards/rejected": -1.3665357828140259,
      "step": 7173
    },
    {
      "epoch": 0.9387748425615441,
      "grad_norm": 8.608526229858398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04998353123664856,
      "logits/rejected": -0.12438929826021194,
      "logps/chosen": -388.41119384765625,
      "logps/rejected": -454.70404052734375,
      "loss": 0.4539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5495426654815674,
      "rewards/margins": 1.0907858610153198,
      "rewards/rejected": -1.6403286457061768,
      "step": 7174
    },
    {
      "epoch": 0.9389057004988959,
      "grad_norm": 9.464247703552246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09656123071908951,
      "logits/rejected": 0.016388872638344765,
      "logps/chosen": -516.6026611328125,
      "logps/rejected": -416.713623046875,
      "loss": 0.8505,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.23024922609329224,
      "rewards/margins": 0.4163471758365631,
      "rewards/rejected": -0.18609797954559326,
      "step": 7175
    },
    {
      "epoch": 0.9390365584362477,
      "grad_norm": 5.641587257385254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28256139159202576,
      "logits/rejected": 0.1662486344575882,
      "logps/chosen": -546.5358276367188,
      "logps/rejected": -419.39996337890625,
      "loss": 0.3025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25843545794487,
      "rewards/margins": 1.6231396198272705,
      "rewards/rejected": -1.8815749883651733,
      "step": 7176
    },
    {
      "epoch": 0.9391674163735995,
      "grad_norm": 5.5093674659729,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030612420290708542,
      "logits/rejected": -0.1976700723171234,
      "logps/chosen": -417.4167175292969,
      "logps/rejected": -505.7939758300781,
      "loss": 0.4708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30888956785202026,
      "rewards/margins": 0.9517872333526611,
      "rewards/rejected": -1.2606768608093262,
      "step": 7177
    },
    {
      "epoch": 0.9392982743109511,
      "grad_norm": 6.956230163574219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07913130521774292,
      "logits/rejected": -0.1597132831811905,
      "logps/chosen": -391.29815673828125,
      "logps/rejected": -546.5215454101562,
      "loss": 0.448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3416127860546112,
      "rewards/margins": 1.630763053894043,
      "rewards/rejected": -1.9723759889602661,
      "step": 7178
    },
    {
      "epoch": 0.9394291322483029,
      "grad_norm": 7.131404876708984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23660948872566223,
      "logits/rejected": -0.19413289427757263,
      "logps/chosen": -392.7103271484375,
      "logps/rejected": -420.8758544921875,
      "loss": 0.4898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2988760471343994,
      "rewards/margins": 1.32230806350708,
      "rewards/rejected": -2.6211843490600586,
      "step": 7179
    },
    {
      "epoch": 0.9395599901856547,
      "grad_norm": 7.8236589431762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0035296231508255005,
      "logits/rejected": 0.20810028910636902,
      "logps/chosen": -371.8830871582031,
      "logps/rejected": -354.6809997558594,
      "loss": 0.5184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3209955096244812,
      "rewards/margins": 1.09810209274292,
      "rewards/rejected": -1.4190977811813354,
      "step": 7180
    },
    {
      "epoch": 0.9396908481230064,
      "grad_norm": 4.887770175933838,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05421808362007141,
      "logits/rejected": -0.04339573159813881,
      "logps/chosen": -492.07958984375,
      "logps/rejected": -382.09576416015625,
      "loss": 0.331,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1453087031841278,
      "rewards/margins": 1.6420989036560059,
      "rewards/rejected": -1.4967902898788452,
      "step": 7181
    },
    {
      "epoch": 0.9398217060603582,
      "grad_norm": 5.83666467666626,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07256408035755157,
      "logits/rejected": -0.03100687265396118,
      "logps/chosen": -467.2276306152344,
      "logps/rejected": -408.414794921875,
      "loss": 0.4177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8408383131027222,
      "rewards/margins": 1.0022035837173462,
      "rewards/rejected": -0.1613653302192688,
      "step": 7182
    },
    {
      "epoch": 0.93995256399771,
      "grad_norm": 9.218916893005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037952810525894165,
      "logits/rejected": -0.05953584611415863,
      "logps/chosen": -342.21099853515625,
      "logps/rejected": -462.9198303222656,
      "loss": 0.8359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16814927756786346,
      "rewards/margins": 0.537227988243103,
      "rewards/rejected": -0.7053772211074829,
      "step": 7183
    },
    {
      "epoch": 0.9400834219350618,
      "grad_norm": 10.463984489440918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1719137728214264,
      "logits/rejected": -0.2171061635017395,
      "logps/chosen": -582.9088134765625,
      "logps/rejected": -540.5595092773438,
      "loss": 0.6356,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6237830519676208,
      "rewards/margins": 0.5838531255722046,
      "rewards/rejected": -1.2076362371444702,
      "step": 7184
    },
    {
      "epoch": 0.9402142798724135,
      "grad_norm": 7.536962985992432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18549269437789917,
      "logits/rejected": 0.0706876814365387,
      "logps/chosen": -468.87469482421875,
      "logps/rejected": -327.0483703613281,
      "loss": 0.6173,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5622926950454712,
      "rewards/margins": 0.4669376313686371,
      "rewards/rejected": 0.0953550636768341,
      "step": 7185
    },
    {
      "epoch": 0.9403451378097653,
      "grad_norm": 6.494504928588867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3687615990638733,
      "logits/rejected": -0.16830456256866455,
      "logps/chosen": -361.6336975097656,
      "logps/rejected": -380.8288269042969,
      "loss": 0.5957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20644092559814453,
      "rewards/margins": 1.1377900838851929,
      "rewards/rejected": -1.344231128692627,
      "step": 7186
    },
    {
      "epoch": 0.940475995747117,
      "grad_norm": 10.609382629394531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3717503845691681,
      "logits/rejected": -0.3281160295009613,
      "logps/chosen": -338.4935302734375,
      "logps/rejected": -384.6194152832031,
      "loss": 0.9172,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.9728204011917114,
      "rewards/margins": 0.28413379192352295,
      "rewards/rejected": -1.256954312324524,
      "step": 7187
    },
    {
      "epoch": 0.9406068536844688,
      "grad_norm": 4.811911106109619,
      "learning_rate": 5e-06,
      "logits/chosen": 0.45017361640930176,
      "logits/rejected": 0.30622637271881104,
      "logps/chosen": -541.5879516601562,
      "logps/rejected": -430.9690246582031,
      "loss": 0.3524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4565095901489258,
      "rewards/margins": 1.2484842538833618,
      "rewards/rejected": -1.7049938440322876,
      "step": 7188
    },
    {
      "epoch": 0.9407377116218205,
      "grad_norm": 6.275534152984619,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16431103646755219,
      "logits/rejected": -0.2523382902145386,
      "logps/chosen": -629.1881103515625,
      "logps/rejected": -536.0518188476562,
      "loss": 0.4374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5909774899482727,
      "rewards/margins": 1.3564631938934326,
      "rewards/rejected": -0.7654857635498047,
      "step": 7189
    },
    {
      "epoch": 0.9408685695591723,
      "grad_norm": 12.180306434631348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34774547815322876,
      "logits/rejected": 0.030963478609919548,
      "logps/chosen": -466.4196472167969,
      "logps/rejected": -579.8853759765625,
      "loss": 0.7635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0655769631266594,
      "rewards/margins": 0.38823583722114563,
      "rewards/rejected": -0.45381274819374084,
      "step": 7190
    },
    {
      "epoch": 0.9409994274965241,
      "grad_norm": 6.79543399810791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.42162632942199707,
      "logits/rejected": 0.37800896167755127,
      "logps/chosen": -459.1403503417969,
      "logps/rejected": -403.80206298828125,
      "loss": 0.4486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5769053101539612,
      "rewards/margins": 0.8148702383041382,
      "rewards/rejected": -0.23796498775482178,
      "step": 7191
    },
    {
      "epoch": 0.9411302854338759,
      "grad_norm": 6.268078327178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009496808052062988,
      "logits/rejected": -0.10907310247421265,
      "logps/chosen": -479.61639404296875,
      "logps/rejected": -463.76580810546875,
      "loss": 0.3123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14062382280826569,
      "rewards/margins": 1.5217894315719604,
      "rewards/rejected": -1.3811655044555664,
      "step": 7192
    },
    {
      "epoch": 0.9412611433712276,
      "grad_norm": 7.75650691986084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35466137528419495,
      "logits/rejected": 0.0067275590263307095,
      "logps/chosen": -402.467529296875,
      "logps/rejected": -310.58770751953125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.673064649105072,
      "rewards/margins": 0.17295120656490326,
      "rewards/rejected": -0.8460158109664917,
      "step": 7193
    },
    {
      "epoch": 0.9413920013085794,
      "grad_norm": 6.072989463806152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20606735348701477,
      "logits/rejected": -0.15359057486057281,
      "logps/chosen": -472.6562805175781,
      "logps/rejected": -455.3285827636719,
      "loss": 0.4004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5596866607666016,
      "rewards/margins": 1.1885368824005127,
      "rewards/rejected": -0.6288502216339111,
      "step": 7194
    },
    {
      "epoch": 0.9415228592459312,
      "grad_norm": 6.65477180480957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04647869989275932,
      "logits/rejected": -0.2642133831977844,
      "logps/chosen": -381.40386962890625,
      "logps/rejected": -321.8758850097656,
      "loss": 0.5559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16685998439788818,
      "rewards/margins": 0.8141939640045166,
      "rewards/rejected": -0.9810539484024048,
      "step": 7195
    },
    {
      "epoch": 0.941653717183283,
      "grad_norm": 6.0382890701293945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3037976026535034,
      "logits/rejected": 0.0946255549788475,
      "logps/chosen": -419.3741455078125,
      "logps/rejected": -425.62603759765625,
      "loss": 0.4564,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1599227637052536,
      "rewards/margins": 1.2296355962753296,
      "rewards/rejected": -1.0697128772735596,
      "step": 7196
    },
    {
      "epoch": 0.9417845751206346,
      "grad_norm": 5.850170612335205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3461853265762329,
      "logits/rejected": -0.14909020066261292,
      "logps/chosen": -505.1696472167969,
      "logps/rejected": -395.1064758300781,
      "loss": 0.4254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5168180465698242,
      "rewards/margins": 0.9217254519462585,
      "rewards/rejected": -1.4385435581207275,
      "step": 7197
    },
    {
      "epoch": 0.9419154330579864,
      "grad_norm": 13.690770149230957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3642936944961548,
      "logits/rejected": -0.27646011114120483,
      "logps/chosen": -485.4331970214844,
      "logps/rejected": -421.0522155761719,
      "loss": 1.365,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.232386827468872,
      "rewards/margins": -0.622836709022522,
      "rewards/rejected": -0.6095501184463501,
      "step": 7198
    },
    {
      "epoch": 0.9420462909953382,
      "grad_norm": 4.58486270904541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.560705840587616,
      "logits/rejected": 0.28974464535713196,
      "logps/chosen": -591.0836791992188,
      "logps/rejected": -427.53936767578125,
      "loss": 0.2969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24148502945899963,
      "rewards/margins": 1.3817176818847656,
      "rewards/rejected": -1.6232028007507324,
      "step": 7199
    },
    {
      "epoch": 0.94217714893269,
      "grad_norm": 10.736387252807617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11485226452350616,
      "logits/rejected": 0.05682198703289032,
      "logps/chosen": -459.6322326660156,
      "logps/rejected": -432.8792724609375,
      "loss": 0.5385,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0866609662771225,
      "rewards/margins": 0.8937848806381226,
      "rewards/rejected": -0.9804458618164062,
      "step": 7200
    },
    {
      "epoch": 0.9423080068700417,
      "grad_norm": 4.809279441833496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016146382316946983,
      "logits/rejected": -0.28453537821769714,
      "logps/chosen": -477.50286865234375,
      "logps/rejected": -376.6017150878906,
      "loss": 0.3021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.00363805890083313,
      "rewards/margins": 1.6787086725234985,
      "rewards/rejected": -1.6823468208312988,
      "step": 7201
    },
    {
      "epoch": 0.9424388648073935,
      "grad_norm": 9.805631637573242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08484923839569092,
      "logits/rejected": -0.20705628395080566,
      "logps/chosen": -571.1680908203125,
      "logps/rejected": -398.2922668457031,
      "loss": 0.8392,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4008983373641968,
      "rewards/margins": 0.14393043518066406,
      "rewards/rejected": -0.5448288321495056,
      "step": 7202
    },
    {
      "epoch": 0.9425697227447453,
      "grad_norm": 3.473606824874878,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13773953914642334,
      "logits/rejected": 0.01082523912191391,
      "logps/chosen": -548.1292114257812,
      "logps/rejected": -388.52203369140625,
      "loss": 0.2566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8612245321273804,
      "rewards/margins": 1.4696985483169556,
      "rewards/rejected": -0.6084739565849304,
      "step": 7203
    },
    {
      "epoch": 0.942700580682097,
      "grad_norm": 7.056812286376953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1466819941997528,
      "logits/rejected": 0.12940610945224762,
      "logps/chosen": -337.5491943359375,
      "logps/rejected": -316.0135192871094,
      "loss": 0.5492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2763591706752777,
      "rewards/margins": 0.6337687969207764,
      "rewards/rejected": -0.9101280570030212,
      "step": 7204
    },
    {
      "epoch": 0.9428314386194487,
      "grad_norm": 7.416533946990967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1660999059677124,
      "logits/rejected": 0.15932990610599518,
      "logps/chosen": -548.860595703125,
      "logps/rejected": -533.8731079101562,
      "loss": 0.518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.645861029624939,
      "rewards/margins": 0.8196457624435425,
      "rewards/rejected": -0.17378465831279755,
      "step": 7205
    },
    {
      "epoch": 0.9429622965568005,
      "grad_norm": 3.7804176807403564,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0884065255522728,
      "logits/rejected": -0.010111082345247269,
      "logps/chosen": -422.5106201171875,
      "logps/rejected": -380.1434631347656,
      "loss": 0.1951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3567352592945099,
      "rewards/margins": 1.8426711559295654,
      "rewards/rejected": -1.4859360456466675,
      "step": 7206
    },
    {
      "epoch": 0.9430931544941523,
      "grad_norm": 7.827816009521484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09683181345462799,
      "logits/rejected": 0.07724115252494812,
      "logps/chosen": -434.6921081542969,
      "logps/rejected": -559.7523803710938,
      "loss": 0.4737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49291279911994934,
      "rewards/margins": 1.1841025352478027,
      "rewards/rejected": -1.6770150661468506,
      "step": 7207
    },
    {
      "epoch": 0.943224012431504,
      "grad_norm": 7.929332256317139,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10966688394546509,
      "logits/rejected": -0.06783746182918549,
      "logps/chosen": -518.1705322265625,
      "logps/rejected": -426.95172119140625,
      "loss": 0.596,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2862062454223633,
      "rewards/margins": 0.5454590320587158,
      "rewards/rejected": -0.25925272703170776,
      "step": 7208
    },
    {
      "epoch": 0.9433548703688558,
      "grad_norm": 5.798507213592529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1768704354763031,
      "logits/rejected": 0.0395745187997818,
      "logps/chosen": -637.931884765625,
      "logps/rejected": -549.8522338867188,
      "loss": 0.3399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4181564450263977,
      "rewards/margins": 1.6006031036376953,
      "rewards/rejected": -1.182446837425232,
      "step": 7209
    },
    {
      "epoch": 0.9434857283062076,
      "grad_norm": 11.739150047302246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07201636582612991,
      "logits/rejected": -0.2667996883392334,
      "logps/chosen": -500.12615966796875,
      "logps/rejected": -371.01788330078125,
      "loss": 0.7886,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7568504214286804,
      "rewards/margins": 0.41963446140289307,
      "rewards/rejected": -1.1764849424362183,
      "step": 7210
    },
    {
      "epoch": 0.9436165862435594,
      "grad_norm": 5.525840759277344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0441841185092926,
      "logits/rejected": 0.12439973652362823,
      "logps/chosen": -223.84161376953125,
      "logps/rejected": -517.9789428710938,
      "loss": 0.2913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.020963072776794434,
      "rewards/margins": 1.5408177375793457,
      "rewards/rejected": -1.5198546648025513,
      "step": 7211
    },
    {
      "epoch": 0.943747444180911,
      "grad_norm": 4.286763668060303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12110154330730438,
      "logits/rejected": -0.30009782314300537,
      "logps/chosen": -662.9447021484375,
      "logps/rejected": -587.879150390625,
      "loss": 0.2619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.495566189289093,
      "rewards/margins": 1.6037553548812866,
      "rewards/rejected": -1.1081892251968384,
      "step": 7212
    },
    {
      "epoch": 0.9438783021182628,
      "grad_norm": 6.511021614074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013005342334508896,
      "logits/rejected": -0.08188384026288986,
      "logps/chosen": -342.4864196777344,
      "logps/rejected": -355.0244445800781,
      "loss": 0.4058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30936354398727417,
      "rewards/margins": 1.1666581630706787,
      "rewards/rejected": -1.4760217666625977,
      "step": 7213
    },
    {
      "epoch": 0.9440091600556146,
      "grad_norm": 7.988498210906982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21395176649093628,
      "logits/rejected": -0.23010113835334778,
      "logps/chosen": -421.85943603515625,
      "logps/rejected": -419.11981201171875,
      "loss": 0.6076,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7681068181991577,
      "rewards/margins": 0.5036963224411011,
      "rewards/rejected": -1.2718031406402588,
      "step": 7214
    },
    {
      "epoch": 0.9441400179929664,
      "grad_norm": 10.261144638061523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06669960916042328,
      "logits/rejected": 0.12459015846252441,
      "logps/chosen": -550.5951538085938,
      "logps/rejected": -458.0810546875,
      "loss": 0.6762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8062109351158142,
      "rewards/margins": 0.7878286838531494,
      "rewards/rejected": -1.5940395593643188,
      "step": 7215
    },
    {
      "epoch": 0.9442708759303181,
      "grad_norm": 5.674532413482666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33926689624786377,
      "logits/rejected": -0.43677282333374023,
      "logps/chosen": -852.712158203125,
      "logps/rejected": -492.5393371582031,
      "loss": 0.3108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2598237991333008,
      "rewards/margins": 1.2045913934707642,
      "rewards/rejected": -0.9447676539421082,
      "step": 7216
    },
    {
      "epoch": 0.9444017338676699,
      "grad_norm": 8.047845840454102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2230893075466156,
      "logits/rejected": 0.21921277046203613,
      "logps/chosen": -455.0028991699219,
      "logps/rejected": -498.6458435058594,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4732549786567688,
      "rewards/margins": 0.3630283772945404,
      "rewards/rejected": 0.11022663861513138,
      "step": 7217
    },
    {
      "epoch": 0.9445325918050217,
      "grad_norm": 7.814116954803467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26146119832992554,
      "logits/rejected": 0.12315277755260468,
      "logps/chosen": -617.3804931640625,
      "logps/rejected": -719.14697265625,
      "loss": 0.4663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27396389842033386,
      "rewards/margins": 1.0187749862670898,
      "rewards/rejected": -1.292738914489746,
      "step": 7218
    },
    {
      "epoch": 0.9446634497423735,
      "grad_norm": 5.074824333190918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031598955392837524,
      "logits/rejected": -0.016341138631105423,
      "logps/chosen": -366.09503173828125,
      "logps/rejected": -428.40081787109375,
      "loss": 0.3558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5512245297431946,
      "rewards/margins": 1.6057733297348022,
      "rewards/rejected": -1.054548740386963,
      "step": 7219
    },
    {
      "epoch": 0.9447943076797252,
      "grad_norm": 7.909444808959961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17045386135578156,
      "logits/rejected": -0.15958572924137115,
      "logps/chosen": -357.822998046875,
      "logps/rejected": -323.27984619140625,
      "loss": 0.5498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4713630676269531,
      "rewards/margins": 1.0306957960128784,
      "rewards/rejected": -1.502058744430542,
      "step": 7220
    },
    {
      "epoch": 0.944925165617077,
      "grad_norm": 8.250089645385742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01781647466123104,
      "logits/rejected": 0.26085424423217773,
      "logps/chosen": -259.75262451171875,
      "logps/rejected": -337.94573974609375,
      "loss": 0.6243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20853197574615479,
      "rewards/margins": 0.38799113035202026,
      "rewards/rejected": -0.5965230464935303,
      "step": 7221
    },
    {
      "epoch": 0.9450560235544287,
      "grad_norm": 13.637527465820312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1301531046628952,
      "logits/rejected": -0.10570371896028519,
      "logps/chosen": -845.2061767578125,
      "logps/rejected": -733.6376342773438,
      "loss": 0.9143,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.4069024920463562,
      "rewards/margins": 0.11537109315395355,
      "rewards/rejected": 0.29153138399124146,
      "step": 7222
    },
    {
      "epoch": 0.9451868814917805,
      "grad_norm": 6.165355205535889,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017448365688323975,
      "logits/rejected": -0.30799323320388794,
      "logps/chosen": -414.59246826171875,
      "logps/rejected": -394.4111328125,
      "loss": 0.489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11918222904205322,
      "rewards/margins": 0.9175451993942261,
      "rewards/rejected": -1.0367273092269897,
      "step": 7223
    },
    {
      "epoch": 0.9453177394291322,
      "grad_norm": 6.9501142501831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12402903288602829,
      "logits/rejected": -0.046369779855012894,
      "logps/chosen": -537.8236083984375,
      "logps/rejected": -592.2953491210938,
      "loss": 0.3813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5128423571586609,
      "rewards/margins": 1.3046164512634277,
      "rewards/rejected": -1.8174591064453125,
      "step": 7224
    },
    {
      "epoch": 0.945448597366484,
      "grad_norm": 8.404215812683105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06826746463775635,
      "logits/rejected": -0.14202037453651428,
      "logps/chosen": -286.0301208496094,
      "logps/rejected": -359.6105041503906,
      "loss": 0.7763,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03339263051748276,
      "rewards/margins": 0.19297632575035095,
      "rewards/rejected": -0.1595836877822876,
      "step": 7225
    },
    {
      "epoch": 0.9455794553038358,
      "grad_norm": 8.59824275970459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05045943707227707,
      "logits/rejected": -0.09183575212955475,
      "logps/chosen": -695.1156005859375,
      "logps/rejected": -543.51953125,
      "loss": 0.6328,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.3389242887496948,
      "rewards/margins": 0.7830507755279541,
      "rewards/rejected": 0.5558733344078064,
      "step": 7226
    },
    {
      "epoch": 0.9457103132411875,
      "grad_norm": 7.399200439453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022551747038960457,
      "logits/rejected": 0.023869115859270096,
      "logps/chosen": -546.4845581054688,
      "logps/rejected": -435.8295593261719,
      "loss": 0.5765,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04615534842014313,
      "rewards/margins": 0.7020065784454346,
      "rewards/rejected": -0.7481619119644165,
      "step": 7227
    },
    {
      "epoch": 0.9458411711785393,
      "grad_norm": 6.441527843475342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07860608398914337,
      "logits/rejected": 0.2569311857223511,
      "logps/chosen": -437.7414855957031,
      "logps/rejected": -425.2701110839844,
      "loss": 0.4329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6280614733695984,
      "rewards/margins": 0.9186609387397766,
      "rewards/rejected": -1.546722412109375,
      "step": 7228
    },
    {
      "epoch": 0.9459720291158911,
      "grad_norm": 6.508197784423828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09370473027229309,
      "logits/rejected": 0.11065812408924103,
      "logps/chosen": -533.1674194335938,
      "logps/rejected": -482.06512451171875,
      "loss": 0.4036,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11658401042222977,
      "rewards/margins": 0.9792590737342834,
      "rewards/rejected": -0.8626751899719238,
      "step": 7229
    },
    {
      "epoch": 0.9461028870532429,
      "grad_norm": 8.054923057556152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13549813628196716,
      "logits/rejected": 0.14065925776958466,
      "logps/chosen": -457.5742492675781,
      "logps/rejected": -524.031982421875,
      "loss": 0.6064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3938736021518707,
      "rewards/margins": 0.46690064668655396,
      "rewards/rejected": -0.8607742786407471,
      "step": 7230
    },
    {
      "epoch": 0.9462337449905945,
      "grad_norm": 9.355792045593262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03133813291788101,
      "logits/rejected": 0.07649105787277222,
      "logps/chosen": -432.6375427246094,
      "logps/rejected": -462.47381591796875,
      "loss": 0.5596,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5357971787452698,
      "rewards/margins": 0.8092865943908691,
      "rewards/rejected": -1.3450837135314941,
      "step": 7231
    },
    {
      "epoch": 0.9463646029279463,
      "grad_norm": 7.0297956466674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18518370389938354,
      "logits/rejected": -0.23291313648223877,
      "logps/chosen": -311.87060546875,
      "logps/rejected": -388.1833190917969,
      "loss": 0.4748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8936065435409546,
      "rewards/margins": 1.2921383380889893,
      "rewards/rejected": -2.1857450008392334,
      "step": 7232
    },
    {
      "epoch": 0.9464954608652981,
      "grad_norm": 6.381115913391113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1656491458415985,
      "logits/rejected": 0.10523710399866104,
      "logps/chosen": -349.7704772949219,
      "logps/rejected": -380.2019348144531,
      "loss": 0.4915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3546791076660156,
      "rewards/margins": 1.3396990299224854,
      "rewards/rejected": -1.6943782567977905,
      "step": 7233
    },
    {
      "epoch": 0.9466263188026499,
      "grad_norm": 7.626072406768799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25074303150177,
      "logits/rejected": 0.5507474541664124,
      "logps/chosen": -545.018798828125,
      "logps/rejected": -604.06103515625,
      "loss": 0.462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06967145204544067,
      "rewards/margins": 1.7775434255599976,
      "rewards/rejected": -1.707871913909912,
      "step": 7234
    },
    {
      "epoch": 0.9467571767400016,
      "grad_norm": 4.796847343444824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0968550592660904,
      "logits/rejected": -0.027435213327407837,
      "logps/chosen": -588.5252685546875,
      "logps/rejected": -626.0704345703125,
      "loss": 0.2778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.45698392391204834,
      "rewards/margins": 2.1098716259002686,
      "rewards/rejected": -1.6528875827789307,
      "step": 7235
    },
    {
      "epoch": 0.9468880346773534,
      "grad_norm": 3.48392653465271,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3872593939304352,
      "logits/rejected": -0.10812927782535553,
      "logps/chosen": -360.6853942871094,
      "logps/rejected": -521.309814453125,
      "loss": 0.1375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13941174745559692,
      "rewards/margins": 2.5211222171783447,
      "rewards/rejected": -2.660533905029297,
      "step": 7236
    },
    {
      "epoch": 0.9470188926147052,
      "grad_norm": 5.462355613708496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1809324026107788,
      "logits/rejected": -0.24264201521873474,
      "logps/chosen": -386.25244140625,
      "logps/rejected": -332.257568359375,
      "loss": 0.4417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38035711646080017,
      "rewards/margins": 0.9604414701461792,
      "rewards/rejected": -1.3407984972000122,
      "step": 7237
    },
    {
      "epoch": 0.947149750552057,
      "grad_norm": 6.940077781677246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022620443254709244,
      "logits/rejected": -0.2589241564273834,
      "logps/chosen": -327.5221862792969,
      "logps/rejected": -496.367431640625,
      "loss": 0.6055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1825132668018341,
      "rewards/margins": 0.7557613849639893,
      "rewards/rejected": -0.938274621963501,
      "step": 7238
    },
    {
      "epoch": 0.9472806084894086,
      "grad_norm": 6.609989643096924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21752668917179108,
      "logits/rejected": 0.11553073674440384,
      "logps/chosen": -465.5290832519531,
      "logps/rejected": -386.2881774902344,
      "loss": 0.4401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22274149954319,
      "rewards/margins": 1.1165779829025269,
      "rewards/rejected": -1.3393194675445557,
      "step": 7239
    },
    {
      "epoch": 0.9474114664267604,
      "grad_norm": 9.402017593383789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14563749730587006,
      "logits/rejected": -0.01641259342432022,
      "logps/chosen": -261.75335693359375,
      "logps/rejected": -380.6426086425781,
      "loss": 0.8309,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5876811742782593,
      "rewards/margins": 0.3853931725025177,
      "rewards/rejected": -0.9730743169784546,
      "step": 7240
    },
    {
      "epoch": 0.9475423243641122,
      "grad_norm": 4.387152671813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18432112038135529,
      "logits/rejected": -0.09513571113348007,
      "logps/chosen": -332.71636962890625,
      "logps/rejected": -336.8462829589844,
      "loss": 0.2404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4427654445171356,
      "rewards/margins": 2.096774101257324,
      "rewards/rejected": -1.6540086269378662,
      "step": 7241
    },
    {
      "epoch": 0.947673182301464,
      "grad_norm": 7.1243205070495605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032737791538238525,
      "logits/rejected": -0.36044323444366455,
      "logps/chosen": -640.3081665039062,
      "logps/rejected": -587.1013793945312,
      "loss": 0.4155,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.025775521993637085,
      "rewards/margins": 1.2234331369400024,
      "rewards/rejected": -1.1976577043533325,
      "step": 7242
    },
    {
      "epoch": 0.9478040402388157,
      "grad_norm": 7.767559051513672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08287029713392258,
      "logits/rejected": 0.058576494455337524,
      "logps/chosen": -399.38616943359375,
      "logps/rejected": -355.06744384765625,
      "loss": 0.7295,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38944950699806213,
      "rewards/margins": 0.31200483441352844,
      "rewards/rejected": -0.7014543414115906,
      "step": 7243
    },
    {
      "epoch": 0.9479348981761675,
      "grad_norm": 7.911339282989502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21172544360160828,
      "logits/rejected": -0.3773230314254761,
      "logps/chosen": -391.7347412109375,
      "logps/rejected": -548.7241821289062,
      "loss": 0.3782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28943395614624023,
      "rewards/margins": 1.6211377382278442,
      "rewards/rejected": -1.3317039012908936,
      "step": 7244
    },
    {
      "epoch": 0.9480657561135193,
      "grad_norm": 7.987666130065918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06773555278778076,
      "logits/rejected": -0.0593687929213047,
      "logps/chosen": -541.5270385742188,
      "logps/rejected": -650.4762573242188,
      "loss": 0.4769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21406292915344238,
      "rewards/margins": 1.569798469543457,
      "rewards/rejected": -1.3557355403900146,
      "step": 7245
    },
    {
      "epoch": 0.9481966140508711,
      "grad_norm": 9.090570449829102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14463892579078674,
      "logits/rejected": -0.012866164557635784,
      "logps/chosen": -677.8964233398438,
      "logps/rejected": -533.1183471679688,
      "loss": 0.5186,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4523754119873047,
      "rewards/margins": 1.2000266313552856,
      "rewards/rejected": -1.6524021625518799,
      "step": 7246
    },
    {
      "epoch": 0.9483274719882228,
      "grad_norm": 9.672630310058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1725257933139801,
      "logits/rejected": -0.16811415553092957,
      "logps/chosen": -267.96319580078125,
      "logps/rejected": -381.32220458984375,
      "loss": 0.8375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5282203555107117,
      "rewards/margins": -0.0966196209192276,
      "rewards/rejected": -0.43160074949264526,
      "step": 7247
    },
    {
      "epoch": 0.9484583299255746,
      "grad_norm": 8.253745079040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06310433149337769,
      "logits/rejected": -0.010992318391799927,
      "logps/chosen": -492.2832336425781,
      "logps/rejected": -467.22723388671875,
      "loss": 0.4734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7623388171195984,
      "rewards/margins": 0.681962788105011,
      "rewards/rejected": -1.4443016052246094,
      "step": 7248
    },
    {
      "epoch": 0.9485891878629263,
      "grad_norm": 5.5552544593811035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12569767236709595,
      "logits/rejected": 0.22448264062404633,
      "logps/chosen": -440.39544677734375,
      "logps/rejected": -451.8338623046875,
      "loss": 0.3485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29433783888816833,
      "rewards/margins": 1.681867003440857,
      "rewards/rejected": -1.3875292539596558,
      "step": 7249
    },
    {
      "epoch": 0.948720045800278,
      "grad_norm": 9.098698616027832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029800735414028168,
      "logits/rejected": 0.24816776812076569,
      "logps/chosen": -319.0322570800781,
      "logps/rejected": -307.2904968261719,
      "loss": 0.6744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6958461999893188,
      "rewards/margins": 0.3684539496898651,
      "rewards/rejected": -1.0643001794815063,
      "step": 7250
    },
    {
      "epoch": 0.9488509037376298,
      "grad_norm": 7.9344048500061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2047719657421112,
      "logits/rejected": -0.43965843319892883,
      "logps/chosen": -584.2327270507812,
      "logps/rejected": -447.2825927734375,
      "loss": 0.3939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4314045011997223,
      "rewards/margins": 1.1142151355743408,
      "rewards/rejected": -1.5456194877624512,
      "step": 7251
    },
    {
      "epoch": 0.9489817616749816,
      "grad_norm": 4.894130706787109,
      "learning_rate": 5e-06,
      "logits/chosen": 0.050915662199258804,
      "logits/rejected": 0.035069212317466736,
      "logps/chosen": -772.2949829101562,
      "logps/rejected": -582.9166259765625,
      "loss": 0.4777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0980970561504364,
      "rewards/margins": 1.1951560974121094,
      "rewards/rejected": -1.2932533025741577,
      "step": 7252
    },
    {
      "epoch": 0.9491126196123334,
      "grad_norm": 4.140394687652588,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12196042388677597,
      "logits/rejected": -0.09186865389347076,
      "logps/chosen": -437.29827880859375,
      "logps/rejected": -487.8291931152344,
      "loss": 0.362,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7184854745864868,
      "rewards/margins": 1.1965405941009521,
      "rewards/rejected": -1.9150257110595703,
      "step": 7253
    },
    {
      "epoch": 0.9492434775496851,
      "grad_norm": 10.451214790344238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18872633576393127,
      "logits/rejected": 0.039015032351017,
      "logps/chosen": -605.7982788085938,
      "logps/rejected": -493.08160400390625,
      "loss": 0.6058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6245531439781189,
      "rewards/margins": 1.0256917476654053,
      "rewards/rejected": -1.650244951248169,
      "step": 7254
    },
    {
      "epoch": 0.9493743354870369,
      "grad_norm": 5.133336067199707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16330480575561523,
      "logits/rejected": -0.028104014694690704,
      "logps/chosen": -346.3208923339844,
      "logps/rejected": -419.9115295410156,
      "loss": 0.3479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18061769008636475,
      "rewards/margins": 1.023944616317749,
      "rewards/rejected": -1.2045624256134033,
      "step": 7255
    },
    {
      "epoch": 0.9495051934243887,
      "grad_norm": 10.053610801696777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08336491882801056,
      "logits/rejected": -0.0507756769657135,
      "logps/chosen": -447.1873779296875,
      "logps/rejected": -477.71307373046875,
      "loss": 0.8081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.159641295671463,
      "rewards/margins": 0.3011825382709503,
      "rewards/rejected": -0.46082383394241333,
      "step": 7256
    },
    {
      "epoch": 0.9496360513617405,
      "grad_norm": 5.059830188751221,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23813149333000183,
      "logits/rejected": -0.29530712962150574,
      "logps/chosen": -521.483642578125,
      "logps/rejected": -427.5420227050781,
      "loss": 0.2646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4496891498565674,
      "rewards/margins": 1.5806260108947754,
      "rewards/rejected": -3.0303151607513428,
      "step": 7257
    },
    {
      "epoch": 0.9497669092990921,
      "grad_norm": 9.895057678222656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06268452107906342,
      "logits/rejected": 0.0021830126643180847,
      "logps/chosen": -507.17889404296875,
      "logps/rejected": -464.392822265625,
      "loss": 0.769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40913450717926025,
      "rewards/margins": 0.567990779876709,
      "rewards/rejected": -0.9771252870559692,
      "step": 7258
    },
    {
      "epoch": 0.9498977672364439,
      "grad_norm": 5.574705123901367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13247159123420715,
      "logits/rejected": 0.024613626301288605,
      "logps/chosen": -444.559326171875,
      "logps/rejected": -445.3835144042969,
      "loss": 0.3112,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3742979168891907,
      "rewards/margins": 2.2639715671539307,
      "rewards/rejected": -2.6382691860198975,
      "step": 7259
    },
    {
      "epoch": 0.9500286251737957,
      "grad_norm": 6.300955772399902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14872899651527405,
      "logits/rejected": -0.08702355623245239,
      "logps/chosen": -640.5369873046875,
      "logps/rejected": -588.369384765625,
      "loss": 0.3241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8139438629150391,
      "rewards/margins": 1.7459685802459717,
      "rewards/rejected": -0.9320247769355774,
      "step": 7260
    },
    {
      "epoch": 0.9501594831111475,
      "grad_norm": 4.592833042144775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3674813210964203,
      "logits/rejected": -0.3212202787399292,
      "logps/chosen": -237.80502319335938,
      "logps/rejected": -259.0434875488281,
      "loss": 0.4051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7233499884605408,
      "rewards/margins": 1.1306042671203613,
      "rewards/rejected": -1.8539541959762573,
      "step": 7261
    },
    {
      "epoch": 0.9502903410484992,
      "grad_norm": 9.41391372680664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2519910931587219,
      "logits/rejected": -0.20418204367160797,
      "logps/chosen": -541.6135864257812,
      "logps/rejected": -590.58837890625,
      "loss": 0.6885,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.937211811542511,
      "rewards/margins": 0.5692859888076782,
      "rewards/rejected": -1.506497859954834,
      "step": 7262
    },
    {
      "epoch": 0.950421198985851,
      "grad_norm": 3.7676351070404053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025592666119337082,
      "logits/rejected": 0.20905916392803192,
      "logps/chosen": -435.11553955078125,
      "logps/rejected": -351.48272705078125,
      "loss": 0.2742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8962486386299133,
      "rewards/margins": 1.7746458053588867,
      "rewards/rejected": -0.8783969879150391,
      "step": 7263
    },
    {
      "epoch": 0.9505520569232028,
      "grad_norm": 17.396120071411133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30745601654052734,
      "logits/rejected": -0.32360997796058655,
      "logps/chosen": -430.2791748046875,
      "logps/rejected": -289.89349365234375,
      "loss": 1.2084,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.5340898036956787,
      "rewards/margins": -0.23689517378807068,
      "rewards/rejected": -1.297194480895996,
      "step": 7264
    },
    {
      "epoch": 0.9506829148605546,
      "grad_norm": 6.749655246734619,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010407492518424988,
      "logits/rejected": -0.03379905968904495,
      "logps/chosen": -473.8310852050781,
      "logps/rejected": -368.65716552734375,
      "loss": 0.4492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1532648354768753,
      "rewards/margins": 0.8636896014213562,
      "rewards/rejected": -1.0169544219970703,
      "step": 7265
    },
    {
      "epoch": 0.9508137727979062,
      "grad_norm": 9.455322265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5470807552337646,
      "logits/rejected": -0.6479825973510742,
      "logps/chosen": -452.9169921875,
      "logps/rejected": -407.6800537109375,
      "loss": 0.5952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7912831902503967,
      "rewards/margins": 1.3267972469329834,
      "rewards/rejected": -2.1180806159973145,
      "step": 7266
    },
    {
      "epoch": 0.950944630735258,
      "grad_norm": 9.496796607971191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.038137830793857574,
      "logits/rejected": -0.2236444056034088,
      "logps/chosen": -622.8060913085938,
      "logps/rejected": -605.6754150390625,
      "loss": 0.6231,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024122431874275208,
      "rewards/margins": 0.8929779529571533,
      "rewards/rejected": -0.9171003103256226,
      "step": 7267
    },
    {
      "epoch": 0.9510754886726098,
      "grad_norm": 6.991641998291016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25272926688194275,
      "logits/rejected": -0.016415996477007866,
      "logps/chosen": -494.7764892578125,
      "logps/rejected": -686.4378662109375,
      "loss": 0.2878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06225910037755966,
      "rewards/margins": 2.422612190246582,
      "rewards/rejected": -2.3603532314300537,
      "step": 7268
    },
    {
      "epoch": 0.9512063466099615,
      "grad_norm": 7.729462623596191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22926174104213715,
      "logits/rejected": 0.2810596823692322,
      "logps/chosen": -529.9840087890625,
      "logps/rejected": -385.4077453613281,
      "loss": 0.4284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.549107551574707,
      "rewards/margins": 1.1776975393295288,
      "rewards/rejected": -1.7268050909042358,
      "step": 7269
    },
    {
      "epoch": 0.9513372045473133,
      "grad_norm": 18.484962463378906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5744439363479614,
      "logits/rejected": -0.37232157588005066,
      "logps/chosen": -554.453857421875,
      "logps/rejected": -474.6375732421875,
      "loss": 1.537,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.036360740661621,
      "rewards/margins": -0.8455953598022461,
      "rewards/rejected": -0.1907653957605362,
      "step": 7270
    },
    {
      "epoch": 0.9514680624846651,
      "grad_norm": 10.824173927307129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03275465965270996,
      "logits/rejected": 0.04703206568956375,
      "logps/chosen": -539.2162475585938,
      "logps/rejected": -523.2177124023438,
      "loss": 0.6875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3699720799922943,
      "rewards/margins": 1.200932502746582,
      "rewards/rejected": -1.5709046125411987,
      "step": 7271
    },
    {
      "epoch": 0.9515989204220169,
      "grad_norm": 7.72266149520874,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034954532980918884,
      "logits/rejected": -0.06987720727920532,
      "logps/chosen": -512.88232421875,
      "logps/rejected": -413.8896789550781,
      "loss": 0.6075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3769569396972656,
      "rewards/margins": 1.1319091320037842,
      "rewards/rejected": -1.5088659524917603,
      "step": 7272
    },
    {
      "epoch": 0.9517297783593686,
      "grad_norm": 7.724076747894287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04401716589927673,
      "logits/rejected": 0.13813702762126923,
      "logps/chosen": -345.5190124511719,
      "logps/rejected": -412.30157470703125,
      "loss": 0.6416,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2874872088432312,
      "rewards/margins": 0.7407248020172119,
      "rewards/rejected": -0.4532375633716583,
      "step": 7273
    },
    {
      "epoch": 0.9518606362967204,
      "grad_norm": 7.667057991027832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.032944660633802414,
      "logits/rejected": -0.05562501773238182,
      "logps/chosen": -482.5850524902344,
      "logps/rejected": -616.2483520507812,
      "loss": 0.6123,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5129657983779907,
      "rewards/margins": 0.6448522210121155,
      "rewards/rejected": -1.157818078994751,
      "step": 7274
    },
    {
      "epoch": 0.9519914942340721,
      "grad_norm": 5.940891265869141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1554339975118637,
      "logits/rejected": -0.22083234786987305,
      "logps/chosen": -336.8890686035156,
      "logps/rejected": -395.1168518066406,
      "loss": 0.3364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0776572972536087,
      "rewards/margins": 1.6630022525787354,
      "rewards/rejected": -1.5853447914123535,
      "step": 7275
    },
    {
      "epoch": 0.9521223521714239,
      "grad_norm": 9.52499771118164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21821969747543335,
      "logits/rejected": -0.17432107031345367,
      "logps/chosen": -639.8909912109375,
      "logps/rejected": -471.09246826171875,
      "loss": 0.5618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04738923907279968,
      "rewards/margins": 1.413156509399414,
      "rewards/rejected": -1.4605457782745361,
      "step": 7276
    },
    {
      "epoch": 0.9522532101087756,
      "grad_norm": 5.495211601257324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2746579349040985,
      "logits/rejected": -0.21122567355632782,
      "logps/chosen": -321.6963195800781,
      "logps/rejected": -319.02978515625,
      "loss": 0.4976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.30701944231987,
      "rewards/margins": 0.9256477355957031,
      "rewards/rejected": -0.6186282634735107,
      "step": 7277
    },
    {
      "epoch": 0.9523840680461274,
      "grad_norm": 7.217264652252197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04047330468893051,
      "logits/rejected": -0.2119000107049942,
      "logps/chosen": -414.9118957519531,
      "logps/rejected": -346.7840270996094,
      "loss": 0.5603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06516589969396591,
      "rewards/margins": 0.8828644752502441,
      "rewards/rejected": -0.8176984786987305,
      "step": 7278
    },
    {
      "epoch": 0.9525149259834792,
      "grad_norm": 5.410076141357422,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029721610248088837,
      "logits/rejected": -0.03389116749167442,
      "logps/chosen": -380.9557800292969,
      "logps/rejected": -378.5345153808594,
      "loss": 0.4064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30398425459861755,
      "rewards/margins": 1.0496553182601929,
      "rewards/rejected": -0.7456710934638977,
      "step": 7279
    },
    {
      "epoch": 0.952645783920831,
      "grad_norm": 9.239121437072754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2244304120540619,
      "logits/rejected": -0.26249101758003235,
      "logps/chosen": -398.548828125,
      "logps/rejected": -385.48223876953125,
      "loss": 0.6476,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19513477385044098,
      "rewards/margins": 0.7548031806945801,
      "rewards/rejected": -0.9499379992485046,
      "step": 7280
    },
    {
      "epoch": 0.9527766418581827,
      "grad_norm": 5.862460613250732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19065812230110168,
      "logits/rejected": -0.14444896578788757,
      "logps/chosen": -292.32904052734375,
      "logps/rejected": -329.0830383300781,
      "loss": 0.454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1051025465130806,
      "rewards/margins": 1.1987451314926147,
      "rewards/rejected": -1.0936427116394043,
      "step": 7281
    },
    {
      "epoch": 0.9529074997955345,
      "grad_norm": 6.269593715667725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09321410208940506,
      "logits/rejected": -0.11593897640705109,
      "logps/chosen": -321.8435974121094,
      "logps/rejected": -321.576904296875,
      "loss": 0.4992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004971891641616821,
      "rewards/margins": 0.9831440448760986,
      "rewards/rejected": -0.9881159067153931,
      "step": 7282
    },
    {
      "epoch": 0.9530383577328863,
      "grad_norm": 11.782820701599121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039212025701999664,
      "logits/rejected": 0.17428773641586304,
      "logps/chosen": -384.8124084472656,
      "logps/rejected": -334.92449951171875,
      "loss": 0.9826,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8982421159744263,
      "rewards/margins": -0.2425745129585266,
      "rewards/rejected": -0.6556674838066101,
      "step": 7283
    },
    {
      "epoch": 0.953169215670238,
      "grad_norm": 6.74468994140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2679787874221802,
      "logits/rejected": 0.024808770045638084,
      "logps/chosen": -711.3404541015625,
      "logps/rejected": -439.2934875488281,
      "loss": 0.587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02970503270626068,
      "rewards/margins": 0.793290376663208,
      "rewards/rejected": -0.8229954242706299,
      "step": 7284
    },
    {
      "epoch": 0.9533000736075897,
      "grad_norm": 10.431441307067871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.530093252658844,
      "logits/rejected": -0.41464486718177795,
      "logps/chosen": -414.33306884765625,
      "logps/rejected": -570.0374145507812,
      "loss": 0.7605,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.35881999135017395,
      "rewards/margins": 0.0760696679353714,
      "rewards/rejected": -0.4348895847797394,
      "step": 7285
    },
    {
      "epoch": 0.9534309315449415,
      "grad_norm": 8.13000774383545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017770394682884216,
      "logits/rejected": 0.052727196365594864,
      "logps/chosen": -446.9598388671875,
      "logps/rejected": -416.84332275390625,
      "loss": 0.6457,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.013688832521438599,
      "rewards/margins": 0.3688848614692688,
      "rewards/rejected": -0.382573664188385,
      "step": 7286
    },
    {
      "epoch": 0.9535617894822933,
      "grad_norm": 6.8659491539001465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0420365035533905,
      "logits/rejected": 0.18111349642276764,
      "logps/chosen": -517.8633422851562,
      "logps/rejected": -494.3623046875,
      "loss": 0.5274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2138267308473587,
      "rewards/margins": 0.886305034160614,
      "rewards/rejected": -1.100131869316101,
      "step": 7287
    },
    {
      "epoch": 0.9536926474196451,
      "grad_norm": 5.311565399169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5950483083724976,
      "logits/rejected": -0.30964621901512146,
      "logps/chosen": -504.2989807128906,
      "logps/rejected": -564.7020263671875,
      "loss": 0.2312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5172528028488159,
      "rewards/margins": 1.8407498598098755,
      "rewards/rejected": -2.3580026626586914,
      "step": 7288
    },
    {
      "epoch": 0.9538235053569968,
      "grad_norm": 8.200032234191895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12415158003568649,
      "logits/rejected": -0.08560221642255783,
      "logps/chosen": -401.5714111328125,
      "logps/rejected": -417.3586120605469,
      "loss": 0.6309,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8458048105239868,
      "rewards/margins": 0.7129091620445251,
      "rewards/rejected": -1.5587139129638672,
      "step": 7289
    },
    {
      "epoch": 0.9539543632943486,
      "grad_norm": 7.003036022186279,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08702536672353745,
      "logits/rejected": 0.33641475439071655,
      "logps/chosen": -419.1858825683594,
      "logps/rejected": -568.6669311523438,
      "loss": 0.4593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4812137484550476,
      "rewards/margins": 1.8805925846099854,
      "rewards/rejected": -2.361806631088257,
      "step": 7290
    },
    {
      "epoch": 0.9540852212317004,
      "grad_norm": 13.446203231811523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15234887599945068,
      "logits/rejected": -0.4136001169681549,
      "logps/chosen": -533.4772338867188,
      "logps/rejected": -451.42559814453125,
      "loss": 0.5218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7196704745292664,
      "rewards/margins": 0.9890501499176025,
      "rewards/rejected": -1.7087206840515137,
      "step": 7291
    },
    {
      "epoch": 0.954216079169052,
      "grad_norm": 5.295931816101074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03177550062537193,
      "logits/rejected": -0.0013284292072057724,
      "logps/chosen": -612.5435180664062,
      "logps/rejected": -637.9443359375,
      "loss": 0.3022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7307697534561157,
      "rewards/margins": 1.5137858390808105,
      "rewards/rejected": -0.78301602602005,
      "step": 7292
    },
    {
      "epoch": 0.9543469371064038,
      "grad_norm": 11.90958023071289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026582660153508186,
      "logits/rejected": 0.022585052996873856,
      "logps/chosen": -579.3572387695312,
      "logps/rejected": -578.5947265625,
      "loss": 1.0845,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.058354537934064865,
      "rewards/margins": 0.23973369598388672,
      "rewards/rejected": -0.18137910962104797,
      "step": 7293
    },
    {
      "epoch": 0.9544777950437556,
      "grad_norm": 5.431685447692871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05843700096011162,
      "logits/rejected": -0.13402104377746582,
      "logps/chosen": -468.8351745605469,
      "logps/rejected": -425.546142578125,
      "loss": 0.3006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12601983547210693,
      "rewards/margins": 1.5923218727111816,
      "rewards/rejected": -1.4663020372390747,
      "step": 7294
    },
    {
      "epoch": 0.9546086529811074,
      "grad_norm": 10.130983352661133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12706011533737183,
      "logits/rejected": -0.24017921090126038,
      "logps/chosen": -637.7098999023438,
      "logps/rejected": -539.3334350585938,
      "loss": 0.8733,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2925049066543579,
      "rewards/margins": 0.1324455440044403,
      "rewards/rejected": -0.4249504506587982,
      "step": 7295
    },
    {
      "epoch": 0.9547395109184591,
      "grad_norm": 10.004117965698242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.045319002121686935,
      "logits/rejected": 0.03411579132080078,
      "logps/chosen": -421.9538269042969,
      "logps/rejected": -444.8016357421875,
      "loss": 0.7206,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7907697558403015,
      "rewards/margins": 1.5266169309616089,
      "rewards/rejected": -2.3173866271972656,
      "step": 7296
    },
    {
      "epoch": 0.9548703688558109,
      "grad_norm": 8.509810447692871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010785244405269623,
      "logits/rejected": 0.06865106523036957,
      "logps/chosen": -371.560302734375,
      "logps/rejected": -312.13287353515625,
      "loss": 0.805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2258535474538803,
      "rewards/margins": 0.5152395367622375,
      "rewards/rejected": -0.7410930395126343,
      "step": 7297
    },
    {
      "epoch": 0.9550012267931627,
      "grad_norm": 6.367170810699463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07037923485040665,
      "logits/rejected": -0.3072066605091095,
      "logps/chosen": -321.073974609375,
      "logps/rejected": -293.8442687988281,
      "loss": 0.7308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10331477969884872,
      "rewards/margins": 0.4064502716064453,
      "rewards/rejected": -0.3031355142593384,
      "step": 7298
    },
    {
      "epoch": 0.9551320847305145,
      "grad_norm": 4.9277567863464355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058050718158483505,
      "logits/rejected": -0.21955420076847076,
      "logps/chosen": -406.20953369140625,
      "logps/rejected": -387.1263732910156,
      "loss": 0.344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0405103862285614,
      "rewards/margins": 1.9187095165252686,
      "rewards/rejected": -1.8781989812850952,
      "step": 7299
    },
    {
      "epoch": 0.9552629426678662,
      "grad_norm": 9.51259708404541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06782124936580658,
      "logits/rejected": 0.14112350344657898,
      "logps/chosen": -485.14788818359375,
      "logps/rejected": -521.823974609375,
      "loss": 0.7531,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0329341888427734,
      "rewards/margins": 0.29404711723327637,
      "rewards/rejected": -1.326981544494629,
      "step": 7300
    },
    {
      "epoch": 0.955393800605218,
      "grad_norm": 7.108832836151123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02790108323097229,
      "logits/rejected": -0.27850818634033203,
      "logps/chosen": -409.9618225097656,
      "logps/rejected": -489.59796142578125,
      "loss": 0.4979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.46881067752838135,
      "rewards/margins": 1.1173737049102783,
      "rewards/rejected": -0.648563027381897,
      "step": 7301
    },
    {
      "epoch": 0.9555246585425697,
      "grad_norm": 5.109039783477783,
      "learning_rate": 5e-06,
      "logits/chosen": -0.059954192489385605,
      "logits/rejected": 0.010312640108168125,
      "logps/chosen": -387.82647705078125,
      "logps/rejected": -366.65863037109375,
      "loss": 0.3651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33497393131256104,
      "rewards/margins": 1.354933261871338,
      "rewards/rejected": -1.0199592113494873,
      "step": 7302
    },
    {
      "epoch": 0.9556555164799215,
      "grad_norm": 6.237002372741699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14397795498371124,
      "logits/rejected": -0.37127751111984253,
      "logps/chosen": -475.4322509765625,
      "logps/rejected": -545.0630493164062,
      "loss": 0.4171,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13617555797100067,
      "rewards/margins": 1.0788135528564453,
      "rewards/rejected": -1.214989185333252,
      "step": 7303
    },
    {
      "epoch": 0.9557863744172732,
      "grad_norm": 5.783864498138428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05549521744251251,
      "logits/rejected": 0.014606133103370667,
      "logps/chosen": -445.5493469238281,
      "logps/rejected": -411.39190673828125,
      "loss": 0.4804,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08260516822338104,
      "rewards/margins": 0.8494815826416016,
      "rewards/rejected": -0.9320867657661438,
      "step": 7304
    },
    {
      "epoch": 0.955917232354625,
      "grad_norm": 3.889662027359009,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20639438927173615,
      "logits/rejected": 0.028875254094600677,
      "logps/chosen": -511.62847900390625,
      "logps/rejected": -405.8853454589844,
      "loss": 0.2623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9710172414779663,
      "rewards/margins": 1.7383291721343994,
      "rewards/rejected": -0.7673121094703674,
      "step": 7305
    },
    {
      "epoch": 0.9560480902919768,
      "grad_norm": 10.108670234680176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09925544261932373,
      "logits/rejected": -0.19679579138755798,
      "logps/chosen": -479.70587158203125,
      "logps/rejected": -609.06591796875,
      "loss": 0.6388,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6599991321563721,
      "rewards/margins": 0.3987300992012024,
      "rewards/rejected": -1.0587291717529297,
      "step": 7306
    },
    {
      "epoch": 0.9561789482293286,
      "grad_norm": 10.106189727783203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1232181116938591,
      "logits/rejected": -0.02248227596282959,
      "logps/chosen": -365.5069580078125,
      "logps/rejected": -437.9259033203125,
      "loss": 0.8798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2575027644634247,
      "rewards/margins": 0.21906523406505585,
      "rewards/rejected": -0.4765680432319641,
      "step": 7307
    },
    {
      "epoch": 0.9563098061666803,
      "grad_norm": 28.184101104736328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09368310123682022,
      "logits/rejected": -0.06977036595344543,
      "logps/chosen": -378.017822265625,
      "logps/rejected": -246.20590209960938,
      "loss": 0.6638,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4317096769809723,
      "rewards/margins": 0.6924667954444885,
      "rewards/rejected": -1.1241763830184937,
      "step": 7308
    },
    {
      "epoch": 0.9564406641040321,
      "grad_norm": 4.360159397125244,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08165988326072693,
      "logits/rejected": -0.15879151225090027,
      "logps/chosen": -521.3255004882812,
      "logps/rejected": -440.0225524902344,
      "loss": 0.2956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5173829197883606,
      "rewards/margins": 1.3427011966705322,
      "rewards/rejected": -1.860083818435669,
      "step": 7309
    },
    {
      "epoch": 0.9565715220413838,
      "grad_norm": 28.484514236450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22637100517749786,
      "logits/rejected": -0.24767839908599854,
      "logps/chosen": -423.69525146484375,
      "logps/rejected": -387.4937744140625,
      "loss": 1.0065,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8307017683982849,
      "rewards/margins": 0.047601550817489624,
      "rewards/rejected": -0.8783032298088074,
      "step": 7310
    },
    {
      "epoch": 0.9567023799787356,
      "grad_norm": 9.92674446105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07196968793869019,
      "logits/rejected": -0.05111680552363396,
      "logps/chosen": -484.9179382324219,
      "logps/rejected": -358.093017578125,
      "loss": 0.6297,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29066377878189087,
      "rewards/margins": 0.5740371346473694,
      "rewards/rejected": -0.8647008538246155,
      "step": 7311
    },
    {
      "epoch": 0.9568332379160873,
      "grad_norm": 3.7700135707855225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2318275421857834,
      "logits/rejected": 0.10079765319824219,
      "logps/chosen": -317.3563537597656,
      "logps/rejected": -324.1982421875,
      "loss": 0.2304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3384076952934265,
      "rewards/margins": 1.6425069570541382,
      "rewards/rejected": -1.3040990829467773,
      "step": 7312
    },
    {
      "epoch": 0.9569640958534391,
      "grad_norm": 6.6564764976501465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1896378993988037,
      "logits/rejected": -0.09976328909397125,
      "logps/chosen": -483.66729736328125,
      "logps/rejected": -491.6221618652344,
      "loss": 0.5696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3434683680534363,
      "rewards/margins": 0.8624886274337769,
      "rewards/rejected": -0.5190203189849854,
      "step": 7313
    },
    {
      "epoch": 0.9570949537907909,
      "grad_norm": 8.689828872680664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040320176631212234,
      "logits/rejected": 0.09453465789556503,
      "logps/chosen": -342.48095703125,
      "logps/rejected": -413.79931640625,
      "loss": 0.5639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.018378496170044,
      "rewards/margins": 0.5516229271888733,
      "rewards/rejected": -1.5700013637542725,
      "step": 7314
    },
    {
      "epoch": 0.9572258117281426,
      "grad_norm": 4.020023345947266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03988049924373627,
      "logits/rejected": -0.04402117431163788,
      "logps/chosen": -521.619873046875,
      "logps/rejected": -498.8292236328125,
      "loss": 0.2835,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.047917164862155914,
      "rewards/margins": 2.171588659286499,
      "rewards/rejected": -2.219506025314331,
      "step": 7315
    },
    {
      "epoch": 0.9573566696654944,
      "grad_norm": 7.773878574371338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03458724915981293,
      "logits/rejected": 0.015701111406087875,
      "logps/chosen": -360.9072265625,
      "logps/rejected": -319.3695373535156,
      "loss": 0.5642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5253130197525024,
      "rewards/margins": 0.4132068455219269,
      "rewards/rejected": -0.9385198354721069,
      "step": 7316
    },
    {
      "epoch": 0.9574875276028462,
      "grad_norm": 6.581567764282227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32276710867881775,
      "logits/rejected": -0.13379475474357605,
      "logps/chosen": -495.41143798828125,
      "logps/rejected": -437.090087890625,
      "loss": 0.3799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5128428936004639,
      "rewards/margins": 1.155849575996399,
      "rewards/rejected": -1.6686925888061523,
      "step": 7317
    },
    {
      "epoch": 0.957618385540198,
      "grad_norm": 6.45283842086792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3195076882839203,
      "logits/rejected": 0.10617487132549286,
      "logps/chosen": -439.0838928222656,
      "logps/rejected": -439.6453857421875,
      "loss": 0.4451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1694570779800415,
      "rewards/margins": 1.2765332460403442,
      "rewards/rejected": -1.4459904432296753,
      "step": 7318
    },
    {
      "epoch": 0.9577492434775496,
      "grad_norm": 9.516569137573242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22142983973026276,
      "logits/rejected": -0.1488797962665558,
      "logps/chosen": -462.9428405761719,
      "logps/rejected": -427.96343994140625,
      "loss": 0.6103,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21873416006565094,
      "rewards/margins": 0.6708180904388428,
      "rewards/rejected": -0.4520839750766754,
      "step": 7319
    },
    {
      "epoch": 0.9578801014149014,
      "grad_norm": 6.831672668457031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.085192009806633,
      "logits/rejected": 0.006794296205043793,
      "logps/chosen": -487.2408447265625,
      "logps/rejected": -412.69952392578125,
      "loss": 0.4586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13110791146755219,
      "rewards/margins": 1.1883803606033325,
      "rewards/rejected": -1.0572724342346191,
      "step": 7320
    },
    {
      "epoch": 0.9580109593522532,
      "grad_norm": 3.7956840991973877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.049114517867565155,
      "logits/rejected": 0.1075567901134491,
      "logps/chosen": -600.31591796875,
      "logps/rejected": -603.973388671875,
      "loss": 0.1707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.176849365234375,
      "rewards/margins": 1.9258252382278442,
      "rewards/rejected": -1.7489758729934692,
      "step": 7321
    },
    {
      "epoch": 0.958141817289605,
      "grad_norm": 6.248319149017334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09229975938796997,
      "logits/rejected": -0.1767636090517044,
      "logps/chosen": -490.72467041015625,
      "logps/rejected": -458.8770751953125,
      "loss": 0.4175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19822824001312256,
      "rewards/margins": 1.0727347135543823,
      "rewards/rejected": -1.2709629535675049,
      "step": 7322
    },
    {
      "epoch": 0.9582726752269567,
      "grad_norm": 10.056647300720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05201689153909683,
      "logits/rejected": 0.1824200302362442,
      "logps/chosen": -375.0429382324219,
      "logps/rejected": -403.32757568359375,
      "loss": 0.7271,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27241724729537964,
      "rewards/margins": 0.23316746950149536,
      "rewards/rejected": -0.505584716796875,
      "step": 7323
    },
    {
      "epoch": 0.9584035331643085,
      "grad_norm": 9.495088577270508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21299532055854797,
      "logits/rejected": -0.023395322263240814,
      "logps/chosen": -377.9155578613281,
      "logps/rejected": -489.96746826171875,
      "loss": 0.4582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.179259791970253,
      "rewards/margins": 1.232419490814209,
      "rewards/rejected": -1.4116793870925903,
      "step": 7324
    },
    {
      "epoch": 0.9585343911016603,
      "grad_norm": 5.416275501251221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14977088570594788,
      "logits/rejected": -0.08174651861190796,
      "logps/chosen": -479.04925537109375,
      "logps/rejected": -568.47509765625,
      "loss": 0.3574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19672143459320068,
      "rewards/margins": 1.8753573894500732,
      "rewards/rejected": -2.0720789432525635,
      "step": 7325
    },
    {
      "epoch": 0.9586652490390121,
      "grad_norm": 8.304025650024414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2845209836959839,
      "logits/rejected": 0.2288699895143509,
      "logps/chosen": -431.06768798828125,
      "logps/rejected": -400.95458984375,
      "loss": 0.6782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13017623126506805,
      "rewards/margins": 0.24479496479034424,
      "rewards/rejected": -0.3749712407588959,
      "step": 7326
    },
    {
      "epoch": 0.9587961069763637,
      "grad_norm": 6.803871154785156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17214080691337585,
      "logits/rejected": 0.0177655890583992,
      "logps/chosen": -306.1775207519531,
      "logps/rejected": -351.4953308105469,
      "loss": 0.5255,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24413500726222992,
      "rewards/margins": 0.7993350028991699,
      "rewards/rejected": -0.555199921131134,
      "step": 7327
    },
    {
      "epoch": 0.9589269649137155,
      "grad_norm": 8.140410423278809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3966205418109894,
      "logits/rejected": -0.35191893577575684,
      "logps/chosen": -493.4685363769531,
      "logps/rejected": -417.5915222167969,
      "loss": 0.642,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2411935329437256,
      "rewards/margins": 0.9457321166992188,
      "rewards/rejected": -2.1869256496429443,
      "step": 7328
    },
    {
      "epoch": 0.9590578228510673,
      "grad_norm": 7.940152168273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040970075875520706,
      "logits/rejected": 0.3215159475803375,
      "logps/chosen": -349.37457275390625,
      "logps/rejected": -406.98724365234375,
      "loss": 0.5219,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.474656879901886,
      "rewards/margins": 1.1814327239990234,
      "rewards/rejected": -1.6560895442962646,
      "step": 7329
    },
    {
      "epoch": 0.9591886807884191,
      "grad_norm": 8.049263954162598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12526626884937286,
      "logits/rejected": -0.3089260458946228,
      "logps/chosen": -405.9039001464844,
      "logps/rejected": -392.4722900390625,
      "loss": 0.7921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6642016172409058,
      "rewards/margins": 0.5269224643707275,
      "rewards/rejected": -1.1911239624023438,
      "step": 7330
    },
    {
      "epoch": 0.9593195387257708,
      "grad_norm": 8.26990032196045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27242350578308105,
      "logits/rejected": -0.11748606711626053,
      "logps/chosen": -370.5982666015625,
      "logps/rejected": -286.68060302734375,
      "loss": 0.5794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47564202547073364,
      "rewards/margins": 0.48051148653030396,
      "rewards/rejected": -0.9561535120010376,
      "step": 7331
    },
    {
      "epoch": 0.9594503966631226,
      "grad_norm": 11.356829643249512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041260942816734314,
      "logits/rejected": -0.024251705035567284,
      "logps/chosen": -487.9209899902344,
      "logps/rejected": -380.8594055175781,
      "loss": 0.8433,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.404814749956131,
      "rewards/margins": 0.37296828627586365,
      "rewards/rejected": -0.7777829766273499,
      "step": 7332
    },
    {
      "epoch": 0.9595812546004744,
      "grad_norm": 6.762019634246826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07395332306623459,
      "logits/rejected": -0.13984675705432892,
      "logps/chosen": -358.7229919433594,
      "logps/rejected": -276.0899963378906,
      "loss": 0.543,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31372034549713135,
      "rewards/margins": 0.6582441329956055,
      "rewards/rejected": -0.971964418888092,
      "step": 7333
    },
    {
      "epoch": 0.9597121125378261,
      "grad_norm": 7.127419948577881,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021795496344566345,
      "logits/rejected": -0.19721736013889313,
      "logps/chosen": -616.5244140625,
      "logps/rejected": -884.8897094726562,
      "loss": 0.3535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06012308597564697,
      "rewards/margins": 1.9212982654571533,
      "rewards/rejected": -1.9814211130142212,
      "step": 7334
    },
    {
      "epoch": 0.9598429704751779,
      "grad_norm": 5.320955276489258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005860574543476105,
      "logits/rejected": -0.12388934940099716,
      "logps/chosen": -459.9136047363281,
      "logps/rejected": -610.4525146484375,
      "loss": 0.3389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005659133195877075,
      "rewards/margins": 1.9458305835723877,
      "rewards/rejected": -1.940171480178833,
      "step": 7335
    },
    {
      "epoch": 0.9599738284125297,
      "grad_norm": 7.614728927612305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06503074616193771,
      "logits/rejected": -0.0917658805847168,
      "logps/chosen": -299.39691162109375,
      "logps/rejected": -304.10186767578125,
      "loss": 0.602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3516502380371094,
      "rewards/margins": 0.4177272617816925,
      "rewards/rejected": -0.7693775296211243,
      "step": 7336
    },
    {
      "epoch": 0.9601046863498814,
      "grad_norm": 6.00060510635376,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0013345181941986084,
      "logits/rejected": -0.10618562996387482,
      "logps/chosen": -421.6097412109375,
      "logps/rejected": -416.3536376953125,
      "loss": 0.3758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6250430345535278,
      "rewards/margins": 2.4083189964294434,
      "rewards/rejected": -1.783275842666626,
      "step": 7337
    },
    {
      "epoch": 0.9602355442872331,
      "grad_norm": 6.379724502563477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11426573991775513,
      "logits/rejected": -0.12586769461631775,
      "logps/chosen": -375.589111328125,
      "logps/rejected": -380.427001953125,
      "loss": 0.4479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5410586595535278,
      "rewards/margins": 0.8283424377441406,
      "rewards/rejected": -1.3694008588790894,
      "step": 7338
    },
    {
      "epoch": 0.9603664022245849,
      "grad_norm": 5.980605125427246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2883153259754181,
      "logits/rejected": -0.23079310357570648,
      "logps/chosen": -544.8760375976562,
      "logps/rejected": -513.2122802734375,
      "loss": 0.3246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01746518537402153,
      "rewards/margins": 1.8061282634735107,
      "rewards/rejected": -1.8235934972763062,
      "step": 7339
    },
    {
      "epoch": 0.9604972601619367,
      "grad_norm": 8.06446647644043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05726875364780426,
      "logits/rejected": -0.02620864100754261,
      "logps/chosen": -493.25628662109375,
      "logps/rejected": -463.21197509765625,
      "loss": 0.5368,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5020792484283447,
      "rewards/margins": 1.0329681634902954,
      "rewards/rejected": -1.5350472927093506,
      "step": 7340
    },
    {
      "epoch": 0.9606281180992885,
      "grad_norm": 6.731294631958008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03607480973005295,
      "logits/rejected": 0.0656740814447403,
      "logps/chosen": -658.7919311523438,
      "logps/rejected": -499.608154296875,
      "loss": 0.4975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019656378775835037,
      "rewards/margins": 1.2851327657699585,
      "rewards/rejected": -1.2654764652252197,
      "step": 7341
    },
    {
      "epoch": 0.9607589760366402,
      "grad_norm": 7.44156551361084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20092245936393738,
      "logits/rejected": -0.007610971108078957,
      "logps/chosen": -251.30633544921875,
      "logps/rejected": -344.25433349609375,
      "loss": 0.4116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7233862280845642,
      "rewards/margins": 1.366687536239624,
      "rewards/rejected": -2.090073823928833,
      "step": 7342
    },
    {
      "epoch": 0.960889833973992,
      "grad_norm": 5.832034587860107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.307973176240921,
      "logits/rejected": 0.3769448399543762,
      "logps/chosen": -416.2579650878906,
      "logps/rejected": -415.91766357421875,
      "loss": 0.3884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3029167056083679,
      "rewards/margins": 1.0339550971984863,
      "rewards/rejected": -1.336871862411499,
      "step": 7343
    },
    {
      "epoch": 0.9610206919113438,
      "grad_norm": 7.77009916305542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18409386277198792,
      "logits/rejected": 0.19472762942314148,
      "logps/chosen": -467.9427490234375,
      "logps/rejected": -485.33477783203125,
      "loss": 0.4814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14999105036258698,
      "rewards/margins": 0.8572885990142822,
      "rewards/rejected": -0.7072975039482117,
      "step": 7344
    },
    {
      "epoch": 0.9611515498486956,
      "grad_norm": 8.362242698669434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2269277274608612,
      "logits/rejected": -0.2205144166946411,
      "logps/chosen": -433.24566650390625,
      "logps/rejected": -517.343017578125,
      "loss": 0.6172,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0400601625442505,
      "rewards/margins": 0.7284865379333496,
      "rewards/rejected": -1.7685465812683105,
      "step": 7345
    },
    {
      "epoch": 0.9612824077860472,
      "grad_norm": 10.464316368103027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1229918897151947,
      "logits/rejected": -0.28461477160453796,
      "logps/chosen": -441.7769470214844,
      "logps/rejected": -340.4058532714844,
      "loss": 0.8391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9258293509483337,
      "rewards/margins": 0.2734905183315277,
      "rewards/rejected": -1.199319839477539,
      "step": 7346
    },
    {
      "epoch": 0.961413265723399,
      "grad_norm": 7.075213432312012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020305171608924866,
      "logits/rejected": 0.06498286873102188,
      "logps/chosen": -481.7261962890625,
      "logps/rejected": -504.9937744140625,
      "loss": 0.4518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13062480092048645,
      "rewards/margins": 0.7997161746025085,
      "rewards/rejected": -0.9303409457206726,
      "step": 7347
    },
    {
      "epoch": 0.9615441236607508,
      "grad_norm": 5.343204021453857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14196431636810303,
      "logits/rejected": 0.2330809235572815,
      "logps/chosen": -338.3849792480469,
      "logps/rejected": -324.8333435058594,
      "loss": 0.3492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19261513650417328,
      "rewards/margins": 1.5467051267623901,
      "rewards/rejected": -1.3540899753570557,
      "step": 7348
    },
    {
      "epoch": 0.9616749815981026,
      "grad_norm": 5.38489294052124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30061203241348267,
      "logits/rejected": -0.015335194766521454,
      "logps/chosen": -559.6900634765625,
      "logps/rejected": -501.76629638671875,
      "loss": 0.2786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.020501524209976196,
      "rewards/margins": 1.5688729286193848,
      "rewards/rejected": -1.5483715534210205,
      "step": 7349
    },
    {
      "epoch": 0.9618058395354543,
      "grad_norm": 5.368016242980957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042183540761470795,
      "logits/rejected": 0.011483009904623032,
      "logps/chosen": -296.2315673828125,
      "logps/rejected": -324.72967529296875,
      "loss": 0.5673,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1793302595615387,
      "rewards/margins": 0.6419344544410706,
      "rewards/rejected": -0.8212646245956421,
      "step": 7350
    },
    {
      "epoch": 0.9619366974728061,
      "grad_norm": 7.176545143127441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14770890772342682,
      "logits/rejected": -0.2814870774745941,
      "logps/chosen": -322.733154296875,
      "logps/rejected": -409.8857727050781,
      "loss": 0.4975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6009763479232788,
      "rewards/margins": 0.9391059875488281,
      "rewards/rejected": -1.5400824546813965,
      "step": 7351
    },
    {
      "epoch": 0.9620675554101579,
      "grad_norm": 7.3727593421936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.118077352643013,
      "logits/rejected": -0.06941031664609909,
      "logps/chosen": -457.9382629394531,
      "logps/rejected": -459.296875,
      "loss": 0.5203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.26077842712402344,
      "rewards/margins": 0.5920107364654541,
      "rewards/rejected": -0.8527891039848328,
      "step": 7352
    },
    {
      "epoch": 0.9621984133475097,
      "grad_norm": 7.829402923583984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14633339643478394,
      "logits/rejected": 0.15949246287345886,
      "logps/chosen": -599.2528076171875,
      "logps/rejected": -422.05902099609375,
      "loss": 0.5726,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5240875482559204,
      "rewards/margins": 0.9409531354904175,
      "rewards/rejected": -1.465040683746338,
      "step": 7353
    },
    {
      "epoch": 0.9623292712848613,
      "grad_norm": 6.894240856170654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0076288990676403046,
      "logits/rejected": -0.10594557225704193,
      "logps/chosen": -469.4358825683594,
      "logps/rejected": -438.90252685546875,
      "loss": 0.5127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3585338890552521,
      "rewards/margins": 0.6741684675216675,
      "rewards/rejected": -1.0327023267745972,
      "step": 7354
    },
    {
      "epoch": 0.9624601292222131,
      "grad_norm": 7.409618854522705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21930405497550964,
      "logits/rejected": -0.08136364817619324,
      "logps/chosen": -356.6662902832031,
      "logps/rejected": -400.4798278808594,
      "loss": 0.4486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7378857731819153,
      "rewards/margins": 1.5790619850158691,
      "rewards/rejected": -2.3169476985931396,
      "step": 7355
    },
    {
      "epoch": 0.9625909871595649,
      "grad_norm": 4.428421497344971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1510992795228958,
      "logits/rejected": -0.18472421169281006,
      "logps/chosen": -373.2593688964844,
      "logps/rejected": -451.17022705078125,
      "loss": 0.3176,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6094970703125,
      "rewards/margins": 1.5811574459075928,
      "rewards/rejected": -2.1906545162200928,
      "step": 7356
    },
    {
      "epoch": 0.9627218450969166,
      "grad_norm": 7.138041019439697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06459105014801025,
      "logits/rejected": -0.2013418823480606,
      "logps/chosen": -231.70774841308594,
      "logps/rejected": -297.6639404296875,
      "loss": 0.5364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45035848021507263,
      "rewards/margins": 1.0124571323394775,
      "rewards/rejected": -1.4628154039382935,
      "step": 7357
    },
    {
      "epoch": 0.9628527030342684,
      "grad_norm": 5.271973609924316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13890501856803894,
      "logits/rejected": -0.26320409774780273,
      "logps/chosen": -478.6090393066406,
      "logps/rejected": -473.257568359375,
      "loss": 0.4192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2363031506538391,
      "rewards/margins": 1.1623550653457642,
      "rewards/rejected": -0.926051914691925,
      "step": 7358
    },
    {
      "epoch": 0.9629835609716202,
      "grad_norm": 8.647327423095703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035847388207912445,
      "logits/rejected": -0.1427835375070572,
      "logps/chosen": -463.3600158691406,
      "logps/rejected": -469.9551086425781,
      "loss": 0.6422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32983723282814026,
      "rewards/margins": 1.0125186443328857,
      "rewards/rejected": -1.3423559665679932,
      "step": 7359
    },
    {
      "epoch": 0.963114418908972,
      "grad_norm": 3.6282289028167725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08719022572040558,
      "logits/rejected": -0.07900624722242355,
      "logps/chosen": -536.1674194335938,
      "logps/rejected": -492.92218017578125,
      "loss": 0.2027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5462953448295593,
      "rewards/margins": 2.0226902961730957,
      "rewards/rejected": -2.5689854621887207,
      "step": 7360
    },
    {
      "epoch": 0.9632452768463237,
      "grad_norm": 34.0926399230957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1058683842420578,
      "logits/rejected": -0.0941835269331932,
      "logps/chosen": -820.1514892578125,
      "logps/rejected": -638.0625,
      "loss": 0.373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9639240503311157,
      "rewards/margins": 1.3056529760360718,
      "rewards/rejected": -2.2695770263671875,
      "step": 7361
    },
    {
      "epoch": 0.9633761347836755,
      "grad_norm": 10.039505004882812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09987647831439972,
      "logits/rejected": 0.19964975118637085,
      "logps/chosen": -451.4525451660156,
      "logps/rejected": -454.7621154785156,
      "loss": 0.7762,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.333915114402771,
      "rewards/margins": 0.17223986983299255,
      "rewards/rejected": -1.506155014038086,
      "step": 7362
    },
    {
      "epoch": 0.9635069927210272,
      "grad_norm": 7.101495742797852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07047542184591293,
      "logits/rejected": -0.07019122689962387,
      "logps/chosen": -533.0955810546875,
      "logps/rejected": -586.038818359375,
      "loss": 0.4822,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2418106198310852,
      "rewards/margins": 0.929524838924408,
      "rewards/rejected": -1.1713354587554932,
      "step": 7363
    },
    {
      "epoch": 0.963637850658379,
      "grad_norm": 11.863448143005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12587898969650269,
      "logits/rejected": -0.2924594283103943,
      "logps/chosen": -408.93505859375,
      "logps/rejected": -454.54351806640625,
      "loss": 0.4524,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7337605953216553,
      "rewards/margins": 0.8570379614830017,
      "rewards/rejected": -1.5907986164093018,
      "step": 7364
    },
    {
      "epoch": 0.9637687085957307,
      "grad_norm": 7.718493461608887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08088776469230652,
      "logits/rejected": -0.514080286026001,
      "logps/chosen": -289.9919128417969,
      "logps/rejected": -276.290771484375,
      "loss": 0.7951,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.518380880355835,
      "rewards/margins": 0.0061807651072740555,
      "rewards/rejected": -1.5245616436004639,
      "step": 7365
    },
    {
      "epoch": 0.9638995665330825,
      "grad_norm": 6.487247467041016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09314261376857758,
      "logits/rejected": -0.2738170325756073,
      "logps/chosen": -360.8483581542969,
      "logps/rejected": -370.0660095214844,
      "loss": 0.4582,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9938660860061646,
      "rewards/margins": 1.429648756980896,
      "rewards/rejected": -2.4235148429870605,
      "step": 7366
    },
    {
      "epoch": 0.9640304244704343,
      "grad_norm": 15.84827709197998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35917386412620544,
      "logits/rejected": -0.26694682240486145,
      "logps/chosen": -406.0849914550781,
      "logps/rejected": -342.2622985839844,
      "loss": 1.0485,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.8862582445144653,
      "rewards/margins": -0.03861785680055618,
      "rewards/rejected": -1.8476401567459106,
      "step": 7367
    },
    {
      "epoch": 0.9641612824077861,
      "grad_norm": 6.6256608963012695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10748499631881714,
      "logits/rejected": 0.08964772522449493,
      "logps/chosen": -358.1842041015625,
      "logps/rejected": -385.2271728515625,
      "loss": 0.7151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8381820321083069,
      "rewards/margins": 0.3584138751029968,
      "rewards/rejected": -1.1965959072113037,
      "step": 7368
    },
    {
      "epoch": 0.9642921403451378,
      "grad_norm": 14.414199829101562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5713639259338379,
      "logits/rejected": -0.3942357897758484,
      "logps/chosen": -351.30255126953125,
      "logps/rejected": -353.0858459472656,
      "loss": 0.9608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6606990098953247,
      "rewards/margins": 0.21664153039455414,
      "rewards/rejected": -1.87734055519104,
      "step": 7369
    },
    {
      "epoch": 0.9644229982824896,
      "grad_norm": 7.860049724578857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19520223140716553,
      "logits/rejected": -0.2262304425239563,
      "logps/chosen": -452.520263671875,
      "logps/rejected": -403.7315979003906,
      "loss": 0.3976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1445972919464111,
      "rewards/margins": 0.9627304673194885,
      "rewards/rejected": -2.107327699661255,
      "step": 7370
    },
    {
      "epoch": 0.9645538562198414,
      "grad_norm": 6.411050796508789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11894906312227249,
      "logits/rejected": -0.0729811042547226,
      "logps/chosen": -526.6659545898438,
      "logps/rejected": -414.3030090332031,
      "loss": 0.3478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48705047369003296,
      "rewards/margins": 1.8877274990081787,
      "rewards/rejected": -2.3747777938842773,
      "step": 7371
    },
    {
      "epoch": 0.9646847141571931,
      "grad_norm": 7.988763809204102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12741844356060028,
      "logits/rejected": 0.04838491231203079,
      "logps/chosen": -321.7525634765625,
      "logps/rejected": -481.17901611328125,
      "loss": 0.5667,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.468496322631836,
      "rewards/margins": 0.6139767169952393,
      "rewards/rejected": -2.082473039627075,
      "step": 7372
    },
    {
      "epoch": 0.9648155720945448,
      "grad_norm": 7.195220470428467,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09750419855117798,
      "logits/rejected": -0.16898436844348907,
      "logps/chosen": -428.54022216796875,
      "logps/rejected": -534.0076904296875,
      "loss": 0.3199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9240005016326904,
      "rewards/margins": 1.7060035467147827,
      "rewards/rejected": -2.6300041675567627,
      "step": 7373
    },
    {
      "epoch": 0.9649464300318966,
      "grad_norm": 7.11204195022583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04468690603971481,
      "logits/rejected": -0.01178865134716034,
      "logps/chosen": -327.78564453125,
      "logps/rejected": -313.54632568359375,
      "loss": 0.4442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05001278221607208,
      "rewards/margins": 1.5375690460205078,
      "rewards/rejected": -1.4875562191009521,
      "step": 7374
    },
    {
      "epoch": 0.9650772879692484,
      "grad_norm": 7.534643173217773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05476488918066025,
      "logits/rejected": 0.15606576204299927,
      "logps/chosen": -616.6559448242188,
      "logps/rejected": -634.1834716796875,
      "loss": 0.5427,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0711669772863388,
      "rewards/margins": 0.8851020932197571,
      "rewards/rejected": -0.8139351010322571,
      "step": 7375
    },
    {
      "epoch": 0.9652081459066002,
      "grad_norm": 11.913846969604492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04604800418019295,
      "logits/rejected": 0.0028404556214809418,
      "logps/chosen": -418.3222351074219,
      "logps/rejected": -552.5466918945312,
      "loss": 0.7249,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6743612289428711,
      "rewards/margins": 0.6835192441940308,
      "rewards/rejected": -1.3578804731369019,
      "step": 7376
    },
    {
      "epoch": 0.9653390038439519,
      "grad_norm": 12.520639419555664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03142743557691574,
      "logits/rejected": -0.1631307303905487,
      "logps/chosen": -362.0207214355469,
      "logps/rejected": -389.19879150390625,
      "loss": 0.8344,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2644426822662354,
      "rewards/margins": 0.135776549577713,
      "rewards/rejected": -1.400219202041626,
      "step": 7377
    },
    {
      "epoch": 0.9654698617813037,
      "grad_norm": 4.659789085388184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06997808068990707,
      "logits/rejected": -0.15151549875736237,
      "logps/chosen": -508.8560791015625,
      "logps/rejected": -424.263916015625,
      "loss": 0.2482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5996687412261963,
      "rewards/margins": 1.4265072345733643,
      "rewards/rejected": -2.0261759757995605,
      "step": 7378
    },
    {
      "epoch": 0.9656007197186555,
      "grad_norm": 7.0494065284729,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4961651861667633,
      "logits/rejected": -0.4696764051914215,
      "logps/chosen": -415.94873046875,
      "logps/rejected": -581.3311767578125,
      "loss": 0.4088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.021150797605514526,
      "rewards/margins": 1.3434487581253052,
      "rewards/rejected": -1.364599585533142,
      "step": 7379
    },
    {
      "epoch": 0.9657315776560071,
      "grad_norm": 3.999767303466797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16661961376667023,
      "logits/rejected": -0.25898319482803345,
      "logps/chosen": -414.862548828125,
      "logps/rejected": -403.0977478027344,
      "loss": 0.2533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6437784433364868,
      "rewards/margins": 1.7543797492980957,
      "rewards/rejected": -2.398157835006714,
      "step": 7380
    },
    {
      "epoch": 0.9658624355933589,
      "grad_norm": 6.787871837615967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014844466000795364,
      "logits/rejected": -0.14848282933235168,
      "logps/chosen": -445.8935852050781,
      "logps/rejected": -388.4443359375,
      "loss": 0.5534,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09231262654066086,
      "rewards/margins": 0.6148252487182617,
      "rewards/rejected": -0.522512674331665,
      "step": 7381
    },
    {
      "epoch": 0.9659932935307107,
      "grad_norm": 3.8854782581329346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025123849511146545,
      "logits/rejected": -0.19594819843769073,
      "logps/chosen": -494.6712951660156,
      "logps/rejected": -415.15863037109375,
      "loss": 0.1998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39794546365737915,
      "rewards/margins": 2.3966171741485596,
      "rewards/rejected": -1.9986718893051147,
      "step": 7382
    },
    {
      "epoch": 0.9661241514680625,
      "grad_norm": 6.443107604980469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2281552255153656,
      "logits/rejected": -0.09006074070930481,
      "logps/chosen": -487.93975830078125,
      "logps/rejected": -586.9555053710938,
      "loss": 0.5409,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2956146001815796,
      "rewards/margins": 0.8855614066123962,
      "rewards/rejected": -1.181175947189331,
      "step": 7383
    },
    {
      "epoch": 0.9662550094054142,
      "grad_norm": 6.9651689529418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20797616243362427,
      "logits/rejected": -0.22783759236335754,
      "logps/chosen": -478.3257751464844,
      "logps/rejected": -535.1519165039062,
      "loss": 0.3882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6682437658309937,
      "rewards/margins": 1.1365036964416504,
      "rewards/rejected": -1.8047477006912231,
      "step": 7384
    },
    {
      "epoch": 0.966385867342766,
      "grad_norm": 3.0732076168060303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03459479659795761,
      "logits/rejected": -0.024544406682252884,
      "logps/chosen": -690.0401611328125,
      "logps/rejected": -487.38543701171875,
      "loss": 0.1727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8282535076141357,
      "rewards/margins": 1.842058539390564,
      "rewards/rejected": -1.0138050317764282,
      "step": 7385
    },
    {
      "epoch": 0.9665167252801178,
      "grad_norm": 6.018550395965576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10133448243141174,
      "logits/rejected": -0.05906536430120468,
      "logps/chosen": -398.5188903808594,
      "logps/rejected": -692.92529296875,
      "loss": 0.411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10762539505958557,
      "rewards/margins": 1.5192146301269531,
      "rewards/rejected": -1.4115893840789795,
      "step": 7386
    },
    {
      "epoch": 0.9666475832174696,
      "grad_norm": 5.613661766052246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07093372941017151,
      "logits/rejected": -0.23464414477348328,
      "logps/chosen": -349.6033020019531,
      "logps/rejected": -394.48236083984375,
      "loss": 0.3138,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4779089093208313,
      "rewards/margins": 1.7719379663467407,
      "rewards/rejected": -2.2498466968536377,
      "step": 7387
    },
    {
      "epoch": 0.9667784411548213,
      "grad_norm": 9.84472942352295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16986984014511108,
      "logits/rejected": -0.24268022179603577,
      "logps/chosen": -330.954833984375,
      "logps/rejected": -350.3868103027344,
      "loss": 0.9549,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4147889614105225,
      "rewards/margins": -0.23759804666042328,
      "rewards/rejected": -1.1771907806396484,
      "step": 7388
    },
    {
      "epoch": 0.966909299092173,
      "grad_norm": 7.568436145782471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022151630371809006,
      "logits/rejected": -0.008981436491012573,
      "logps/chosen": -460.12603759765625,
      "logps/rejected": -508.99517822265625,
      "loss": 0.4809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14776208996772766,
      "rewards/margins": 0.7597823143005371,
      "rewards/rejected": -0.9075443744659424,
      "step": 7389
    },
    {
      "epoch": 0.9670401570295248,
      "grad_norm": 9.813872337341309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0025842562317848206,
      "logits/rejected": -0.260817289352417,
      "logps/chosen": -452.0667724609375,
      "logps/rejected": -412.11529541015625,
      "loss": 0.6501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5441078543663025,
      "rewards/margins": 0.782079815864563,
      "rewards/rejected": -1.3261876106262207,
      "step": 7390
    },
    {
      "epoch": 0.9671710149668766,
      "grad_norm": 9.519261360168457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16646897792816162,
      "logits/rejected": -0.21817144751548767,
      "logps/chosen": -447.5025634765625,
      "logps/rejected": -401.35333251953125,
      "loss": 0.6747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0738941431045532,
      "rewards/margins": 0.35312631726264954,
      "rewards/rejected": -1.42702054977417,
      "step": 7391
    },
    {
      "epoch": 0.9673018729042283,
      "grad_norm": 9.066112518310547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19392311573028564,
      "logits/rejected": 0.05933631956577301,
      "logps/chosen": -605.6026611328125,
      "logps/rejected": -399.57354736328125,
      "loss": 0.7611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9508019089698792,
      "rewards/margins": 0.3230161666870117,
      "rewards/rejected": -1.273818016052246,
      "step": 7392
    },
    {
      "epoch": 0.9674327308415801,
      "grad_norm": 10.26583194732666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2902335226535797,
      "logits/rejected": -0.21210375428199768,
      "logps/chosen": -572.8697509765625,
      "logps/rejected": -551.7026977539062,
      "loss": 0.6656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6321184039115906,
      "rewards/margins": 1.034560203552246,
      "rewards/rejected": -1.6666786670684814,
      "step": 7393
    },
    {
      "epoch": 0.9675635887789319,
      "grad_norm": 5.968193054199219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12414209544658661,
      "logits/rejected": -0.07209616154432297,
      "logps/chosen": -334.1251220703125,
      "logps/rejected": -303.8504943847656,
      "loss": 0.4914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46291661262512207,
      "rewards/margins": 1.1242656707763672,
      "rewards/rejected": -1.5871822834014893,
      "step": 7394
    },
    {
      "epoch": 0.9676944467162837,
      "grad_norm": 5.06872034072876,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17376810312271118,
      "logits/rejected": -0.08792103081941605,
      "logps/chosen": -379.6773681640625,
      "logps/rejected": -408.25506591796875,
      "loss": 0.3404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47273197770118713,
      "rewards/margins": 1.4594594240188599,
      "rewards/rejected": -1.932191252708435,
      "step": 7395
    },
    {
      "epoch": 0.9678253046536354,
      "grad_norm": 8.585983276367188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20138737559318542,
      "logits/rejected": 0.04457689821720123,
      "logps/chosen": -368.9494323730469,
      "logps/rejected": -441.2671813964844,
      "loss": 0.5549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7202471494674683,
      "rewards/margins": 1.3772293329238892,
      "rewards/rejected": -2.0974764823913574,
      "step": 7396
    },
    {
      "epoch": 0.9679561625909872,
      "grad_norm": 4.972629070281982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.036160700023174286,
      "logits/rejected": -0.001588163897395134,
      "logps/chosen": -336.5899658203125,
      "logps/rejected": -396.8857421875,
      "loss": 0.2567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12749740481376648,
      "rewards/margins": 2.0989034175872803,
      "rewards/rejected": -1.9714059829711914,
      "step": 7397
    },
    {
      "epoch": 0.968087020528339,
      "grad_norm": 10.558160781860352,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023922450840473175,
      "logits/rejected": -0.14214792847633362,
      "logps/chosen": -511.2831115722656,
      "logps/rejected": -438.13739013671875,
      "loss": 0.5435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2902491092681885,
      "rewards/margins": 1.7021324634552002,
      "rewards/rejected": -2.9923813343048096,
      "step": 7398
    },
    {
      "epoch": 0.9682178784656907,
      "grad_norm": 10.552948951721191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034880563616752625,
      "logits/rejected": -0.17610491812229156,
      "logps/chosen": -419.2392578125,
      "logps/rejected": -467.37847900390625,
      "loss": 0.8197,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8024993538856506,
      "rewards/margins": 0.7069395184516907,
      "rewards/rejected": -1.5094387531280518,
      "step": 7399
    },
    {
      "epoch": 0.9683487364030424,
      "grad_norm": 7.013615131378174,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08117801696062088,
      "logits/rejected": -0.05672202259302139,
      "logps/chosen": -445.19891357421875,
      "logps/rejected": -394.00677490234375,
      "loss": 0.59,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6058311462402344,
      "rewards/margins": 0.7357238531112671,
      "rewards/rejected": -1.3415549993515015,
      "step": 7400
    },
    {
      "epoch": 0.9684795943403942,
      "grad_norm": 11.336899757385254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1230623871088028,
      "logits/rejected": 0.2562806010246277,
      "logps/chosen": -492.4095764160156,
      "logps/rejected": -416.025634765625,
      "loss": 0.9139,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.33940887451171875,
      "rewards/margins": 0.04395981878042221,
      "rewards/rejected": -0.38336867094039917,
      "step": 7401
    },
    {
      "epoch": 0.968610452277746,
      "grad_norm": 12.378840446472168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40895313024520874,
      "logits/rejected": -0.2945987582206726,
      "logps/chosen": -568.108642578125,
      "logps/rejected": -512.3504638671875,
      "loss": 0.744,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9754137396812439,
      "rewards/margins": 0.4699983298778534,
      "rewards/rejected": -1.4454121589660645,
      "step": 7402
    },
    {
      "epoch": 0.9687413102150977,
      "grad_norm": 6.834551811218262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03580916300415993,
      "logits/rejected": -0.12275370210409164,
      "logps/chosen": -348.33538818359375,
      "logps/rejected": -404.9935302734375,
      "loss": 0.4266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3890100419521332,
      "rewards/margins": 1.2291418313980103,
      "rewards/rejected": -1.6181519031524658,
      "step": 7403
    },
    {
      "epoch": 0.9688721681524495,
      "grad_norm": 7.974302291870117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2625529170036316,
      "logits/rejected": -0.10915377736091614,
      "logps/chosen": -283.75091552734375,
      "logps/rejected": -363.08050537109375,
      "loss": 0.7282,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2793718576431274,
      "rewards/margins": 0.21241086721420288,
      "rewards/rejected": -1.4917826652526855,
      "step": 7404
    },
    {
      "epoch": 0.9690030260898013,
      "grad_norm": 3.2077317237854004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009494611993432045,
      "logits/rejected": -0.14130285382270813,
      "logps/chosen": -292.6560974121094,
      "logps/rejected": -308.8158874511719,
      "loss": 0.3056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3355354070663452,
      "rewards/margins": 1.4090182781219482,
      "rewards/rejected": -1.0734827518463135,
      "step": 7405
    },
    {
      "epoch": 0.9691338840271531,
      "grad_norm": 6.713824272155762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14907853305339813,
      "logits/rejected": 0.037351205945014954,
      "logps/chosen": -435.0868835449219,
      "logps/rejected": -427.90325927734375,
      "loss": 0.5112,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.27174320816993713,
      "rewards/margins": 1.1335482597351074,
      "rewards/rejected": -0.8618049621582031,
      "step": 7406
    },
    {
      "epoch": 0.9692647419645047,
      "grad_norm": 7.2160162925720215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17873188853263855,
      "logits/rejected": -0.04891742393374443,
      "logps/chosen": -643.7503662109375,
      "logps/rejected": -498.07293701171875,
      "loss": 0.5217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2594377398490906,
      "rewards/margins": 0.5691801309585571,
      "rewards/rejected": -0.30974239110946655,
      "step": 7407
    },
    {
      "epoch": 0.9693955999018565,
      "grad_norm": 12.257861137390137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16923779249191284,
      "logits/rejected": 0.026354704052209854,
      "logps/chosen": -557.0396118164062,
      "logps/rejected": -424.4710693359375,
      "loss": 1.0033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.252492904663086,
      "rewards/margins": 0.061850979924201965,
      "rewards/rejected": -1.314343810081482,
      "step": 7408
    },
    {
      "epoch": 0.9695264578392083,
      "grad_norm": 8.101288795471191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29684966802597046,
      "logits/rejected": 0.37042149901390076,
      "logps/chosen": -440.11456298828125,
      "logps/rejected": -439.321044921875,
      "loss": 0.6092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9698003530502319,
      "rewards/margins": 0.4130314588546753,
      "rewards/rejected": -1.3828318119049072,
      "step": 7409
    },
    {
      "epoch": 0.9696573157765601,
      "grad_norm": 7.438313007354736,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12079967558383942,
      "logits/rejected": -0.07887890934944153,
      "logps/chosen": -511.434326171875,
      "logps/rejected": -506.5930480957031,
      "loss": 0.5168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09327948093414307,
      "rewards/margins": 0.9300851821899414,
      "rewards/rejected": -1.023364543914795,
      "step": 7410
    },
    {
      "epoch": 0.9697881737139118,
      "grad_norm": 7.8448166847229,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11660732328891754,
      "logits/rejected": -0.0573711022734642,
      "logps/chosen": -593.311767578125,
      "logps/rejected": -560.182373046875,
      "loss": 0.4745,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12409702688455582,
      "rewards/margins": 0.8421623706817627,
      "rewards/rejected": -0.9662593603134155,
      "step": 7411
    },
    {
      "epoch": 0.9699190316512636,
      "grad_norm": 8.616141319274902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09988322854042053,
      "logits/rejected": 0.001046597957611084,
      "logps/chosen": -546.5446166992188,
      "logps/rejected": -489.72113037109375,
      "loss": 0.5556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5298740863800049,
      "rewards/margins": 0.6276282072067261,
      "rewards/rejected": -1.1575021743774414,
      "step": 7412
    },
    {
      "epoch": 0.9700498895886154,
      "grad_norm": 9.456954002380371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13556623458862305,
      "logits/rejected": 0.05407637357711792,
      "logps/chosen": -711.152587890625,
      "logps/rejected": -673.366943359375,
      "loss": 0.6144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2498062252998352,
      "rewards/margins": 1.0644397735595703,
      "rewards/rejected": -0.8146336078643799,
      "step": 7413
    },
    {
      "epoch": 0.9701807475259672,
      "grad_norm": 8.497560501098633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10401933640241623,
      "logits/rejected": 0.0034534335136413574,
      "logps/chosen": -386.1768798828125,
      "logps/rejected": -445.5285339355469,
      "loss": 0.53,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48482245206832886,
      "rewards/margins": 0.7928011417388916,
      "rewards/rejected": -1.2776236534118652,
      "step": 7414
    },
    {
      "epoch": 0.9703116054633188,
      "grad_norm": 8.761480331420898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17473164200782776,
      "logits/rejected": -0.1586967408657074,
      "logps/chosen": -220.59323120117188,
      "logps/rejected": -472.57659912109375,
      "loss": 0.4413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6486648321151733,
      "rewards/margins": 1.1628564596176147,
      "rewards/rejected": -1.8115215301513672,
      "step": 7415
    },
    {
      "epoch": 0.9704424634006706,
      "grad_norm": 13.610328674316406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10437875241041183,
      "logits/rejected": -0.21163161098957062,
      "logps/chosen": -686.7946166992188,
      "logps/rejected": -761.5928344726562,
      "loss": 0.375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1606598198413849,
      "rewards/margins": 1.8776267766952515,
      "rewards/rejected": -1.716966986656189,
      "step": 7416
    },
    {
      "epoch": 0.9705733213380224,
      "grad_norm": 7.39308500289917,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21515601873397827,
      "logits/rejected": -0.1665738821029663,
      "logps/chosen": -457.6314392089844,
      "logps/rejected": -529.3094482421875,
      "loss": 0.6055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09850084036588669,
      "rewards/margins": 0.8201891183853149,
      "rewards/rejected": -0.7216883897781372,
      "step": 7417
    },
    {
      "epoch": 0.9707041792753742,
      "grad_norm": 8.848457336425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15919986367225647,
      "logits/rejected": -0.1237935721874237,
      "logps/chosen": -504.34375,
      "logps/rejected": -481.77557373046875,
      "loss": 0.6251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6633388996124268,
      "rewards/margins": 0.8579314947128296,
      "rewards/rejected": -1.5212703943252563,
      "step": 7418
    },
    {
      "epoch": 0.9708350372127259,
      "grad_norm": 8.31839370727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19526401162147522,
      "logits/rejected": -0.4144362807273865,
      "logps/chosen": -421.74951171875,
      "logps/rejected": -436.72442626953125,
      "loss": 0.6552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6834954023361206,
      "rewards/margins": 0.7937507629394531,
      "rewards/rejected": -1.4772460460662842,
      "step": 7419
    },
    {
      "epoch": 0.9709658951500777,
      "grad_norm": 8.665265083312988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14515487849712372,
      "logits/rejected": -0.1390233337879181,
      "logps/chosen": -370.25885009765625,
      "logps/rejected": -412.30462646484375,
      "loss": 0.5713,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28682518005371094,
      "rewards/margins": 0.9091759324073792,
      "rewards/rejected": -1.1960010528564453,
      "step": 7420
    },
    {
      "epoch": 0.9710967530874295,
      "grad_norm": 7.305261611938477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20583927631378174,
      "logits/rejected": -0.11137671023607254,
      "logps/chosen": -519.5218505859375,
      "logps/rejected": -645.9703369140625,
      "loss": 0.4429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9325889945030212,
      "rewards/margins": 1.1577450037002563,
      "rewards/rejected": -2.090333938598633,
      "step": 7421
    },
    {
      "epoch": 0.9712276110247812,
      "grad_norm": 6.722828388214111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24223093688488007,
      "logits/rejected": -0.6200573444366455,
      "logps/chosen": -573.36279296875,
      "logps/rejected": -415.8668212890625,
      "loss": 0.509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3125528395175934,
      "rewards/margins": 1.2682653665542603,
      "rewards/rejected": -1.5808181762695312,
      "step": 7422
    },
    {
      "epoch": 0.971358468962133,
      "grad_norm": 9.734522819519043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26284492015838623,
      "logits/rejected": -0.254379004240036,
      "logps/chosen": -508.2425231933594,
      "logps/rejected": -604.5759887695312,
      "loss": 0.5652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16221047937870026,
      "rewards/margins": 1.1273785829544067,
      "rewards/rejected": -1.2895889282226562,
      "step": 7423
    },
    {
      "epoch": 0.9714893268994848,
      "grad_norm": 12.8685302734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08131837099790573,
      "logits/rejected": -0.24429558217525482,
      "logps/chosen": -475.7300720214844,
      "logps/rejected": -543.3103637695312,
      "loss": 0.7014,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.810529351234436,
      "rewards/margins": 0.5124276876449585,
      "rewards/rejected": -1.3229570388793945,
      "step": 7424
    },
    {
      "epoch": 0.9716201848368365,
      "grad_norm": 8.379158973693848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14366461336612701,
      "logits/rejected": 0.15877549350261688,
      "logps/chosen": -590.956298828125,
      "logps/rejected": -640.6912231445312,
      "loss": 0.516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30829697847366333,
      "rewards/margins": 0.5980008840560913,
      "rewards/rejected": -0.9062978625297546,
      "step": 7425
    },
    {
      "epoch": 0.9717510427741882,
      "grad_norm": 7.249230861663818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02922707051038742,
      "logits/rejected": -0.236780047416687,
      "logps/chosen": -503.076904296875,
      "logps/rejected": -397.8341979980469,
      "loss": 0.5841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.553732693195343,
      "rewards/margins": 0.771990180015564,
      "rewards/rejected": -0.21825753152370453,
      "step": 7426
    },
    {
      "epoch": 0.97188190071154,
      "grad_norm": 3.3889245986938477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04386161267757416,
      "logits/rejected": -0.07091058790683746,
      "logps/chosen": -445.851318359375,
      "logps/rejected": -517.4439086914062,
      "loss": 0.1709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.031280517578125,
      "rewards/margins": 1.961949110031128,
      "rewards/rejected": -1.9932295083999634,
      "step": 7427
    },
    {
      "epoch": 0.9720127586488918,
      "grad_norm": 7.476287841796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023806948214769363,
      "logits/rejected": -0.2516558766365051,
      "logps/chosen": -565.6708984375,
      "logps/rejected": -498.27557373046875,
      "loss": 0.3723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09874533116817474,
      "rewards/margins": 1.2515537738800049,
      "rewards/rejected": -1.3502992391586304,
      "step": 7428
    },
    {
      "epoch": 0.9721436165862436,
      "grad_norm": 6.723684310913086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07929201424121857,
      "logits/rejected": 0.17861047387123108,
      "logps/chosen": -432.3873291015625,
      "logps/rejected": -321.4977111816406,
      "loss": 0.4453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38028082251548767,
      "rewards/margins": 1.2614973783493042,
      "rewards/rejected": -1.6417782306671143,
      "step": 7429
    },
    {
      "epoch": 0.9722744745235953,
      "grad_norm": 9.284342765808105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3568669259548187,
      "logits/rejected": -0.15764310956001282,
      "logps/chosen": -602.7848510742188,
      "logps/rejected": -368.454833984375,
      "loss": 0.5251,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9990288019180298,
      "rewards/margins": 0.7860126495361328,
      "rewards/rejected": -1.785041332244873,
      "step": 7430
    },
    {
      "epoch": 0.9724053324609471,
      "grad_norm": 7.41832160949707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1763833910226822,
      "logits/rejected": 0.06788473576307297,
      "logps/chosen": -435.6446533203125,
      "logps/rejected": -457.4392395019531,
      "loss": 0.6118,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.048171985894441605,
      "rewards/margins": 0.5476371645927429,
      "rewards/rejected": -0.5958092212677002,
      "step": 7431
    },
    {
      "epoch": 0.9725361903982989,
      "grad_norm": 8.781905174255371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15464435517787933,
      "logits/rejected": -0.08936150372028351,
      "logps/chosen": -405.178466796875,
      "logps/rejected": -376.3653869628906,
      "loss": 0.6055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.814228892326355,
      "rewards/margins": 0.8693565130233765,
      "rewards/rejected": -1.6835851669311523,
      "step": 7432
    },
    {
      "epoch": 0.9726670483356507,
      "grad_norm": 8.103676795959473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015917018055915833,
      "logits/rejected": -0.26745790243148804,
      "logps/chosen": -520.2673950195312,
      "logps/rejected": -373.0804443359375,
      "loss": 0.5701,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8669641017913818,
      "rewards/margins": 1.2249853610992432,
      "rewards/rejected": -2.091949462890625,
      "step": 7433
    },
    {
      "epoch": 0.9727979062730023,
      "grad_norm": 6.529773712158203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07091807574033737,
      "logits/rejected": 0.10982372611761093,
      "logps/chosen": -500.133544921875,
      "logps/rejected": -415.3163146972656,
      "loss": 0.4805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24362143874168396,
      "rewards/margins": 0.6090399026870728,
      "rewards/rejected": -0.8526613712310791,
      "step": 7434
    },
    {
      "epoch": 0.9729287642103541,
      "grad_norm": 8.82538890838623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23775221407413483,
      "logits/rejected": -0.09648013114929199,
      "logps/chosen": -359.2325744628906,
      "logps/rejected": -522.2857055664062,
      "loss": 0.7308,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.675231397151947,
      "rewards/margins": 0.345262348651886,
      "rewards/rejected": -1.020493745803833,
      "step": 7435
    },
    {
      "epoch": 0.9730596221477059,
      "grad_norm": 5.936532497406006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17789481580257416,
      "logits/rejected": -0.3150080442428589,
      "logps/chosen": -292.84429931640625,
      "logps/rejected": -356.7794494628906,
      "loss": 0.5141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9561991691589355,
      "rewards/margins": 1.0782747268676758,
      "rewards/rejected": -2.0344738960266113,
      "step": 7436
    },
    {
      "epoch": 0.9731904800850577,
      "grad_norm": 9.110255241394043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027198806405067444,
      "logits/rejected": -0.22867988049983978,
      "logps/chosen": -442.9178466796875,
      "logps/rejected": -466.2754211425781,
      "loss": 0.4333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5147468447685242,
      "rewards/margins": 1.27119779586792,
      "rewards/rejected": -1.7859445810317993,
      "step": 7437
    },
    {
      "epoch": 0.9733213380224094,
      "grad_norm": 4.937448024749756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30959051847457886,
      "logits/rejected": -0.4579453468322754,
      "logps/chosen": -483.17144775390625,
      "logps/rejected": -508.86334228515625,
      "loss": 0.2888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.056394580751657486,
      "rewards/margins": 1.4861401319503784,
      "rewards/rejected": -1.5425349473953247,
      "step": 7438
    },
    {
      "epoch": 0.9734521959597612,
      "grad_norm": 7.500956058502197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09607589244842529,
      "logits/rejected": -0.17947882413864136,
      "logps/chosen": -634.06103515625,
      "logps/rejected": -676.2288818359375,
      "loss": 0.4839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21295982599258423,
      "rewards/margins": 1.063875436782837,
      "rewards/rejected": -0.8509155511856079,
      "step": 7439
    },
    {
      "epoch": 0.973583053897113,
      "grad_norm": 7.971965789794922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06800840049982071,
      "logits/rejected": -0.0860467329621315,
      "logps/chosen": -618.4779663085938,
      "logps/rejected": -365.2929992675781,
      "loss": 0.4649,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6838598251342773,
      "rewards/margins": 0.8090195655822754,
      "rewards/rejected": -1.4928795099258423,
      "step": 7440
    },
    {
      "epoch": 0.9737139118344648,
      "grad_norm": 7.592958450317383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0854744240641594,
      "logits/rejected": -0.20077313482761383,
      "logps/chosen": -508.18292236328125,
      "logps/rejected": -466.8039245605469,
      "loss": 0.4168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.785770833492279,
      "rewards/margins": 1.0880141258239746,
      "rewards/rejected": -1.8737850189208984,
      "step": 7441
    },
    {
      "epoch": 0.9738447697718164,
      "grad_norm": 4.483470916748047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26093101501464844,
      "logits/rejected": 0.38743314146995544,
      "logps/chosen": -307.93292236328125,
      "logps/rejected": -402.62103271484375,
      "loss": 0.2957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6311653256416321,
      "rewards/margins": 1.5088348388671875,
      "rewards/rejected": -2.140000104904175,
      "step": 7442
    },
    {
      "epoch": 0.9739756277091682,
      "grad_norm": 6.708509922027588,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17732268571853638,
      "logits/rejected": 0.26612210273742676,
      "logps/chosen": -403.9805603027344,
      "logps/rejected": -362.4238586425781,
      "loss": 0.4106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17606067657470703,
      "rewards/margins": 1.052438735961914,
      "rewards/rejected": -1.2284995317459106,
      "step": 7443
    },
    {
      "epoch": 0.97410648564652,
      "grad_norm": 8.302053451538086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07374522089958191,
      "logits/rejected": -0.10073365271091461,
      "logps/chosen": -595.5059814453125,
      "logps/rejected": -442.36065673828125,
      "loss": 0.6005,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15526656806468964,
      "rewards/margins": 0.7898256778717041,
      "rewards/rejected": -0.9450922608375549,
      "step": 7444
    },
    {
      "epoch": 0.9742373435838717,
      "grad_norm": 7.281593322753906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008698832243680954,
      "logits/rejected": -0.3036191463470459,
      "logps/chosen": -627.594970703125,
      "logps/rejected": -379.4818420410156,
      "loss": 0.4559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1958404779434204,
      "rewards/margins": 1.0860941410064697,
      "rewards/rejected": -2.2819344997406006,
      "step": 7445
    },
    {
      "epoch": 0.9743682015212235,
      "grad_norm": 8.943415641784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019962631165981293,
      "logits/rejected": -0.28257042169570923,
      "logps/chosen": -473.5572509765625,
      "logps/rejected": -345.66229248046875,
      "loss": 0.6267,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3303806185722351,
      "rewards/margins": 0.3972373604774475,
      "rewards/rejected": -0.7276179790496826,
      "step": 7446
    },
    {
      "epoch": 0.9744990594585753,
      "grad_norm": 7.728731155395508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09359671920537949,
      "logits/rejected": -0.16723227500915527,
      "logps/chosen": -509.0498046875,
      "logps/rejected": -576.1151123046875,
      "loss": 0.4179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5514582991600037,
      "rewards/margins": 1.7592132091522217,
      "rewards/rejected": -2.31067156791687,
      "step": 7447
    },
    {
      "epoch": 0.9746299173959271,
      "grad_norm": 10.80465030670166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14013363420963287,
      "logits/rejected": -0.1129375621676445,
      "logps/chosen": -481.251708984375,
      "logps/rejected": -558.1632080078125,
      "loss": 0.8539,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.771293580532074,
      "rewards/margins": 0.3661292791366577,
      "rewards/rejected": -1.137422800064087,
      "step": 7448
    },
    {
      "epoch": 0.9747607753332788,
      "grad_norm": 7.5384521484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47487902641296387,
      "logits/rejected": -0.33469322323799133,
      "logps/chosen": -581.74853515625,
      "logps/rejected": -545.65185546875,
      "loss": 0.4453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.974531888961792,
      "rewards/margins": 1.0071226358413696,
      "rewards/rejected": -1.9816545248031616,
      "step": 7449
    },
    {
      "epoch": 0.9748916332706306,
      "grad_norm": 5.692592620849609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13625024259090424,
      "logits/rejected": -0.20610588788986206,
      "logps/chosen": -462.9930114746094,
      "logps/rejected": -366.1095886230469,
      "loss": 0.4153,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4709016978740692,
      "rewards/margins": 0.906890332698822,
      "rewards/rejected": -1.3777920007705688,
      "step": 7450
    },
    {
      "epoch": 0.9750224912079823,
      "grad_norm": 7.23446798324585,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08513319492340088,
      "logits/rejected": -0.19275762140750885,
      "logps/chosen": -525.4854125976562,
      "logps/rejected": -451.453857421875,
      "loss": 0.4763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26054131984710693,
      "rewards/margins": 1.2660237550735474,
      "rewards/rejected": -1.0054823160171509,
      "step": 7451
    },
    {
      "epoch": 0.9751533491453341,
      "grad_norm": 9.497793197631836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04164481908082962,
      "logits/rejected": -0.07469919323921204,
      "logps/chosen": -535.413818359375,
      "logps/rejected": -559.136474609375,
      "loss": 0.5002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7481590509414673,
      "rewards/margins": 1.12408447265625,
      "rewards/rejected": -1.8722435235977173,
      "step": 7452
    },
    {
      "epoch": 0.9752842070826858,
      "grad_norm": 4.5881147384643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15105406939983368,
      "logits/rejected": -0.07894069701433182,
      "logps/chosen": -460.888671875,
      "logps/rejected": -378.325439453125,
      "loss": 0.3335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09651928395032883,
      "rewards/margins": 1.2242279052734375,
      "rewards/rejected": -1.1277086734771729,
      "step": 7453
    },
    {
      "epoch": 0.9754150650200376,
      "grad_norm": 8.591400146484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2648342251777649,
      "logits/rejected": 0.13563893735408783,
      "logps/chosen": -585.3086547851562,
      "logps/rejected": -622.5657958984375,
      "loss": 0.5154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0380761623382568,
      "rewards/margins": 0.9890196919441223,
      "rewards/rejected": -2.0270957946777344,
      "step": 7454
    },
    {
      "epoch": 0.9755459229573894,
      "grad_norm": 5.174232482910156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.431095689535141,
      "logits/rejected": -0.2940484285354614,
      "logps/chosen": -497.72119140625,
      "logps/rejected": -466.2793273925781,
      "loss": 0.3947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16147537529468536,
      "rewards/margins": 1.2568615674972534,
      "rewards/rejected": -1.4183368682861328,
      "step": 7455
    },
    {
      "epoch": 0.9756767808947412,
      "grad_norm": 9.888612747192383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010377811267971992,
      "logits/rejected": 0.0852329432964325,
      "logps/chosen": -493.57550048828125,
      "logps/rejected": -612.8115234375,
      "loss": 0.8181,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031153112649917603,
      "rewards/margins": 0.3342476189136505,
      "rewards/rejected": -0.3654007017612457,
      "step": 7456
    },
    {
      "epoch": 0.9758076388320929,
      "grad_norm": 7.693365573883057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10724667459726334,
      "logits/rejected": -0.25012922286987305,
      "logps/chosen": -469.1983642578125,
      "logps/rejected": -512.7017211914062,
      "loss": 0.58,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.069983720779419,
      "rewards/margins": 0.8375414609909058,
      "rewards/rejected": -1.9075250625610352,
      "step": 7457
    },
    {
      "epoch": 0.9759384967694447,
      "grad_norm": 11.175509452819824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12374566495418549,
      "logits/rejected": -0.14662685990333557,
      "logps/chosen": -548.421142578125,
      "logps/rejected": -586.4429321289062,
      "loss": 0.6683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2719196379184723,
      "rewards/margins": 0.7676124572753906,
      "rewards/rejected": -1.0395320653915405,
      "step": 7458
    },
    {
      "epoch": 0.9760693547067965,
      "grad_norm": 9.699555397033691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3083106577396393,
      "logits/rejected": -0.18183821439743042,
      "logps/chosen": -442.4010009765625,
      "logps/rejected": -482.1617126464844,
      "loss": 0.7905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8851397037506104,
      "rewards/margins": 0.5031439065933228,
      "rewards/rejected": -1.388283610343933,
      "step": 7459
    },
    {
      "epoch": 0.9762002126441482,
      "grad_norm": 10.420448303222656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10928304493427277,
      "logits/rejected": -0.1298457682132721,
      "logps/chosen": -519.5432739257812,
      "logps/rejected": -422.3782653808594,
      "loss": 0.6542,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8400158286094666,
      "rewards/margins": 0.2708767056465149,
      "rewards/rejected": -1.1108925342559814,
      "step": 7460
    },
    {
      "epoch": 0.9763310705814999,
      "grad_norm": 4.57499361038208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021405242383480072,
      "logits/rejected": -0.2650803029537201,
      "logps/chosen": -459.391357421875,
      "logps/rejected": -504.66546630859375,
      "loss": 0.3153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20393158495426178,
      "rewards/margins": 1.4027570486068726,
      "rewards/rejected": -1.6066886186599731,
      "step": 7461
    },
    {
      "epoch": 0.9764619285188517,
      "grad_norm": 9.404160499572754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15641002357006073,
      "logits/rejected": -0.31882980465888977,
      "logps/chosen": -546.3084716796875,
      "logps/rejected": -504.84991455078125,
      "loss": 0.8491,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13013304769992828,
      "rewards/margins": 0.47813680768013,
      "rewards/rejected": -0.6082698702812195,
      "step": 7462
    },
    {
      "epoch": 0.9765927864562035,
      "grad_norm": 5.034912109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06413324922323227,
      "logits/rejected": -0.045535869896411896,
      "logps/chosen": -549.5903930664062,
      "logps/rejected": -315.2974548339844,
      "loss": 0.2746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23753413558006287,
      "rewards/margins": 1.3484807014465332,
      "rewards/rejected": -1.5860148668289185,
      "step": 7463
    },
    {
      "epoch": 0.9767236443935553,
      "grad_norm": 7.706593990325928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06537050008773804,
      "logits/rejected": -0.20802119374275208,
      "logps/chosen": -457.50445556640625,
      "logps/rejected": -346.50146484375,
      "loss": 0.5089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8070158958435059,
      "rewards/margins": 0.5247545838356018,
      "rewards/rejected": -1.3317703008651733,
      "step": 7464
    },
    {
      "epoch": 0.976854502330907,
      "grad_norm": 7.68333101272583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.142679363489151,
      "logits/rejected": -0.1455705463886261,
      "logps/chosen": -721.9403686523438,
      "logps/rejected": -654.199462890625,
      "loss": 0.5087,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17883186042308807,
      "rewards/margins": 0.9200379252433777,
      "rewards/rejected": -1.098869800567627,
      "step": 7465
    },
    {
      "epoch": 0.9769853602682588,
      "grad_norm": 9.992457389831543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16108864545822144,
      "logits/rejected": -0.2332962155342102,
      "logps/chosen": -529.2388916015625,
      "logps/rejected": -588.5784301757812,
      "loss": 0.6615,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7164576053619385,
      "rewards/margins": 0.6654903292655945,
      "rewards/rejected": -1.3819479942321777,
      "step": 7466
    },
    {
      "epoch": 0.9771162182056106,
      "grad_norm": 7.483346939086914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16495315730571747,
      "logits/rejected": 0.001251816749572754,
      "logps/chosen": -375.1280517578125,
      "logps/rejected": -418.20672607421875,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23117294907569885,
      "rewards/margins": 0.9342374801635742,
      "rewards/rejected": -1.1654103994369507,
      "step": 7467
    },
    {
      "epoch": 0.9772470761429622,
      "grad_norm": 9.773097038269043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06292371451854706,
      "logits/rejected": 0.016625339165329933,
      "logps/chosen": -360.37225341796875,
      "logps/rejected": -414.5817565917969,
      "loss": 0.7135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6131523847579956,
      "rewards/margins": 1.1105464696884155,
      "rewards/rejected": -1.723698616027832,
      "step": 7468
    },
    {
      "epoch": 0.977377934080314,
      "grad_norm": 7.915722370147705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06017009913921356,
      "logits/rejected": -0.22064165771007538,
      "logps/chosen": -668.6561279296875,
      "logps/rejected": -588.4418334960938,
      "loss": 0.6028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07291565090417862,
      "rewards/margins": 1.0395921468734741,
      "rewards/rejected": -1.1125078201293945,
      "step": 7469
    },
    {
      "epoch": 0.9775087920176658,
      "grad_norm": 10.754199028015137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37456047534942627,
      "logits/rejected": -0.3551904559135437,
      "logps/chosen": -376.4302978515625,
      "logps/rejected": -232.0184783935547,
      "loss": 0.7363,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1890283823013306,
      "rewards/margins": 0.38678908348083496,
      "rewards/rejected": -1.5758174657821655,
      "step": 7470
    },
    {
      "epoch": 0.9776396499550176,
      "grad_norm": 6.116600513458252,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014605635777115822,
      "logits/rejected": -0.001522463746368885,
      "logps/chosen": -355.3171691894531,
      "logps/rejected": -444.5439147949219,
      "loss": 0.4758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5178819894790649,
      "rewards/margins": 1.328848958015442,
      "rewards/rejected": -1.8467309474945068,
      "step": 7471
    },
    {
      "epoch": 0.9777705078923693,
      "grad_norm": 12.526126861572266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2946667969226837,
      "logits/rejected": -0.34942108392715454,
      "logps/chosen": -784.3540649414062,
      "logps/rejected": -546.87060546875,
      "loss": 0.9572,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.24049973487854,
      "rewards/margins": -0.2290266454219818,
      "rewards/rejected": -1.0114731788635254,
      "step": 7472
    },
    {
      "epoch": 0.9779013658297211,
      "grad_norm": 6.240302085876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1415191888809204,
      "logits/rejected": -0.33414506912231445,
      "logps/chosen": -386.9261169433594,
      "logps/rejected": -301.709228515625,
      "loss": 0.407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09287533164024353,
      "rewards/margins": 1.0677698850631714,
      "rewards/rejected": -1.1606453657150269,
      "step": 7473
    },
    {
      "epoch": 0.9780322237670729,
      "grad_norm": 11.942521095275879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26140284538269043,
      "logits/rejected": 0.14212827384471893,
      "logps/chosen": -518.1205444335938,
      "logps/rejected": -337.536376953125,
      "loss": 0.793,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5321825742721558,
      "rewards/margins": 0.559972882270813,
      "rewards/rejected": -1.0921554565429688,
      "step": 7474
    },
    {
      "epoch": 0.9781630817044247,
      "grad_norm": 7.281480312347412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2448326051235199,
      "logits/rejected": 0.072080098092556,
      "logps/chosen": -595.658935546875,
      "logps/rejected": -598.1600341796875,
      "loss": 0.4912,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26714038848876953,
      "rewards/margins": 0.6251140832901001,
      "rewards/rejected": -0.8922544717788696,
      "step": 7475
    },
    {
      "epoch": 0.9782939396417764,
      "grad_norm": 5.227209091186523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15335282683372498,
      "logits/rejected": -0.20641887187957764,
      "logps/chosen": -381.62432861328125,
      "logps/rejected": -851.949462890625,
      "loss": 0.2795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.014850991778075695,
      "rewards/margins": 1.2988855838775635,
      "rewards/rejected": -1.3137365579605103,
      "step": 7476
    },
    {
      "epoch": 0.9784247975791281,
      "grad_norm": 7.778903007507324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06529223918914795,
      "logits/rejected": 0.1657460629940033,
      "logps/chosen": -566.2079467773438,
      "logps/rejected": -519.2546997070312,
      "loss": 0.5511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.25902748107910156,
      "rewards/margins": 0.9856346249580383,
      "rewards/rejected": -0.7266071438789368,
      "step": 7477
    },
    {
      "epoch": 0.9785556555164799,
      "grad_norm": 6.642740249633789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17211613059043884,
      "logits/rejected": -0.2009337991476059,
      "logps/chosen": -260.3890380859375,
      "logps/rejected": -322.0852966308594,
      "loss": 0.5585,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17406557500362396,
      "rewards/margins": 0.642098069190979,
      "rewards/rejected": -0.46803244948387146,
      "step": 7478
    },
    {
      "epoch": 0.9786865134538317,
      "grad_norm": 8.301730155944824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030736513435840607,
      "logits/rejected": -0.15986084938049316,
      "logps/chosen": -399.16973876953125,
      "logps/rejected": -482.23822021484375,
      "loss": 0.5461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08147810399532318,
      "rewards/margins": 0.635517954826355,
      "rewards/rejected": -0.7169959545135498,
      "step": 7479
    },
    {
      "epoch": 0.9788173713911834,
      "grad_norm": 12.671220779418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3674451410770416,
      "logits/rejected": -0.17058256268501282,
      "logps/chosen": -531.1688232421875,
      "logps/rejected": -520.4387817382812,
      "loss": 1.0982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7017679214477539,
      "rewards/margins": -0.22872866690158844,
      "rewards/rejected": -0.47303926944732666,
      "step": 7480
    },
    {
      "epoch": 0.9789482293285352,
      "grad_norm": 7.278364658355713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4147881865501404,
      "logits/rejected": -0.3823484480381012,
      "logps/chosen": -395.7065734863281,
      "logps/rejected": -438.6670837402344,
      "loss": 0.5583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8202255964279175,
      "rewards/margins": 0.9435341358184814,
      "rewards/rejected": -1.7637596130371094,
      "step": 7481
    },
    {
      "epoch": 0.979079087265887,
      "grad_norm": 5.032322406768799,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15974640846252441,
      "logits/rejected": -0.2926619052886963,
      "logps/chosen": -541.7010498046875,
      "logps/rejected": -452.8087158203125,
      "loss": 0.2913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4214881956577301,
      "rewards/margins": 1.457061767578125,
      "rewards/rejected": -1.8785499334335327,
      "step": 7482
    },
    {
      "epoch": 0.9792099452032388,
      "grad_norm": 12.224133491516113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07173842191696167,
      "logits/rejected": -0.09591913223266602,
      "logps/chosen": -476.3774719238281,
      "logps/rejected": -477.8274230957031,
      "loss": 0.8554,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.06513022631406784,
      "rewards/margins": -0.09374085813760757,
      "rewards/rejected": 0.028610605746507645,
      "step": 7483
    },
    {
      "epoch": 0.9793408031405905,
      "grad_norm": 6.938110828399658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03291194140911102,
      "logits/rejected": -0.05341607332229614,
      "logps/chosen": -376.4476318359375,
      "logps/rejected": -315.09429931640625,
      "loss": 0.6057,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5100772976875305,
      "rewards/margins": 0.49683114886283875,
      "rewards/rejected": -1.0069084167480469,
      "step": 7484
    },
    {
      "epoch": 0.9794716610779423,
      "grad_norm": 8.682291984558105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07417783886194229,
      "logits/rejected": -0.35519152879714966,
      "logps/chosen": -342.89654541015625,
      "logps/rejected": -356.1600036621094,
      "loss": 0.5398,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17984752357006073,
      "rewards/margins": 0.4961288571357727,
      "rewards/rejected": -0.6759763956069946,
      "step": 7485
    },
    {
      "epoch": 0.979602519015294,
      "grad_norm": 7.787806987762451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008776888251304626,
      "logits/rejected": -0.09583467245101929,
      "logps/chosen": -396.2575378417969,
      "logps/rejected": -486.1640319824219,
      "loss": 0.5152,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0797512531280518,
      "rewards/margins": 0.6945663094520569,
      "rewards/rejected": -1.7743176221847534,
      "step": 7486
    },
    {
      "epoch": 0.9797333769526457,
      "grad_norm": 9.108711242675781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09870322793722153,
      "logits/rejected": -0.09889285266399384,
      "logps/chosen": -488.33746337890625,
      "logps/rejected": -431.9629821777344,
      "loss": 0.7619,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14475110173225403,
      "rewards/margins": -5.29363751411438e-05,
      "rewards/rejected": -0.1446981430053711,
      "step": 7487
    },
    {
      "epoch": 0.9798642348899975,
      "grad_norm": 13.332606315612793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1427597999572754,
      "logits/rejected": -0.2414928674697876,
      "logps/chosen": -451.7419738769531,
      "logps/rejected": -488.0594482421875,
      "loss": 1.0158,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.090792179107666,
      "rewards/margins": -0.18815669417381287,
      "rewards/rejected": -0.9026353359222412,
      "step": 7488
    },
    {
      "epoch": 0.9799950928273493,
      "grad_norm": 12.080348014831543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2633291780948639,
      "logits/rejected": 0.0737452283501625,
      "logps/chosen": -349.5666198730469,
      "logps/rejected": -372.38043212890625,
      "loss": 1.0671,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.27136173844337463,
      "rewards/margins": 0.016306884586811066,
      "rewards/rejected": -0.2876685857772827,
      "step": 7489
    },
    {
      "epoch": 0.9801259507647011,
      "grad_norm": 8.442986488342285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18418022990226746,
      "logits/rejected": -0.007724776864051819,
      "logps/chosen": -474.3710021972656,
      "logps/rejected": -532.321533203125,
      "loss": 0.6181,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0075098127126693726,
      "rewards/margins": 0.7701543569564819,
      "rewards/rejected": -0.7626445293426514,
      "step": 7490
    },
    {
      "epoch": 0.9802568087020528,
      "grad_norm": 6.6239728927612305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2564907670021057,
      "logits/rejected": 0.19020269811153412,
      "logps/chosen": -488.3039855957031,
      "logps/rejected": -403.3856506347656,
      "loss": 0.3636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.38522303104400635,
      "rewards/margins": 1.6677043437957764,
      "rewards/rejected": -1.2824814319610596,
      "step": 7491
    },
    {
      "epoch": 0.9803876666394046,
      "grad_norm": 4.942065715789795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24036352336406708,
      "logits/rejected": -0.2511917054653168,
      "logps/chosen": -385.6026916503906,
      "logps/rejected": -366.3386535644531,
      "loss": 0.3402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5102875232696533,
      "rewards/margins": 1.0935920476913452,
      "rewards/rejected": -1.6038795709609985,
      "step": 7492
    },
    {
      "epoch": 0.9805185245767564,
      "grad_norm": 10.893728256225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01780267059803009,
      "logits/rejected": 0.030971046537160873,
      "logps/chosen": -397.28619384765625,
      "logps/rejected": -515.2901611328125,
      "loss": 0.8387,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.255863755941391,
      "rewards/margins": -0.1339794099330902,
      "rewards/rejected": -0.12188434600830078,
      "step": 7493
    },
    {
      "epoch": 0.9806493825141082,
      "grad_norm": 4.148833274841309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03211842477321625,
      "logits/rejected": -0.18214267492294312,
      "logps/chosen": -565.445068359375,
      "logps/rejected": -455.4028015136719,
      "loss": 0.2708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40970784425735474,
      "rewards/margins": 1.4161113500595093,
      "rewards/rejected": -1.0064035654067993,
      "step": 7494
    },
    {
      "epoch": 0.9807802404514598,
      "grad_norm": 5.947285175323486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2503420114517212,
      "logits/rejected": 0.30188262462615967,
      "logps/chosen": -448.2192687988281,
      "logps/rejected": -470.9989013671875,
      "loss": 0.3607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13702696561813354,
      "rewards/margins": 1.0289424657821655,
      "rewards/rejected": -0.8919155597686768,
      "step": 7495
    },
    {
      "epoch": 0.9809110983888116,
      "grad_norm": 7.264775276184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13103674352169037,
      "logits/rejected": -0.3027847409248352,
      "logps/chosen": -451.58367919921875,
      "logps/rejected": -466.30908203125,
      "loss": 0.3258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1202346086502075,
      "rewards/margins": 1.7286484241485596,
      "rewards/rejected": -2.8488831520080566,
      "step": 7496
    },
    {
      "epoch": 0.9810419563261634,
      "grad_norm": 6.307861328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012495279312133789,
      "logits/rejected": 0.07113998383283615,
      "logps/chosen": -368.4323425292969,
      "logps/rejected": -554.0924682617188,
      "loss": 0.355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0643707513809204,
      "rewards/margins": 1.4587469100952148,
      "rewards/rejected": -2.523117780685425,
      "step": 7497
    },
    {
      "epoch": 0.9811728142635152,
      "grad_norm": 7.795148849487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04244493693113327,
      "logits/rejected": -0.15059134364128113,
      "logps/chosen": -634.511962890625,
      "logps/rejected": -579.3168334960938,
      "loss": 0.5559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.33729496598243713,
      "rewards/margins": 0.74941486120224,
      "rewards/rejected": -0.41211986541748047,
      "step": 7498
    },
    {
      "epoch": 0.9813036722008669,
      "grad_norm": 9.398808479309082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33729586005210876,
      "logits/rejected": 0.12501764297485352,
      "logps/chosen": -408.35076904296875,
      "logps/rejected": -406.681640625,
      "loss": 0.557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15425890684127808,
      "rewards/margins": 0.7632001638412476,
      "rewards/rejected": -0.6089413166046143,
      "step": 7499
    },
    {
      "epoch": 0.9814345301382187,
      "grad_norm": 8.243711471557617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08867190033197403,
      "logits/rejected": -0.23358586430549622,
      "logps/chosen": -762.13134765625,
      "logps/rejected": -448.7191162109375,
      "loss": 0.4701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5747215747833252,
      "rewards/margins": 0.8154448866844177,
      "rewards/rejected": -1.3901664018630981,
      "step": 7500
    },
    {
      "epoch": 0.9815653880755705,
      "grad_norm": 5.847175598144531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11848710477352142,
      "logits/rejected": -0.018715664744377136,
      "logps/chosen": -421.4958801269531,
      "logps/rejected": -456.29449462890625,
      "loss": 0.4167,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28978443145751953,
      "rewards/margins": 1.14437735080719,
      "rewards/rejected": -0.8545928597450256,
      "step": 7501
    },
    {
      "epoch": 0.9816962460129223,
      "grad_norm": 4.616623401641846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18345095217227936,
      "logits/rejected": -0.14206139743328094,
      "logps/chosen": -304.4836120605469,
      "logps/rejected": -384.6999816894531,
      "loss": 0.4092,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10318546742200851,
      "rewards/margins": 1.1782104969024658,
      "rewards/rejected": -1.2813959121704102,
      "step": 7502
    },
    {
      "epoch": 0.981827103950274,
      "grad_norm": 11.052711486816406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016944514587521553,
      "logits/rejected": -0.019377559423446655,
      "logps/chosen": -403.3253479003906,
      "logps/rejected": -360.2648010253906,
      "loss": 1.011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8931047916412354,
      "rewards/margins": 0.03081798553466797,
      "rewards/rejected": -0.9239227175712585,
      "step": 7503
    },
    {
      "epoch": 0.9819579618876257,
      "grad_norm": 6.72404146194458,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08996529877185822,
      "logits/rejected": 0.01468794047832489,
      "logps/chosen": -363.8907775878906,
      "logps/rejected": -342.8726501464844,
      "loss": 0.4985,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7238010168075562,
      "rewards/margins": 1.1549324989318848,
      "rewards/rejected": -1.8787333965301514,
      "step": 7504
    },
    {
      "epoch": 0.9820888198249775,
      "grad_norm": 3.941988229751587,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14267608523368835,
      "logits/rejected": -0.06505472958087921,
      "logps/chosen": -377.53857421875,
      "logps/rejected": -357.476318359375,
      "loss": 0.2264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05489901453256607,
      "rewards/margins": 1.5211507081985474,
      "rewards/rejected": -1.5760498046875,
      "step": 7505
    },
    {
      "epoch": 0.9822196777623293,
      "grad_norm": 6.729527950286865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12922276556491852,
      "logits/rejected": -0.08271616697311401,
      "logps/chosen": -521.7353515625,
      "logps/rejected": -523.9161376953125,
      "loss": 0.4628,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.394900918006897,
      "rewards/margins": 0.8874521255493164,
      "rewards/rejected": -0.49255120754241943,
      "step": 7506
    },
    {
      "epoch": 0.982350535699681,
      "grad_norm": 9.364575386047363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11195846647024155,
      "logits/rejected": -0.18841542303562164,
      "logps/chosen": -392.526123046875,
      "logps/rejected": -337.3777770996094,
      "loss": 0.826,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9639622569084167,
      "rewards/margins": 0.37259039282798767,
      "rewards/rejected": -1.336552619934082,
      "step": 7507
    },
    {
      "epoch": 0.9824813936370328,
      "grad_norm": 8.169820785522461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17365074157714844,
      "logits/rejected": 0.04121383652091026,
      "logps/chosen": -292.6480712890625,
      "logps/rejected": -329.35491943359375,
      "loss": 0.664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.328338086605072,
      "rewards/margins": 0.5651651620864868,
      "rewards/rejected": -0.8935031890869141,
      "step": 7508
    },
    {
      "epoch": 0.9826122515743846,
      "grad_norm": 6.991896152496338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06147190183401108,
      "logits/rejected": 0.10135436803102493,
      "logps/chosen": -510.48114013671875,
      "logps/rejected": -458.6547546386719,
      "loss": 0.5744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3951677680015564,
      "rewards/margins": 0.4975461959838867,
      "rewards/rejected": -0.8927140235900879,
      "step": 7509
    },
    {
      "epoch": 0.9827431095117363,
      "grad_norm": 9.624642372131348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21894490718841553,
      "logits/rejected": -0.09314695745706558,
      "logps/chosen": -529.2208862304688,
      "logps/rejected": -496.56475830078125,
      "loss": 0.7077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.933303713798523,
      "rewards/margins": 0.3418308198451996,
      "rewards/rejected": -1.2751344442367554,
      "step": 7510
    },
    {
      "epoch": 0.9828739674490881,
      "grad_norm": 7.691706657409668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2793557941913605,
      "logits/rejected": -0.16022855043411255,
      "logps/chosen": -520.0401000976562,
      "logps/rejected": -468.9198913574219,
      "loss": 0.5277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5930494070053101,
      "rewards/margins": 1.1228084564208984,
      "rewards/rejected": -1.715857982635498,
      "step": 7511
    },
    {
      "epoch": 0.9830048253864399,
      "grad_norm": 5.8746018409729,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21476498246192932,
      "logits/rejected": -0.2644179165363312,
      "logps/chosen": -427.9635009765625,
      "logps/rejected": -440.9737548828125,
      "loss": 0.3688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2507798969745636,
      "rewards/margins": 1.380668044090271,
      "rewards/rejected": -1.1298881769180298,
      "step": 7512
    },
    {
      "epoch": 0.9831356833237916,
      "grad_norm": 7.352372169494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11081989109516144,
      "logits/rejected": -0.15122205018997192,
      "logps/chosen": -567.7451782226562,
      "logps/rejected": -436.89361572265625,
      "loss": 0.4184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.39079129695892334,
      "rewards/margins": 1.2962777614593506,
      "rewards/rejected": -0.9054862856864929,
      "step": 7513
    },
    {
      "epoch": 0.9832665412611433,
      "grad_norm": 8.788378715515137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09513714164495468,
      "logits/rejected": -0.3605262041091919,
      "logps/chosen": -428.2076416015625,
      "logps/rejected": -330.4561462402344,
      "loss": 0.637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0502816438674927,
      "rewards/margins": 0.9918569326400757,
      "rewards/rejected": -2.0421388149261475,
      "step": 7514
    },
    {
      "epoch": 0.9833973991984951,
      "grad_norm": 6.562192440032959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30218639969825745,
      "logits/rejected": -0.19930286705493927,
      "logps/chosen": -364.7085876464844,
      "logps/rejected": -409.5854187011719,
      "loss": 0.5328,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013629533350467682,
      "rewards/margins": 0.7447611093521118,
      "rewards/rejected": -0.7311314940452576,
      "step": 7515
    },
    {
      "epoch": 0.9835282571358469,
      "grad_norm": 3.441232204437256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.116083063185215,
      "logits/rejected": -0.17702996730804443,
      "logps/chosen": -392.5689697265625,
      "logps/rejected": -368.5143127441406,
      "loss": 0.2062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29949951171875,
      "rewards/margins": 1.629797339439392,
      "rewards/rejected": -1.9292969703674316,
      "step": 7516
    },
    {
      "epoch": 0.9836591150731987,
      "grad_norm": 5.702877998352051,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1113777756690979,
      "logits/rejected": -0.34645476937294006,
      "logps/chosen": -461.80352783203125,
      "logps/rejected": -474.29656982421875,
      "loss": 0.3896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07011795043945312,
      "rewards/margins": 1.816909670829773,
      "rewards/rejected": -1.7467918395996094,
      "step": 7517
    },
    {
      "epoch": 0.9837899730105504,
      "grad_norm": 8.81908893585205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21804487705230713,
      "logits/rejected": -0.19645705819129944,
      "logps/chosen": -439.3249206542969,
      "logps/rejected": -381.80572509765625,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.908639132976532,
      "rewards/margins": 1.1949909925460815,
      "rewards/rejected": -2.1036300659179688,
      "step": 7518
    },
    {
      "epoch": 0.9839208309479022,
      "grad_norm": 5.357447624206543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32875335216522217,
      "logits/rejected": -0.31061992049217224,
      "logps/chosen": -319.0087585449219,
      "logps/rejected": -383.990234375,
      "loss": 0.3667,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16906259953975677,
      "rewards/margins": 1.0431272983551025,
      "rewards/rejected": -1.2121899127960205,
      "step": 7519
    },
    {
      "epoch": 0.984051688885254,
      "grad_norm": 4.291853427886963,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03267398476600647,
      "logits/rejected": -0.1688809096813202,
      "logps/chosen": -547.1596069335938,
      "logps/rejected": -532.12744140625,
      "loss": 0.188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1510261744260788,
      "rewards/margins": 1.94808828830719,
      "rewards/rejected": -1.7970621585845947,
      "step": 7520
    },
    {
      "epoch": 0.9841825468226058,
      "grad_norm": 4.453023433685303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005342632532119751,
      "logits/rejected": -0.17275157570838928,
      "logps/chosen": -457.4114990234375,
      "logps/rejected": -453.16143798828125,
      "loss": 0.3374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11652259528636932,
      "rewards/margins": 1.7491791248321533,
      "rewards/rejected": -1.8657015562057495,
      "step": 7521
    },
    {
      "epoch": 0.9843134047599574,
      "grad_norm": 8.060099601745605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0724220871925354,
      "logits/rejected": -0.13258329033851624,
      "logps/chosen": -486.9053649902344,
      "logps/rejected": -499.6876220703125,
      "loss": 0.7405,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5708398818969727,
      "rewards/margins": 0.11936972290277481,
      "rewards/rejected": -0.6902095675468445,
      "step": 7522
    },
    {
      "epoch": 0.9844442626973092,
      "grad_norm": 4.634377956390381,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07434958219528198,
      "logits/rejected": -0.383314311504364,
      "logps/chosen": -474.7254638671875,
      "logps/rejected": -733.5618896484375,
      "loss": 0.1842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37918147444725037,
      "rewards/margins": 2.3995988368988037,
      "rewards/rejected": -2.778780221939087,
      "step": 7523
    },
    {
      "epoch": 0.984575120634661,
      "grad_norm": 4.589515209197998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11521880328655243,
      "logits/rejected": -0.1726161539554596,
      "logps/chosen": -472.58428955078125,
      "logps/rejected": -480.533447265625,
      "loss": 0.391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02475862205028534,
      "rewards/margins": 0.9920058250427246,
      "rewards/rejected": -1.0167644023895264,
      "step": 7524
    },
    {
      "epoch": 0.9847059785720128,
      "grad_norm": 8.627349853515625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44518929719924927,
      "logits/rejected": -0.4302685856819153,
      "logps/chosen": -439.408203125,
      "logps/rejected": -390.64794921875,
      "loss": 0.671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6603032946586609,
      "rewards/margins": 0.11660066246986389,
      "rewards/rejected": -0.7769038677215576,
      "step": 7525
    },
    {
      "epoch": 0.9848368365093645,
      "grad_norm": 7.418137550354004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17738758027553558,
      "logits/rejected": -0.08378347754478455,
      "logps/chosen": -450.4950256347656,
      "logps/rejected": -385.1700439453125,
      "loss": 0.4632,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.553593099117279,
      "rewards/margins": 1.4172403812408447,
      "rewards/rejected": -1.970833420753479,
      "step": 7526
    },
    {
      "epoch": 0.9849676944467163,
      "grad_norm": 6.135590553283691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.052994512021541595,
      "logits/rejected": -0.11565278470516205,
      "logps/chosen": -305.6142578125,
      "logps/rejected": -268.861083984375,
      "loss": 0.5419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7757502794265747,
      "rewards/margins": 0.5539464354515076,
      "rewards/rejected": -1.329696774482727,
      "step": 7527
    },
    {
      "epoch": 0.9850985523840681,
      "grad_norm": 8.770426750183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24714556336402893,
      "logits/rejected": -0.342995822429657,
      "logps/chosen": -478.99957275390625,
      "logps/rejected": -482.82537841796875,
      "loss": 0.5977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2772623300552368,
      "rewards/margins": 1.0109213590621948,
      "rewards/rejected": -1.288183569908142,
      "step": 7528
    },
    {
      "epoch": 0.9852294103214199,
      "grad_norm": 5.259671211242676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032683126628398895,
      "logits/rejected": -0.22405202686786652,
      "logps/chosen": -406.68035888671875,
      "logps/rejected": -370.36041259765625,
      "loss": 0.3706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7500154972076416,
      "rewards/margins": 0.9850311279296875,
      "rewards/rejected": -1.735046625137329,
      "step": 7529
    },
    {
      "epoch": 0.9853602682587715,
      "grad_norm": 7.757032871246338,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08231865614652634,
      "logits/rejected": 0.11351534724235535,
      "logps/chosen": -420.76898193359375,
      "logps/rejected": -489.4810791015625,
      "loss": 0.4887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9890823364257812,
      "rewards/margins": 1.0892913341522217,
      "rewards/rejected": -2.078373670578003,
      "step": 7530
    },
    {
      "epoch": 0.9854911261961233,
      "grad_norm": 8.338006973266602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.075943224132061,
      "logits/rejected": -0.10445510596036911,
      "logps/chosen": -362.4814147949219,
      "logps/rejected": -417.26898193359375,
      "loss": 0.5645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47903043031692505,
      "rewards/margins": 0.5427284240722656,
      "rewards/rejected": -1.0217589139938354,
      "step": 7531
    },
    {
      "epoch": 0.9856219841334751,
      "grad_norm": 6.995889663696289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06381289660930634,
      "logits/rejected": 0.0026543252170085907,
      "logps/chosen": -336.19464111328125,
      "logps/rejected": -413.4124755859375,
      "loss": 0.4611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0163698196411133,
      "rewards/margins": 1.034629225730896,
      "rewards/rejected": -2.050999164581299,
      "step": 7532
    },
    {
      "epoch": 0.9857528420708268,
      "grad_norm": 15.583702087402344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0888976976275444,
      "logits/rejected": -0.12912529706954956,
      "logps/chosen": -447.8193359375,
      "logps/rejected": -507.8267822265625,
      "loss": 0.7298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1756383180618286,
      "rewards/margins": 0.7033510208129883,
      "rewards/rejected": -1.8789894580841064,
      "step": 7533
    },
    {
      "epoch": 0.9858837000081786,
      "grad_norm": 5.1063408851623535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2790631055831909,
      "logits/rejected": -0.3360911011695862,
      "logps/chosen": -446.3759765625,
      "logps/rejected": -386.8382568359375,
      "loss": 0.3222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5809537172317505,
      "rewards/margins": 1.6810357570648193,
      "rewards/rejected": -1.1000818014144897,
      "step": 7534
    },
    {
      "epoch": 0.9860145579455304,
      "grad_norm": 4.236240386962891,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34932857751846313,
      "logits/rejected": -0.5450969338417053,
      "logps/chosen": -301.95672607421875,
      "logps/rejected": -389.4377746582031,
      "loss": 0.347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.228009581565857,
      "rewards/margins": 1.2092894315719604,
      "rewards/rejected": -2.4372990131378174,
      "step": 7535
    },
    {
      "epoch": 0.9861454158828822,
      "grad_norm": 7.493476867675781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17168796062469482,
      "logits/rejected": 0.025331657379865646,
      "logps/chosen": -685.7887573242188,
      "logps/rejected": -587.7176513671875,
      "loss": 0.4495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18132340908050537,
      "rewards/margins": 0.8448635935783386,
      "rewards/rejected": -1.0261869430541992,
      "step": 7536
    },
    {
      "epoch": 0.9862762738202339,
      "grad_norm": 10.06167984008789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13319173455238342,
      "logits/rejected": 0.21702174842357635,
      "logps/chosen": -674.1197509765625,
      "logps/rejected": -865.3692016601562,
      "loss": 0.8714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38448071479797363,
      "rewards/margins": 0.16452175378799438,
      "rewards/rejected": -0.5490024089813232,
      "step": 7537
    },
    {
      "epoch": 0.9864071317575857,
      "grad_norm": 8.170690536499023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22922202944755554,
      "logits/rejected": -0.1818510890007019,
      "logps/chosen": -415.08740234375,
      "logps/rejected": -418.2684631347656,
      "loss": 0.494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7033689618110657,
      "rewards/margins": 1.0506107807159424,
      "rewards/rejected": -1.7539796829223633,
      "step": 7538
    },
    {
      "epoch": 0.9865379896949374,
      "grad_norm": 7.625425338745117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.056726884096860886,
      "logits/rejected": -0.10548534989356995,
      "logps/chosen": -521.7199096679688,
      "logps/rejected": -542.4413452148438,
      "loss": 0.4473,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06484432518482208,
      "rewards/margins": 1.3673970699310303,
      "rewards/rejected": -1.4322413206100464,
      "step": 7539
    },
    {
      "epoch": 0.9866688476322892,
      "grad_norm": 5.330674648284912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14563100039958954,
      "logits/rejected": 0.015042003244161606,
      "logps/chosen": -509.10491943359375,
      "logps/rejected": -516.4952392578125,
      "loss": 0.2983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6988398432731628,
      "rewards/margins": 1.48565673828125,
      "rewards/rejected": -0.7868170142173767,
      "step": 7540
    },
    {
      "epoch": 0.9867997055696409,
      "grad_norm": 11.519933700561523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08393505215644836,
      "logits/rejected": -0.2548588216304779,
      "logps/chosen": -483.1116638183594,
      "logps/rejected": -514.9959106445312,
      "loss": 0.8036,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.031187891960144,
      "rewards/margins": 0.8147592544555664,
      "rewards/rejected": -1.8459471464157104,
      "step": 7541
    },
    {
      "epoch": 0.9869305635069927,
      "grad_norm": 6.747495174407959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03699621185660362,
      "logits/rejected": 0.022364091128110886,
      "logps/chosen": -443.0483703613281,
      "logps/rejected": -411.6039123535156,
      "loss": 0.6235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7538867592811584,
      "rewards/margins": 0.3251078128814697,
      "rewards/rejected": -1.078994631767273,
      "step": 7542
    },
    {
      "epoch": 0.9870614214443445,
      "grad_norm": 6.074503421783447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16522163152694702,
      "logits/rejected": 0.053861312568187714,
      "logps/chosen": -456.1646728515625,
      "logps/rejected": -319.2176818847656,
      "loss": 0.4346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6557173132896423,
      "rewards/margins": 0.7613807916641235,
      "rewards/rejected": -1.417098045349121,
      "step": 7543
    },
    {
      "epoch": 0.9871922793816963,
      "grad_norm": 6.6367878913879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01696889102458954,
      "logits/rejected": -0.23609109222888947,
      "logps/chosen": -344.5122375488281,
      "logps/rejected": -431.580810546875,
      "loss": 0.5686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4121946692466736,
      "rewards/margins": 0.8044386506080627,
      "rewards/rejected": -1.2166332006454468,
      "step": 7544
    },
    {
      "epoch": 0.987323137319048,
      "grad_norm": 9.062310218811035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048866819590330124,
      "logits/rejected": -0.1346408724784851,
      "logps/chosen": -386.7805480957031,
      "logps/rejected": -485.8304443359375,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7256170511245728,
      "rewards/margins": 0.8131533265113831,
      "rewards/rejected": -1.538770318031311,
      "step": 7545
    },
    {
      "epoch": 0.9874539952563998,
      "grad_norm": 10.544988632202148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035011522471904755,
      "logits/rejected": -0.3311503231525421,
      "logps/chosen": -537.398681640625,
      "logps/rejected": -384.1637268066406,
      "loss": 0.6271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7821630835533142,
      "rewards/margins": 0.776690661907196,
      "rewards/rejected": -1.5588537454605103,
      "step": 7546
    },
    {
      "epoch": 0.9875848531937516,
      "grad_norm": 16.140729904174805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17889976501464844,
      "logits/rejected": -0.05873074382543564,
      "logps/chosen": -686.3961791992188,
      "logps/rejected": -389.8494873046875,
      "loss": 1.1142,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7766767740249634,
      "rewards/margins": 0.08003583550453186,
      "rewards/rejected": -0.8567126989364624,
      "step": 7547
    },
    {
      "epoch": 0.9877157111311033,
      "grad_norm": 4.023625373840332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1510239988565445,
      "logits/rejected": -0.11650126427412033,
      "logps/chosen": -711.2255859375,
      "logps/rejected": -614.8212890625,
      "loss": 0.2149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8255099654197693,
      "rewards/margins": 1.7760952711105347,
      "rewards/rejected": -0.9505851864814758,
      "step": 7548
    },
    {
      "epoch": 0.987846569068455,
      "grad_norm": 8.819842338562012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14769376814365387,
      "logits/rejected": -0.28746888041496277,
      "logps/chosen": -422.57122802734375,
      "logps/rejected": -371.91021728515625,
      "loss": 0.7161,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.809548020362854,
      "rewards/margins": 0.356628954410553,
      "rewards/rejected": -1.1661771535873413,
      "step": 7549
    },
    {
      "epoch": 0.9879774270058068,
      "grad_norm": 7.84552001953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22431840002536774,
      "logits/rejected": 0.06410528719425201,
      "logps/chosen": -630.1052856445312,
      "logps/rejected": -555.2296142578125,
      "loss": 0.5315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5679056644439697,
      "rewards/margins": 0.7807811498641968,
      "rewards/rejected": -1.348686695098877,
      "step": 7550
    },
    {
      "epoch": 0.9881082849431586,
      "grad_norm": 4.812210559844971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1609136313199997,
      "logits/rejected": -0.055031437426805496,
      "logps/chosen": -515.688720703125,
      "logps/rejected": -421.45208740234375,
      "loss": 0.324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05734997242689133,
      "rewards/margins": 1.6457895040512085,
      "rewards/rejected": -1.5884394645690918,
      "step": 7551
    },
    {
      "epoch": 0.9882391428805103,
      "grad_norm": 5.908181667327881,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03130985051393509,
      "logits/rejected": -0.14495262503623962,
      "logps/chosen": -642.28125,
      "logps/rejected": -617.328857421875,
      "loss": 0.3064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22001686692237854,
      "rewards/margins": 1.3729296922683716,
      "rewards/rejected": -1.1529128551483154,
      "step": 7552
    },
    {
      "epoch": 0.9883700008178621,
      "grad_norm": 7.508233547210693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08454819023609161,
      "logits/rejected": 0.10284557938575745,
      "logps/chosen": -574.2225952148438,
      "logps/rejected": -557.018310546875,
      "loss": 0.2694,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3667628765106201,
      "rewards/margins": 2.045987129211426,
      "rewards/rejected": -2.412750005722046,
      "step": 7553
    },
    {
      "epoch": 0.9885008587552139,
      "grad_norm": 10.352219581604004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017985669896006584,
      "logits/rejected": -0.023529335856437683,
      "logps/chosen": -574.3560180664062,
      "logps/rejected": -642.6546020507812,
      "loss": 0.6351,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023972898721694946,
      "rewards/margins": 0.8250318765640259,
      "rewards/rejected": -0.8010589480400085,
      "step": 7554
    },
    {
      "epoch": 0.9886317166925657,
      "grad_norm": 4.364593505859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07946344465017319,
      "logits/rejected": -0.1039315015077591,
      "logps/chosen": -429.770263671875,
      "logps/rejected": -350.85491943359375,
      "loss": 0.276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3474905490875244,
      "rewards/margins": 1.767358422279358,
      "rewards/rejected": -1.419867753982544,
      "step": 7555
    },
    {
      "epoch": 0.9887625746299173,
      "grad_norm": 7.764747619628906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013104408979415894,
      "logits/rejected": 0.16965267062187195,
      "logps/chosen": -475.90972900390625,
      "logps/rejected": -471.6759033203125,
      "loss": 0.408,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.054851770401001,
      "rewards/margins": 1.3676942586898804,
      "rewards/rejected": -2.422545909881592,
      "step": 7556
    },
    {
      "epoch": 0.9888934325672691,
      "grad_norm": 7.44568395614624,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2367737889289856,
      "logits/rejected": -0.34016504883766174,
      "logps/chosen": -338.0647277832031,
      "logps/rejected": -289.56982421875,
      "loss": 0.5086,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6827089786529541,
      "rewards/margins": 1.0588419437408447,
      "rewards/rejected": -1.7415508031845093,
      "step": 7557
    },
    {
      "epoch": 0.9890242905046209,
      "grad_norm": 8.237037658691406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3241649866104126,
      "logits/rejected": -0.5085351467132568,
      "logps/chosen": -660.932861328125,
      "logps/rejected": -311.60308837890625,
      "loss": 0.9205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5313021540641785,
      "rewards/margins": 0.20047667622566223,
      "rewards/rejected": -0.7317786812782288,
      "step": 7558
    },
    {
      "epoch": 0.9891551484419727,
      "grad_norm": 8.415572166442871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0707269012928009,
      "logits/rejected": -0.10879838466644287,
      "logps/chosen": -446.1212463378906,
      "logps/rejected": -458.0686950683594,
      "loss": 0.5324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3433553874492645,
      "rewards/margins": 0.9813578724861145,
      "rewards/rejected": -1.324713110923767,
      "step": 7559
    },
    {
      "epoch": 0.9892860063793244,
      "grad_norm": 7.58537483215332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09120037406682968,
      "logits/rejected": -0.11714620888233185,
      "logps/chosen": -418.32940673828125,
      "logps/rejected": -445.6270751953125,
      "loss": 0.5242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010875299572944641,
      "rewards/margins": 0.9156280755996704,
      "rewards/rejected": -0.9265034198760986,
      "step": 7560
    },
    {
      "epoch": 0.9894168643166762,
      "grad_norm": 10.18525505065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09288519620895386,
      "logits/rejected": 0.049039557576179504,
      "logps/chosen": -643.0939331054688,
      "logps/rejected": -722.3606567382812,
      "loss": 0.8375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40381211042404175,
      "rewards/margins": 0.7533072829246521,
      "rewards/rejected": -1.1571193933486938,
      "step": 7561
    },
    {
      "epoch": 0.989547722254028,
      "grad_norm": 6.095245361328125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12186944484710693,
      "logits/rejected": -0.059645816683769226,
      "logps/chosen": -457.04339599609375,
      "logps/rejected": -452.34857177734375,
      "loss": 0.5455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20910221338272095,
      "rewards/margins": 1.0713814496994019,
      "rewards/rejected": -1.280483603477478,
      "step": 7562
    },
    {
      "epoch": 0.9896785801913798,
      "grad_norm": 6.128969669342041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1731150895357132,
      "logits/rejected": -0.2163742482662201,
      "logps/chosen": -275.36572265625,
      "logps/rejected": -307.8485107421875,
      "loss": 0.5446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8683056235313416,
      "rewards/margins": 0.5128181576728821,
      "rewards/rejected": -1.3811237812042236,
      "step": 7563
    },
    {
      "epoch": 0.9898094381287315,
      "grad_norm": 11.86763858795166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10535161942243576,
      "logits/rejected": 0.054308295249938965,
      "logps/chosen": -380.16473388671875,
      "logps/rejected": -356.7718200683594,
      "loss": 0.8904,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.46948763728141785,
      "rewards/margins": 0.13894161581993103,
      "rewards/rejected": -0.6084291338920593,
      "step": 7564
    },
    {
      "epoch": 0.9899402960660832,
      "grad_norm": 8.868593215942383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.147232785820961,
      "logits/rejected": -0.19194680452346802,
      "logps/chosen": -510.6962890625,
      "logps/rejected": -515.1056518554688,
      "loss": 0.4302,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18259547650814056,
      "rewards/margins": 1.4273680448532104,
      "rewards/rejected": -1.6099634170532227,
      "step": 7565
    },
    {
      "epoch": 0.990071154003435,
      "grad_norm": 7.282806396484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040917541831731796,
      "logits/rejected": 0.014862306416034698,
      "logps/chosen": -327.6563720703125,
      "logps/rejected": -312.01177978515625,
      "loss": 0.8153,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7252840399742126,
      "rewards/margins": 0.6095985174179077,
      "rewards/rejected": -1.3348826169967651,
      "step": 7566
    },
    {
      "epoch": 0.9902020119407868,
      "grad_norm": 6.663718223571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023052163422107697,
      "logits/rejected": -0.11811945587396622,
      "logps/chosen": -500.2504577636719,
      "logps/rejected": -482.9827880859375,
      "loss": 0.4308,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9175958633422852,
      "rewards/margins": 1.5841747522354126,
      "rewards/rejected": -2.501770496368408,
      "step": 7567
    },
    {
      "epoch": 0.9903328698781385,
      "grad_norm": 6.3502726554870605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22608374059200287,
      "logits/rejected": 0.09217479825019836,
      "logps/chosen": -407.0398254394531,
      "logps/rejected": -332.27459716796875,
      "loss": 0.5058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.757167398929596,
      "rewards/margins": 1.2379472255706787,
      "rewards/rejected": -1.9951145648956299,
      "step": 7568
    },
    {
      "epoch": 0.9904637278154903,
      "grad_norm": 8.653646469116211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33914363384246826,
      "logits/rejected": -0.3105032742023468,
      "logps/chosen": -349.8013916015625,
      "logps/rejected": -349.1418151855469,
      "loss": 0.7758,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18838027119636536,
      "rewards/margins": -0.03603688254952431,
      "rewards/rejected": -0.15234336256980896,
      "step": 7569
    },
    {
      "epoch": 0.9905945857528421,
      "grad_norm": 9.691336631774902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21006646752357483,
      "logits/rejected": -0.3789057433605194,
      "logps/chosen": -436.27227783203125,
      "logps/rejected": -433.5035705566406,
      "loss": 0.5454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15449070930480957,
      "rewards/margins": 1.0417652130126953,
      "rewards/rejected": -1.1962559223175049,
      "step": 7570
    },
    {
      "epoch": 0.9907254436901939,
      "grad_norm": 4.861032962799072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03940741717815399,
      "logits/rejected": 0.14063231647014618,
      "logps/chosen": -418.51837158203125,
      "logps/rejected": -436.64306640625,
      "loss": 0.3568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35012075304985046,
      "rewards/margins": 1.1534755229949951,
      "rewards/rejected": -1.5035964250564575,
      "step": 7571
    },
    {
      "epoch": 0.9908563016275456,
      "grad_norm": 4.3142218589782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04846511781215668,
      "logits/rejected": -0.0469699427485466,
      "logps/chosen": -505.3472900390625,
      "logps/rejected": -462.6983642578125,
      "loss": 0.17,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3254348635673523,
      "rewards/margins": 2.797672986984253,
      "rewards/rejected": -2.472238302230835,
      "step": 7572
    },
    {
      "epoch": 0.9909871595648974,
      "grad_norm": 11.550569534301758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08856567740440369,
      "logits/rejected": 0.08265373110771179,
      "logps/chosen": -436.3390197753906,
      "logps/rejected": -350.37884521484375,
      "loss": 0.963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9518982172012329,
      "rewards/margins": 0.3107060194015503,
      "rewards/rejected": -1.2626042366027832,
      "step": 7573
    },
    {
      "epoch": 0.9911180175022491,
      "grad_norm": 9.074779510498047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16901063919067383,
      "logits/rejected": -0.08373134583234787,
      "logps/chosen": -483.369384765625,
      "logps/rejected": -362.08941650390625,
      "loss": 0.4881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6156883239746094,
      "rewards/margins": 1.00312340259552,
      "rewards/rejected": -1.618811845779419,
      "step": 7574
    },
    {
      "epoch": 0.9912488754396008,
      "grad_norm": 4.8659257888793945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20047728717327118,
      "logits/rejected": -0.011232767254114151,
      "logps/chosen": -488.646728515625,
      "logps/rejected": -492.7457275390625,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8012294769287109,
      "rewards/margins": 1.6392920017242432,
      "rewards/rejected": -0.8380626440048218,
      "step": 7575
    },
    {
      "epoch": 0.9913797333769526,
      "grad_norm": 8.906665802001953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2849702835083008,
      "logits/rejected": -0.12874267995357513,
      "logps/chosen": -448.06768798828125,
      "logps/rejected": -489.4040832519531,
      "loss": 0.773,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6009852886199951,
      "rewards/margins": 0.39709681272506714,
      "rewards/rejected": -0.9980822205543518,
      "step": 7576
    },
    {
      "epoch": 0.9915105913143044,
      "grad_norm": 6.955283164978027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007563521154224873,
      "logits/rejected": -0.03457004204392433,
      "logps/chosen": -499.94012451171875,
      "logps/rejected": -498.1590270996094,
      "loss": 0.4332,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3824274241924286,
      "rewards/margins": 1.4199323654174805,
      "rewards/rejected": -1.037505030632019,
      "step": 7577
    },
    {
      "epoch": 0.9916414492516562,
      "grad_norm": 4.315802574157715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0653928816318512,
      "logits/rejected": 0.06817682832479477,
      "logps/chosen": -408.5970458984375,
      "logps/rejected": -347.4148864746094,
      "loss": 0.2536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.314810574054718,
      "rewards/margins": 1.572410225868225,
      "rewards/rejected": -1.2575997114181519,
      "step": 7578
    },
    {
      "epoch": 0.9917723071890079,
      "grad_norm": 4.476546764373779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12127862125635147,
      "logits/rejected": -0.04457114636898041,
      "logps/chosen": -548.0610961914062,
      "logps/rejected": -466.19219970703125,
      "loss": 0.3257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28516826033592224,
      "rewards/margins": 1.5028327703475952,
      "rewards/rejected": -1.217664361000061,
      "step": 7579
    },
    {
      "epoch": 0.9919031651263597,
      "grad_norm": 9.219989776611328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028398510068655014,
      "logits/rejected": 0.008077550679445267,
      "logps/chosen": -353.1223449707031,
      "logps/rejected": -407.13824462890625,
      "loss": 0.3953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03618239611387253,
      "rewards/margins": 1.2875527143478394,
      "rewards/rejected": -1.323735237121582,
      "step": 7580
    },
    {
      "epoch": 0.9920340230637115,
      "grad_norm": 11.010741233825684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.048595406115055084,
      "logits/rejected": -0.0804649218916893,
      "logps/chosen": -376.3543701171875,
      "logps/rejected": -449.266845703125,
      "loss": 1.0243,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.699593186378479,
      "rewards/margins": 0.11847138404846191,
      "rewards/rejected": -0.8180644512176514,
      "step": 7581
    },
    {
      "epoch": 0.9921648810010633,
      "grad_norm": 5.232708930969238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1386967897415161,
      "logits/rejected": -0.11999592185020447,
      "logps/chosen": -341.2533874511719,
      "logps/rejected": -385.198486328125,
      "loss": 0.3126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.469681978225708,
      "rewards/margins": 1.5416932106018066,
      "rewards/rejected": -1.0720112323760986,
      "step": 7582
    },
    {
      "epoch": 0.9922957389384149,
      "grad_norm": 11.586252212524414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22671252489089966,
      "logits/rejected": 0.01240365207195282,
      "logps/chosen": -586.5628051757812,
      "logps/rejected": -675.1097412109375,
      "loss": 0.6801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29250040650367737,
      "rewards/margins": 0.5605835914611816,
      "rewards/rejected": -0.8530840277671814,
      "step": 7583
    },
    {
      "epoch": 0.9924265968757667,
      "grad_norm": 8.612066268920898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23632705211639404,
      "logits/rejected": -0.2580787241458893,
      "logps/chosen": -369.9415283203125,
      "logps/rejected": -338.2522888183594,
      "loss": 0.6784,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.082370638847351,
      "rewards/margins": 0.5954108238220215,
      "rewards/rejected": -1.677781581878662,
      "step": 7584
    },
    {
      "epoch": 0.9925574548131185,
      "grad_norm": 5.188323020935059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09492435306310654,
      "logits/rejected": 0.033632539212703705,
      "logps/chosen": -558.9688720703125,
      "logps/rejected": -505.07568359375,
      "loss": 0.3938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2031286358833313,
      "rewards/margins": 1.6379262208938599,
      "rewards/rejected": -1.8410547971725464,
      "step": 7585
    },
    {
      "epoch": 0.9926883127504703,
      "grad_norm": 9.675424575805664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3441573679447174,
      "logits/rejected": 0.34779393672943115,
      "logps/chosen": -362.5426025390625,
      "logps/rejected": -460.35467529296875,
      "loss": 0.6728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2029193937778473,
      "rewards/margins": 1.014447569847107,
      "rewards/rejected": -0.811528205871582,
      "step": 7586
    },
    {
      "epoch": 0.992819170687822,
      "grad_norm": 8.058716773986816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1702701896429062,
      "logits/rejected": 0.14464154839515686,
      "logps/chosen": -399.2582702636719,
      "logps/rejected": -585.7371826171875,
      "loss": 0.6081,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10596580803394318,
      "rewards/margins": 0.8800514340400696,
      "rewards/rejected": -0.7740855813026428,
      "step": 7587
    },
    {
      "epoch": 0.9929500286251738,
      "grad_norm": 5.173795700073242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019387178122997284,
      "logits/rejected": -0.16276344656944275,
      "logps/chosen": -511.7652282714844,
      "logps/rejected": -580.7076416015625,
      "loss": 0.2402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.055642351508140564,
      "rewards/margins": 2.010289192199707,
      "rewards/rejected": -1.9546467065811157,
      "step": 7588
    },
    {
      "epoch": 0.9930808865625256,
      "grad_norm": 6.323868751525879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3667011559009552,
      "logits/rejected": 0.2024461179971695,
      "logps/chosen": -396.93804931640625,
      "logps/rejected": -338.81402587890625,
      "loss": 0.4482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05882032960653305,
      "rewards/margins": 0.8168493509292603,
      "rewards/rejected": -0.7580289840698242,
      "step": 7589
    },
    {
      "epoch": 0.9932117444998774,
      "grad_norm": 8.447834014892578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3083859086036682,
      "logits/rejected": -0.359910786151886,
      "logps/chosen": -478.9756164550781,
      "logps/rejected": -464.476318359375,
      "loss": 0.6155,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5627745389938354,
      "rewards/margins": 0.44395947456359863,
      "rewards/rejected": -2.0067338943481445,
      "step": 7590
    },
    {
      "epoch": 0.993342602437229,
      "grad_norm": 6.057772159576416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2166471928358078,
      "logits/rejected": -0.16468487679958344,
      "logps/chosen": -311.439208984375,
      "logps/rejected": -345.38995361328125,
      "loss": 0.3764,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5020915269851685,
      "rewards/margins": 1.1210039854049683,
      "rewards/rejected": -1.6230956315994263,
      "step": 7591
    },
    {
      "epoch": 0.9934734603745808,
      "grad_norm": 8.379435539245605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13289885222911835,
      "logits/rejected": -0.060541704297065735,
      "logps/chosen": -326.9358825683594,
      "logps/rejected": -374.9160461425781,
      "loss": 0.5192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5315834283828735,
      "rewards/margins": 1.0370992422103882,
      "rewards/rejected": -1.5686827898025513,
      "step": 7592
    },
    {
      "epoch": 0.9936043183119326,
      "grad_norm": 7.985056400299072,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055553123354911804,
      "logits/rejected": -0.036873266100883484,
      "logps/chosen": -568.6115112304688,
      "logps/rejected": -468.8964538574219,
      "loss": 0.5131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6654134392738342,
      "rewards/margins": 1.4369350671768188,
      "rewards/rejected": -2.102348566055298,
      "step": 7593
    },
    {
      "epoch": 0.9937351762492844,
      "grad_norm": 4.959327220916748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1287054717540741,
      "logits/rejected": -0.204936683177948,
      "logps/chosen": -346.39300537109375,
      "logps/rejected": -393.9866943359375,
      "loss": 0.3033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24039536714553833,
      "rewards/margins": 1.7610923051834106,
      "rewards/rejected": -1.520696997642517,
      "step": 7594
    },
    {
      "epoch": 0.9938660341866361,
      "grad_norm": 6.891501426696777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24756507575511932,
      "logits/rejected": 0.19784365594387054,
      "logps/chosen": -328.456787109375,
      "logps/rejected": -432.7461853027344,
      "loss": 0.4539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08254164457321167,
      "rewards/margins": 0.8608114719390869,
      "rewards/rejected": -0.9433531165122986,
      "step": 7595
    },
    {
      "epoch": 0.9939968921239879,
      "grad_norm": 5.53287935256958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4032166600227356,
      "logits/rejected": -0.2747722566127777,
      "logps/chosen": -459.83837890625,
      "logps/rejected": -338.57989501953125,
      "loss": 0.5542,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9626792669296265,
      "rewards/margins": 1.0602807998657227,
      "rewards/rejected": -2.0229601860046387,
      "step": 7596
    },
    {
      "epoch": 0.9941277500613397,
      "grad_norm": 8.260467529296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.036601871252059937,
      "logits/rejected": -0.07898266613483429,
      "logps/chosen": -455.097900390625,
      "logps/rejected": -495.4592590332031,
      "loss": 0.6575,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9467604160308838,
      "rewards/margins": 0.359269917011261,
      "rewards/rejected": -1.3060302734375,
      "step": 7597
    },
    {
      "epoch": 0.9942586079986914,
      "grad_norm": 6.881750583648682,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1199764609336853,
      "logits/rejected": 0.14357873797416687,
      "logps/chosen": -305.8981018066406,
      "logps/rejected": -411.35150146484375,
      "loss": 0.538,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6246258616447449,
      "rewards/margins": 0.9775872826576233,
      "rewards/rejected": -1.6022131443023682,
      "step": 7598
    },
    {
      "epoch": 0.9943894659360432,
      "grad_norm": 12.573953628540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10055765509605408,
      "logits/rejected": -0.2994372248649597,
      "logps/chosen": -434.34161376953125,
      "logps/rejected": -539.6133422851562,
      "loss": 0.8732,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.730633556842804,
      "rewards/margins": -0.08236903697252274,
      "rewards/rejected": -0.6482645273208618,
      "step": 7599
    },
    {
      "epoch": 0.994520323873395,
      "grad_norm": 7.930933475494385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03586575388908386,
      "logits/rejected": 0.015852918848395348,
      "logps/chosen": -447.1629943847656,
      "logps/rejected": -415.9682922363281,
      "loss": 0.5379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18205642700195312,
      "rewards/margins": 0.9350403547286987,
      "rewards/rejected": -0.7529839277267456,
      "step": 7600
    },
    {
      "epoch": 0.9946511818107467,
      "grad_norm": 4.64609432220459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014985859394073486,
      "logits/rejected": -0.058938659727573395,
      "logps/chosen": -457.0689697265625,
      "logps/rejected": -475.1159362792969,
      "loss": 0.2895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3924461603164673,
      "rewards/margins": 1.2618675231933594,
      "rewards/rejected": -0.8694214224815369,
      "step": 7601
    },
    {
      "epoch": 0.9947820397480984,
      "grad_norm": 8.543231964111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03246027231216431,
      "logits/rejected": -0.31728896498680115,
      "logps/chosen": -356.3690185546875,
      "logps/rejected": -332.3846435546875,
      "loss": 0.6376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7121977210044861,
      "rewards/margins": 0.7345415353775024,
      "rewards/rejected": -1.4467393159866333,
      "step": 7602
    },
    {
      "epoch": 0.9949128976854502,
      "grad_norm": 5.684285640716553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20796290040016174,
      "logits/rejected": -0.15592345595359802,
      "logps/chosen": -312.89642333984375,
      "logps/rejected": -350.490234375,
      "loss": 0.5023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06777039170265198,
      "rewards/margins": 0.8642971515655518,
      "rewards/rejected": -0.9320675134658813,
      "step": 7603
    },
    {
      "epoch": 0.995043755622802,
      "grad_norm": 9.815230369567871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043172724545001984,
      "logits/rejected": -0.14708152413368225,
      "logps/chosen": -512.3275146484375,
      "logps/rejected": -436.6160583496094,
      "loss": 0.6321,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.49423640966415405,
      "rewards/margins": 0.45543137192726135,
      "rewards/rejected": -0.949667751789093,
      "step": 7604
    },
    {
      "epoch": 0.9951746135601538,
      "grad_norm": 7.231756687164307,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047025490552186966,
      "logits/rejected": -0.27699998021125793,
      "logps/chosen": -732.90673828125,
      "logps/rejected": -490.481689453125,
      "loss": 0.4199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36175310611724854,
      "rewards/margins": 1.0784721374511719,
      "rewards/rejected": -1.4402252435684204,
      "step": 7605
    },
    {
      "epoch": 0.9953054714975055,
      "grad_norm": 6.021477222442627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01669144630432129,
      "logits/rejected": -0.019784804433584213,
      "logps/chosen": -481.53680419921875,
      "logps/rejected": -322.72052001953125,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24667586386203766,
      "rewards/margins": 1.291745662689209,
      "rewards/rejected": -1.5384215116500854,
      "step": 7606
    },
    {
      "epoch": 0.9954363294348573,
      "grad_norm": 8.897640228271484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054286062717437744,
      "logits/rejected": -0.215311661362648,
      "logps/chosen": -558.93603515625,
      "logps/rejected": -524.281005859375,
      "loss": 0.6794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2804466485977173,
      "rewards/margins": 0.4425908923149109,
      "rewards/rejected": -1.7230374813079834,
      "step": 7607
    },
    {
      "epoch": 0.9955671873722091,
      "grad_norm": 8.648575782775879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25025612115859985,
      "logits/rejected": -0.04632692039012909,
      "logps/chosen": -595.2613525390625,
      "logps/rejected": -651.4144287109375,
      "loss": 0.7962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2268731594085693,
      "rewards/margins": 0.5009593963623047,
      "rewards/rejected": -1.7278324365615845,
      "step": 7608
    },
    {
      "epoch": 0.9956980453095609,
      "grad_norm": 12.138490676879883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1874660700559616,
      "logits/rejected": -0.11449878662824631,
      "logps/chosen": -413.6451721191406,
      "logps/rejected": -637.1383056640625,
      "loss": 0.7394,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2742477059364319,
      "rewards/margins": 0.21079277992248535,
      "rewards/rejected": -0.48504045605659485,
      "step": 7609
    },
    {
      "epoch": 0.9958289032469125,
      "grad_norm": 8.382844924926758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021248940378427505,
      "logits/rejected": -0.1014653816819191,
      "logps/chosen": -549.1204833984375,
      "logps/rejected": -512.8846435546875,
      "loss": 0.6779,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0774337649345398,
      "rewards/margins": 0.5727792978286743,
      "rewards/rejected": -0.49534550309181213,
      "step": 7610
    },
    {
      "epoch": 0.9959597611842643,
      "grad_norm": 4.791844844818115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0999593436717987,
      "logits/rejected": 0.02632160484790802,
      "logps/chosen": -443.447998046875,
      "logps/rejected": -328.6794128417969,
      "loss": 0.3117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14429357647895813,
      "rewards/margins": 1.452608346939087,
      "rewards/rejected": -1.5969018936157227,
      "step": 7611
    },
    {
      "epoch": 0.9960906191216161,
      "grad_norm": 6.8108062744140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057644449174404144,
      "logits/rejected": 0.017977476119995117,
      "logps/chosen": -472.7877197265625,
      "logps/rejected": -412.40545654296875,
      "loss": 0.5571,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6202051043510437,
      "rewards/margins": 0.9371124505996704,
      "rewards/rejected": -1.5573176145553589,
      "step": 7612
    },
    {
      "epoch": 0.9962214770589679,
      "grad_norm": 6.221358776092529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28442108631134033,
      "logits/rejected": -0.15972918272018433,
      "logps/chosen": -303.5560607910156,
      "logps/rejected": -396.6623229980469,
      "loss": 0.3902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0819514989852905,
      "rewards/margins": 1.5545915365219116,
      "rewards/rejected": -2.6365432739257812,
      "step": 7613
    },
    {
      "epoch": 0.9963523349963196,
      "grad_norm": 12.180991172790527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42914560437202454,
      "logits/rejected": -0.4546803832054138,
      "logps/chosen": -379.2716979980469,
      "logps/rejected": -338.8600769042969,
      "loss": 0.8523,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1923803091049194,
      "rewards/margins": 0.5961697101593018,
      "rewards/rejected": -1.7885499000549316,
      "step": 7614
    },
    {
      "epoch": 0.9964831929336714,
      "grad_norm": 5.196313858032227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22962921857833862,
      "logits/rejected": -0.1253422051668167,
      "logps/chosen": -558.877685546875,
      "logps/rejected": -342.8310546875,
      "loss": 0.3569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5179738998413086,
      "rewards/margins": 1.4573488235473633,
      "rewards/rejected": -1.9753228425979614,
      "step": 7615
    },
    {
      "epoch": 0.9966140508710232,
      "grad_norm": 5.895669937133789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1356838047504425,
      "logits/rejected": 0.15090128779411316,
      "logps/chosen": -398.593505859375,
      "logps/rejected": -407.0101623535156,
      "loss": 0.3876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10242348164319992,
      "rewards/margins": 1.1267364025115967,
      "rewards/rejected": -1.024312973022461,
      "step": 7616
    },
    {
      "epoch": 0.996744908808375,
      "grad_norm": 7.785981178283691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06490346044301987,
      "logits/rejected": 0.05518895387649536,
      "logps/chosen": -319.80609130859375,
      "logps/rejected": -422.42437744140625,
      "loss": 0.5235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.057589106261730194,
      "rewards/margins": 0.8450345993041992,
      "rewards/rejected": -0.90262371301651,
      "step": 7617
    },
    {
      "epoch": 0.9968757667457266,
      "grad_norm": 4.979681491851807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1997045874595642,
      "logits/rejected": -0.10449333488941193,
      "logps/chosen": -540.2069091796875,
      "logps/rejected": -537.0823364257812,
      "loss": 0.2943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17721976339817047,
      "rewards/margins": 1.3421906232833862,
      "rewards/rejected": -1.1649707555770874,
      "step": 7618
    },
    {
      "epoch": 0.9970066246830784,
      "grad_norm": 10.175046920776367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12070588767528534,
      "logits/rejected": 0.05010214075446129,
      "logps/chosen": -399.31414794921875,
      "logps/rejected": -456.20562744140625,
      "loss": 1.063,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.862931489944458,
      "rewards/margins": -0.11003591120243073,
      "rewards/rejected": -0.7528955936431885,
      "step": 7619
    },
    {
      "epoch": 0.9971374826204302,
      "grad_norm": 8.659857749938965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25170713663101196,
      "logits/rejected": 0.04672255367040634,
      "logps/chosen": -577.1638793945312,
      "logps/rejected": -499.5761413574219,
      "loss": 0.7465,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.38657647371292114,
      "rewards/margins": 0.23350372910499573,
      "rewards/rejected": -0.6200801730155945,
      "step": 7620
    },
    {
      "epoch": 0.9972683405577819,
      "grad_norm": 10.396437644958496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22504587471485138,
      "logits/rejected": -0.29651564359664917,
      "logps/chosen": -313.9002990722656,
      "logps/rejected": -409.85595703125,
      "loss": 0.9598,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3184239864349365,
      "rewards/margins": -0.1352745145559311,
      "rewards/rejected": -1.1831495761871338,
      "step": 7621
    },
    {
      "epoch": 0.9973991984951337,
      "grad_norm": 8.591353416442871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14387325942516327,
      "logits/rejected": -0.39907532930374146,
      "logps/chosen": -435.1181335449219,
      "logps/rejected": -530.422607421875,
      "loss": 0.4434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7328190207481384,
      "rewards/margins": 1.1184122562408447,
      "rewards/rejected": -1.8512312173843384,
      "step": 7622
    },
    {
      "epoch": 0.9975300564324855,
      "grad_norm": 7.880405902862549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25231894850730896,
      "logits/rejected": -0.06634905189275742,
      "logps/chosen": -344.8710632324219,
      "logps/rejected": -417.7788391113281,
      "loss": 0.6615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0135418176651,
      "rewards/margins": 0.4645199179649353,
      "rewards/rejected": -1.4780616760253906,
      "step": 7623
    },
    {
      "epoch": 0.9976609143698373,
      "grad_norm": 12.032133102416992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0825033038854599,
      "logits/rejected": -0.32043927907943726,
      "logps/chosen": -463.0851745605469,
      "logps/rejected": -527.0892333984375,
      "loss": 0.747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2391834259033203,
      "rewards/margins": 0.27643394470214844,
      "rewards/rejected": -1.5156174898147583,
      "step": 7624
    },
    {
      "epoch": 0.997791772307189,
      "grad_norm": 6.254075050354004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1765802800655365,
      "logits/rejected": -0.32355666160583496,
      "logps/chosen": -458.80035400390625,
      "logps/rejected": -437.1790466308594,
      "loss": 0.3297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7249578833580017,
      "rewards/margins": 1.26573646068573,
      "rewards/rejected": -0.5407785177230835,
      "step": 7625
    },
    {
      "epoch": 0.9979226302445408,
      "grad_norm": 9.008219718933105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27482396364212036,
      "logits/rejected": -0.16979782283306122,
      "logps/chosen": -495.4965515136719,
      "logps/rejected": -402.65203857421875,
      "loss": 0.5075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1932292878627777,
      "rewards/margins": 1.0731977224349976,
      "rewards/rejected": -1.266426920890808,
      "step": 7626
    },
    {
      "epoch": 0.9980534881818925,
      "grad_norm": 5.574734210968018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12800158560276031,
      "logits/rejected": 0.018662799149751663,
      "logps/chosen": -391.3480224609375,
      "logps/rejected": -391.26239013671875,
      "loss": 0.3024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006100930273532867,
      "rewards/margins": 1.8307688236236572,
      "rewards/rejected": -1.8246679306030273,
      "step": 7627
    },
    {
      "epoch": 0.9981843461192443,
      "grad_norm": 7.943087100982666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011510573327541351,
      "logits/rejected": -0.08287711441516876,
      "logps/chosen": -412.7003479003906,
      "logps/rejected": -448.23223876953125,
      "loss": 0.6538,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5778014659881592,
      "rewards/margins": 0.27023088932037354,
      "rewards/rejected": -0.8480324149131775,
      "step": 7628
    },
    {
      "epoch": 0.998315204056596,
      "grad_norm": 8.655770301818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0494031086564064,
      "logits/rejected": -0.07033225893974304,
      "logps/chosen": -541.482666015625,
      "logps/rejected": -450.3981018066406,
      "loss": 0.5784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3278530240058899,
      "rewards/margins": 1.0136079788208008,
      "rewards/rejected": -1.3414610624313354,
      "step": 7629
    },
    {
      "epoch": 0.9984460619939478,
      "grad_norm": 11.359267234802246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2312770038843155,
      "logits/rejected": 0.21442526578903198,
      "logps/chosen": -514.205810546875,
      "logps/rejected": -478.5023193359375,
      "loss": 0.8514,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9660571813583374,
      "rewards/margins": 0.04558331519365311,
      "rewards/rejected": -1.0116405487060547,
      "step": 7630
    },
    {
      "epoch": 0.9985769199312996,
      "grad_norm": 5.729302883148193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06492815911769867,
      "logits/rejected": -0.37938567996025085,
      "logps/chosen": -545.3582763671875,
      "logps/rejected": -376.3365783691406,
      "loss": 0.3051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03610019385814667,
      "rewards/margins": 2.159912109375,
      "rewards/rejected": -2.123811960220337,
      "step": 7631
    },
    {
      "epoch": 0.9987077778686514,
      "grad_norm": 6.171191215515137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14289195835590363,
      "logits/rejected": -0.3007865846157074,
      "logps/chosen": -600.8255615234375,
      "logps/rejected": -424.54168701171875,
      "loss": 0.4191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.021904170513153076,
      "rewards/margins": 1.2787199020385742,
      "rewards/rejected": -1.300623893737793,
      "step": 7632
    },
    {
      "epoch": 0.9988386358060031,
      "grad_norm": 12.630667686462402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23470225930213928,
      "logits/rejected": -0.07121074199676514,
      "logps/chosen": -564.54736328125,
      "logps/rejected": -460.5189208984375,
      "loss": 0.8985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5389722585678101,
      "rewards/margins": 0.26360514760017395,
      "rewards/rejected": -0.8025774359703064,
      "step": 7633
    },
    {
      "epoch": 0.9989694937433549,
      "grad_norm": 8.153536796569824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24010205268859863,
      "logits/rejected": -0.3880446255207062,
      "logps/chosen": -534.1491088867188,
      "logps/rejected": -459.7062072753906,
      "loss": 0.4817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.196989044547081,
      "rewards/margins": 0.6575038433074951,
      "rewards/rejected": -0.8544929623603821,
      "step": 7634
    },
    {
      "epoch": 0.9991003516807067,
      "grad_norm": 8.319198608398438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06903322041034698,
      "logits/rejected": -0.008356545120477676,
      "logps/chosen": -547.1503295898438,
      "logps/rejected": -424.0007629394531,
      "loss": 0.5166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06980839371681213,
      "rewards/margins": 1.833272933959961,
      "rewards/rejected": -1.7634645700454712,
      "step": 7635
    },
    {
      "epoch": 0.9992312096180584,
      "grad_norm": 9.396063804626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13216142356395721,
      "logits/rejected": -0.018913712352514267,
      "logps/chosen": -500.2117919921875,
      "logps/rejected": -518.8797607421875,
      "loss": 0.8077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2853538393974304,
      "rewards/margins": 1.0870951414108276,
      "rewards/rejected": -1.3724489212036133,
      "step": 7636
    },
    {
      "epoch": 0.9993620675554101,
      "grad_norm": 7.8655829429626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21418774127960205,
      "logits/rejected": -0.07276903837919235,
      "logps/chosen": -418.1285400390625,
      "logps/rejected": -459.794921875,
      "loss": 0.4705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1707342118024826,
      "rewards/margins": 1.4147791862487793,
      "rewards/rejected": -1.2440448999404907,
      "step": 7637
    },
    {
      "epoch": 0.9994929254927619,
      "grad_norm": 15.280534744262695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2737753987312317,
      "logits/rejected": -0.10220834612846375,
      "logps/chosen": -492.0494689941406,
      "logps/rejected": -478.3504943847656,
      "loss": 0.9044,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2044505923986435,
      "rewards/margins": 0.5895254611968994,
      "rewards/rejected": -0.7939760088920593,
      "step": 7638
    },
    {
      "epoch": 0.9996237834301137,
      "grad_norm": 6.536489963531494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18151737749576569,
      "logits/rejected": -0.08789240568876266,
      "logps/chosen": -435.4515686035156,
      "logps/rejected": -441.2384033203125,
      "loss": 0.4121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07204990088939667,
      "rewards/margins": 1.0740461349487305,
      "rewards/rejected": -1.1460959911346436,
      "step": 7639
    },
    {
      "epoch": 0.9997546413674654,
      "grad_norm": 6.683962821960449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35844045877456665,
      "logits/rejected": -0.1379028707742691,
      "logps/chosen": -321.68756103515625,
      "logps/rejected": -388.46014404296875,
      "loss": 0.4725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.281642347574234,
      "rewards/margins": 1.0379527807235718,
      "rewards/rejected": -1.3195950984954834,
      "step": 7640
    },
    {
      "epoch": 0.9998854993048172,
      "grad_norm": 7.0827131271362305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14290371537208557,
      "logits/rejected": -0.4176368713378906,
      "logps/chosen": -291.3672180175781,
      "logps/rejected": -413.15374755859375,
      "loss": 0.5828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6925944089889526,
      "rewards/margins": 0.8879086971282959,
      "rewards/rejected": -1.580503225326538,
      "step": 7641
    },
    {
      "epoch": 1.0,
      "grad_norm": 5.107250690460205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04557402804493904,
      "logits/rejected": -0.07493440061807632,
      "logps/chosen": -502.6329650878906,
      "logps/rejected": -374.9247131347656,
      "loss": 0.2953,
      "rewards/accuracies": 0.8571428656578064,
      "rewards/chosen": 0.24532949924468994,
      "rewards/margins": 1.7804127931594849,
      "rewards/rejected": -1.5350834131240845,
      "step": 7642
    },
    {
      "epoch": 1.0001308579373518,
      "grad_norm": 4.299830436706543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09557723999023438,
      "logits/rejected": -0.07504232227802277,
      "logps/chosen": -401.9480895996094,
      "logps/rejected": -465.06207275390625,
      "loss": 0.2953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0856090784072876,
      "rewards/margins": 1.2953717708587646,
      "rewards/rejected": -2.3809807300567627,
      "step": 7643
    },
    {
      "epoch": 1.0002617158747036,
      "grad_norm": 8.206923484802246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08076970279216766,
      "logits/rejected": -0.13592496514320374,
      "logps/chosen": -442.8602294921875,
      "logps/rejected": -528.9133911132812,
      "loss": 0.5452,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17348690330982208,
      "rewards/margins": 0.8575634360313416,
      "rewards/rejected": -1.0310503244400024,
      "step": 7644
    },
    {
      "epoch": 1.0003925738120554,
      "grad_norm": 2.04135799407959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0845920741558075,
      "logits/rejected": -0.17023210227489471,
      "logps/chosen": -329.44000244140625,
      "logps/rejected": -352.8016662597656,
      "loss": 0.0986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.057311635464429855,
      "rewards/margins": 2.651695966720581,
      "rewards/rejected": -2.709007978439331,
      "step": 7645
    },
    {
      "epoch": 1.000523431749407,
      "grad_norm": 5.167677402496338,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20656926929950714,
      "logits/rejected": -0.14047996699810028,
      "logps/chosen": -555.9459228515625,
      "logps/rejected": -480.3643798828125,
      "loss": 0.3484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5690670609474182,
      "rewards/margins": 0.9853060245513916,
      "rewards/rejected": -1.5543729066848755,
      "step": 7646
    },
    {
      "epoch": 1.0006542896867587,
      "grad_norm": 7.564789772033691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15495164692401886,
      "logits/rejected": -0.33865106105804443,
      "logps/chosen": -531.07666015625,
      "logps/rejected": -552.3160400390625,
      "loss": 0.4506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49892598390579224,
      "rewards/margins": 0.9969396591186523,
      "rewards/rejected": -1.4958655834197998,
      "step": 7647
    },
    {
      "epoch": 1.0007851476241105,
      "grad_norm": 8.616921424865723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1453338861465454,
      "logits/rejected": -0.28875112533569336,
      "logps/chosen": -487.78094482421875,
      "logps/rejected": -561.5577392578125,
      "loss": 0.5931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7629677057266235,
      "rewards/margins": 0.6644506454467773,
      "rewards/rejected": -1.4274183511734009,
      "step": 7648
    },
    {
      "epoch": 1.0009160055614623,
      "grad_norm": 7.624189853668213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09943341463804245,
      "logits/rejected": -0.0501088872551918,
      "logps/chosen": -497.45086669921875,
      "logps/rejected": -292.1531677246094,
      "loss": 0.4312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.492553174495697,
      "rewards/margins": 1.0223567485809326,
      "rewards/rejected": -1.5149097442626953,
      "step": 7649
    },
    {
      "epoch": 1.001046863498814,
      "grad_norm": 5.643551349639893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17497897148132324,
      "logits/rejected": -0.4622742533683777,
      "logps/chosen": -318.61322021484375,
      "logps/rejected": -312.03900146484375,
      "loss": 0.5087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.482280969619751,
      "rewards/margins": 0.49685513973236084,
      "rewards/rejected": -0.9791361689567566,
      "step": 7650
    },
    {
      "epoch": 1.001177721436166,
      "grad_norm": 6.695739269256592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20121611654758453,
      "logits/rejected": -0.15961354970932007,
      "logps/chosen": -437.0050048828125,
      "logps/rejected": -560.8992309570312,
      "loss": 0.347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28410589694976807,
      "rewards/margins": 1.3847870826721191,
      "rewards/rejected": -1.6688929796218872,
      "step": 7651
    },
    {
      "epoch": 1.0013085793735177,
      "grad_norm": 8.743058204650879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36234334111213684,
      "logits/rejected": -0.36591872572898865,
      "logps/chosen": -465.54931640625,
      "logps/rejected": -494.494384765625,
      "loss": 0.6388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9215682744979858,
      "rewards/margins": 0.2866390347480774,
      "rewards/rejected": -1.208207368850708,
      "step": 7652
    },
    {
      "epoch": 1.0014394373108695,
      "grad_norm": 7.062389373779297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16358564794063568,
      "logits/rejected": -0.26864930987358093,
      "logps/chosen": -428.483154296875,
      "logps/rejected": -512.7468872070312,
      "loss": 0.4409,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2347715198993683,
      "rewards/margins": 0.8950300216674805,
      "rewards/rejected": -0.6602585315704346,
      "step": 7653
    },
    {
      "epoch": 1.001570295248221,
      "grad_norm": 6.094478130340576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2783425450325012,
      "logits/rejected": 0.15052583813667297,
      "logps/chosen": -500.96136474609375,
      "logps/rejected": -373.68798828125,
      "loss": 0.3963,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.713993489742279,
      "rewards/margins": 1.2269515991210938,
      "rewards/rejected": -1.9409451484680176,
      "step": 7654
    },
    {
      "epoch": 1.0017011531855728,
      "grad_norm": 4.926355838775635,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19102519750595093,
      "logits/rejected": -0.14573577046394348,
      "logps/chosen": -479.5913391113281,
      "logps/rejected": -592.80322265625,
      "loss": 0.2758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09952524304389954,
      "rewards/margins": 1.766349196434021,
      "rewards/rejected": -1.6668238639831543,
      "step": 7655
    },
    {
      "epoch": 1.0018320111229246,
      "grad_norm": 10.454030990600586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0026993313804268837,
      "logits/rejected": -0.20781256258487701,
      "logps/chosen": -511.48785400390625,
      "logps/rejected": -418.119873046875,
      "loss": 0.7082,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4015182852745056,
      "rewards/margins": 0.8659383058547974,
      "rewards/rejected": -1.2674565315246582,
      "step": 7656
    },
    {
      "epoch": 1.0019628690602764,
      "grad_norm": 4.587002754211426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04616710543632507,
      "logits/rejected": -0.32082486152648926,
      "logps/chosen": -514.6589965820312,
      "logps/rejected": -549.228515625,
      "loss": 0.2621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3187066912651062,
      "rewards/margins": 1.994957685470581,
      "rewards/rejected": -1.676250696182251,
      "step": 7657
    },
    {
      "epoch": 1.0020937269976282,
      "grad_norm": 3.3838095664978027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25017350912094116,
      "logits/rejected": -0.16492712497711182,
      "logps/chosen": -527.1351928710938,
      "logps/rejected": -582.052001953125,
      "loss": 0.1591,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23764288425445557,
      "rewards/margins": 2.133775472640991,
      "rewards/rejected": -2.3714182376861572,
      "step": 7658
    },
    {
      "epoch": 1.00222458493498,
      "grad_norm": 11.082157135009766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09171827137470245,
      "logits/rejected": 0.08022896200418472,
      "logps/chosen": -573.9595947265625,
      "logps/rejected": -568.8221435546875,
      "loss": 1.0403,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.32998788356781,
      "rewards/margins": -0.11874058842658997,
      "rewards/rejected": -1.2112473249435425,
      "step": 7659
    },
    {
      "epoch": 1.0023554428723318,
      "grad_norm": 4.3329596519470215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27834877371788025,
      "logits/rejected": -0.40306684374809265,
      "logps/chosen": -555.1954956054688,
      "logps/rejected": -509.33087158203125,
      "loss": 0.2699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8073437213897705,
      "rewards/margins": 1.4820300340652466,
      "rewards/rejected": -2.2893738746643066,
      "step": 7660
    },
    {
      "epoch": 1.0024863008096836,
      "grad_norm": 10.042677879333496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16871950030326843,
      "logits/rejected": -0.1399306356906891,
      "logps/chosen": -462.22113037109375,
      "logps/rejected": -459.486328125,
      "loss": 0.6152,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4786268472671509,
      "rewards/margins": 0.9656100273132324,
      "rewards/rejected": -2.4442367553710938,
      "step": 7661
    },
    {
      "epoch": 1.0026171587470352,
      "grad_norm": 6.91562557220459,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07172316312789917,
      "logits/rejected": 0.09016288816928864,
      "logps/chosen": -464.6582946777344,
      "logps/rejected": -500.9389343261719,
      "loss": 0.4856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.38459017872810364,
      "rewards/margins": 1.0239770412445068,
      "rewards/rejected": -0.6393869519233704,
      "step": 7662
    },
    {
      "epoch": 1.002748016684387,
      "grad_norm": 11.636700630187988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05430735647678375,
      "logits/rejected": 0.0899389386177063,
      "logps/chosen": -407.9727783203125,
      "logps/rejected": -278.32098388671875,
      "loss": 0.8793,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8076630234718323,
      "rewards/margins": -0.0004108548164367676,
      "rewards/rejected": -0.8072521090507507,
      "step": 7663
    },
    {
      "epoch": 1.0028788746217387,
      "grad_norm": 10.99160385131836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15774913132190704,
      "logits/rejected": -0.26970598101615906,
      "logps/chosen": -329.77276611328125,
      "logps/rejected": -352.2408142089844,
      "loss": 0.7558,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5212585926055908,
      "rewards/margins": 0.32901763916015625,
      "rewards/rejected": -0.8502762317657471,
      "step": 7664
    },
    {
      "epoch": 1.0030097325590905,
      "grad_norm": 9.211210250854492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12267306447029114,
      "logits/rejected": -0.028982002288103104,
      "logps/chosen": -540.7316284179688,
      "logps/rejected": -530.1146240234375,
      "loss": 0.7407,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5456184148788452,
      "rewards/margins": 0.3440563380718231,
      "rewards/rejected": -0.889674723148346,
      "step": 7665
    },
    {
      "epoch": 1.0031405904964423,
      "grad_norm": 7.219423770904541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06923297047615051,
      "logits/rejected": 0.13310150802135468,
      "logps/chosen": -602.7025146484375,
      "logps/rejected": -441.96990966796875,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3640638291835785,
      "rewards/margins": 1.7043483257293701,
      "rewards/rejected": -1.3402844667434692,
      "step": 7666
    },
    {
      "epoch": 1.0032714484337941,
      "grad_norm": 16.30390739440918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1975073218345642,
      "logits/rejected": -0.3237953186035156,
      "logps/chosen": -560.0538940429688,
      "logps/rejected": -433.7720031738281,
      "loss": 0.5895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4583277106285095,
      "rewards/margins": 1.478360652923584,
      "rewards/rejected": -1.9366883039474487,
      "step": 7667
    },
    {
      "epoch": 1.003402306371146,
      "grad_norm": 9.547670364379883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003177858889102936,
      "logits/rejected": -0.1227557510137558,
      "logps/chosen": -432.8390808105469,
      "logps/rejected": -653.9945678710938,
      "loss": 0.5733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6440706253051758,
      "rewards/margins": 0.7328134775161743,
      "rewards/rejected": -1.37688410282135,
      "step": 7668
    },
    {
      "epoch": 1.0035331643084975,
      "grad_norm": 6.312440395355225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2658645510673523,
      "logits/rejected": 0.24942493438720703,
      "logps/chosen": -462.50830078125,
      "logps/rejected": -437.51019287109375,
      "loss": 0.4187,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10629138350486755,
      "rewards/margins": 0.9697666168212891,
      "rewards/rejected": -1.076058030128479,
      "step": 7669
    },
    {
      "epoch": 1.0036640222458493,
      "grad_norm": 8.89255428314209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10179595649242401,
      "logits/rejected": 0.023176446557044983,
      "logps/chosen": -383.99395751953125,
      "logps/rejected": -474.36224365234375,
      "loss": 0.4147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.014571413397789001,
      "rewards/margins": 1.016816258430481,
      "rewards/rejected": -1.0022448301315308,
      "step": 7670
    },
    {
      "epoch": 1.003794880183201,
      "grad_norm": 5.73705530166626,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052588287740945816,
      "logits/rejected": -0.005276061594486237,
      "logps/chosen": -456.9871520996094,
      "logps/rejected": -403.4784851074219,
      "loss": 0.3694,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03418789058923721,
      "rewards/margins": 1.3031649589538574,
      "rewards/rejected": -1.2689770460128784,
      "step": 7671
    },
    {
      "epoch": 1.0039257381205529,
      "grad_norm": 5.954068660736084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03768390417098999,
      "logits/rejected": -0.008109360933303833,
      "logps/chosen": -490.99853515625,
      "logps/rejected": -489.31182861328125,
      "loss": 0.3437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14540405571460724,
      "rewards/margins": 1.2301323413848877,
      "rewards/rejected": -1.0847282409667969,
      "step": 7672
    },
    {
      "epoch": 1.0040565960579046,
      "grad_norm": 10.637394905090332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.272036075592041,
      "logits/rejected": -0.2644491493701935,
      "logps/chosen": -419.0760803222656,
      "logps/rejected": -348.988037109375,
      "loss": 0.6565,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0359392166137695,
      "rewards/margins": 0.20064297318458557,
      "rewards/rejected": -1.2365822792053223,
      "step": 7673
    },
    {
      "epoch": 1.0041874539952564,
      "grad_norm": 6.967790603637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14272645115852356,
      "logits/rejected": -0.005852453410625458,
      "logps/chosen": -564.1451416015625,
      "logps/rejected": -572.4904174804688,
      "loss": 0.3931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33739954233169556,
      "rewards/margins": 1.260974645614624,
      "rewards/rejected": -1.5983741283416748,
      "step": 7674
    },
    {
      "epoch": 1.0043183119326082,
      "grad_norm": 5.987496376037598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035758744925260544,
      "logits/rejected": 0.04727201908826828,
      "logps/chosen": -417.0401306152344,
      "logps/rejected": -409.7779541015625,
      "loss": 0.5498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05315170809626579,
      "rewards/margins": 0.7783129215240479,
      "rewards/rejected": -0.7251611948013306,
      "step": 7675
    },
    {
      "epoch": 1.00444916986996,
      "grad_norm": 4.518027305603027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5781476497650146,
      "logits/rejected": -0.3390057384967804,
      "logps/chosen": -240.89964294433594,
      "logps/rejected": -324.19061279296875,
      "loss": 0.3815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19762438535690308,
      "rewards/margins": 0.8960475921630859,
      "rewards/rejected": -1.0936720371246338,
      "step": 7676
    },
    {
      "epoch": 1.0045800278073116,
      "grad_norm": 7.996395587921143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0009163841605186462,
      "logits/rejected": -0.14160893857479095,
      "logps/chosen": -478.499267578125,
      "logps/rejected": -414.2593688964844,
      "loss": 0.4916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4702630937099457,
      "rewards/margins": 0.8818163275718689,
      "rewards/rejected": -1.3520793914794922,
      "step": 7677
    },
    {
      "epoch": 1.0047108857446634,
      "grad_norm": 5.521874904632568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32960668206214905,
      "logits/rejected": -0.34470611810684204,
      "logps/chosen": -500.9974670410156,
      "logps/rejected": -409.8948974609375,
      "loss": 0.3652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.803959310054779,
      "rewards/margins": 1.0153204202651978,
      "rewards/rejected": -1.819279670715332,
      "step": 7678
    },
    {
      "epoch": 1.0048417436820152,
      "grad_norm": 8.392467498779297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34304124116897583,
      "logits/rejected": -0.03727303445339203,
      "logps/chosen": -440.590087890625,
      "logps/rejected": -417.1070251464844,
      "loss": 0.6264,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7999870777130127,
      "rewards/margins": 0.44965988397598267,
      "rewards/rejected": -1.2496470212936401,
      "step": 7679
    },
    {
      "epoch": 1.004972601619367,
      "grad_norm": 7.130107402801514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02378012053668499,
      "logits/rejected": 0.03501461446285248,
      "logps/chosen": -493.75909423828125,
      "logps/rejected": -476.5673828125,
      "loss": 0.4894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03890933096408844,
      "rewards/margins": 0.7455034255981445,
      "rewards/rejected": -0.7844128012657166,
      "step": 7680
    },
    {
      "epoch": 1.0051034595567188,
      "grad_norm": 8.165499687194824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011125404387712479,
      "logits/rejected": 0.13801926374435425,
      "logps/chosen": -472.0658264160156,
      "logps/rejected": -524.56298828125,
      "loss": 0.4254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4683791995048523,
      "rewards/margins": 1.3280845880508423,
      "rewards/rejected": -1.7964637279510498,
      "step": 7681
    },
    {
      "epoch": 1.0052343174940706,
      "grad_norm": 6.028484344482422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41343045234680176,
      "logits/rejected": -0.13966426253318787,
      "logps/chosen": -396.7543640136719,
      "logps/rejected": -600.9342041015625,
      "loss": 0.4034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4278402626514435,
      "rewards/margins": 1.5197969675064087,
      "rewards/rejected": -1.9476370811462402,
      "step": 7682
    },
    {
      "epoch": 1.0053651754314223,
      "grad_norm": 11.58273696899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019100788980722427,
      "logits/rejected": 0.16727854311466217,
      "logps/chosen": -568.3964233398438,
      "logps/rejected": -433.1435241699219,
      "loss": 0.7408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7660887837409973,
      "rewards/margins": 0.07720108330249786,
      "rewards/rejected": -0.8432899117469788,
      "step": 7683
    },
    {
      "epoch": 1.0054960333687741,
      "grad_norm": 11.19632339477539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055111829191446304,
      "logits/rejected": -0.12252231687307358,
      "logps/chosen": -439.73333740234375,
      "logps/rejected": -380.8363037109375,
      "loss": 0.7545,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5455577373504639,
      "rewards/margins": 0.19952617585659027,
      "rewards/rejected": -1.7450839281082153,
      "step": 7684
    },
    {
      "epoch": 1.0056268913061257,
      "grad_norm": 8.016613006591797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013093583285808563,
      "logits/rejected": -0.08611536026000977,
      "logps/chosen": -525.457275390625,
      "logps/rejected": -483.20770263671875,
      "loss": 0.4383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5735406875610352,
      "rewards/margins": 1.2181951999664307,
      "rewards/rejected": -1.7917358875274658,
      "step": 7685
    },
    {
      "epoch": 1.0057577492434775,
      "grad_norm": 9.235943794250488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09623433649539948,
      "logits/rejected": -0.07182773947715759,
      "logps/chosen": -439.925048828125,
      "logps/rejected": -453.22552490234375,
      "loss": 0.5865,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.003016710281372,
      "rewards/margins": 0.5589605569839478,
      "rewards/rejected": -1.5619771480560303,
      "step": 7686
    },
    {
      "epoch": 1.0058886071808293,
      "grad_norm": 6.771087646484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04968242347240448,
      "logits/rejected": 0.013952162116765976,
      "logps/chosen": -645.1150512695312,
      "logps/rejected": -512.3560791015625,
      "loss": 0.391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02231600135564804,
      "rewards/margins": 1.1540242433547974,
      "rewards/rejected": -1.1317082643508911,
      "step": 7687
    },
    {
      "epoch": 1.006019465118181,
      "grad_norm": 9.911001205444336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11307072639465332,
      "logits/rejected": -0.037882570177316666,
      "logps/chosen": -577.7781372070312,
      "logps/rejected": -422.35137939453125,
      "loss": 0.5604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22269898653030396,
      "rewards/margins": 0.7062047719955444,
      "rewards/rejected": -0.9289038181304932,
      "step": 7688
    },
    {
      "epoch": 1.0061503230555329,
      "grad_norm": 5.251974105834961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3238587975502014,
      "logits/rejected": 0.3373228907585144,
      "logps/chosen": -439.1438903808594,
      "logps/rejected": -443.81768798828125,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9351328015327454,
      "rewards/margins": 1.7101067304611206,
      "rewards/rejected": -0.7749741077423096,
      "step": 7689
    },
    {
      "epoch": 1.0062811809928847,
      "grad_norm": 6.3214240074157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04111620783805847,
      "logits/rejected": -0.19957050681114197,
      "logps/chosen": -431.3221130371094,
      "logps/rejected": -498.66265869140625,
      "loss": 0.4016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38143694400787354,
      "rewards/margins": 1.303916096687317,
      "rewards/rejected": -1.6853530406951904,
      "step": 7690
    },
    {
      "epoch": 1.0064120389302365,
      "grad_norm": 6.042658805847168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1144484132528305,
      "logits/rejected": -0.26068785786628723,
      "logps/chosen": -690.6121215820312,
      "logps/rejected": -460.6744689941406,
      "loss": 0.3664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09928150475025177,
      "rewards/margins": 1.240692138671875,
      "rewards/rejected": -1.1414105892181396,
      "step": 7691
    },
    {
      "epoch": 1.006542896867588,
      "grad_norm": 4.2043962478637695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2837325930595398,
      "logits/rejected": 0.273457795381546,
      "logps/chosen": -836.5194091796875,
      "logps/rejected": -505.769775390625,
      "loss": 0.1869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9335486888885498,
      "rewards/margins": 2.3146426677703857,
      "rewards/rejected": -1.381093978881836,
      "step": 7692
    },
    {
      "epoch": 1.0066737548049398,
      "grad_norm": 4.664556503295898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004182014148682356,
      "logits/rejected": -0.049441948533058167,
      "logps/chosen": -484.6614990234375,
      "logps/rejected": -378.30767822265625,
      "loss": 0.2819,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3913429379463196,
      "rewards/margins": 1.4444118738174438,
      "rewards/rejected": -1.053068995475769,
      "step": 7693
    },
    {
      "epoch": 1.0068046127422916,
      "grad_norm": 8.989413261413574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4620964527130127,
      "logits/rejected": -0.4232577681541443,
      "logps/chosen": -339.5587158203125,
      "logps/rejected": -378.85150146484375,
      "loss": 0.5828,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5089523792266846,
      "rewards/margins": 0.6625560522079468,
      "rewards/rejected": -1.1715084314346313,
      "step": 7694
    },
    {
      "epoch": 1.0069354706796434,
      "grad_norm": 6.896424770355225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12553229928016663,
      "logits/rejected": -0.12012054026126862,
      "logps/chosen": -516.0701904296875,
      "logps/rejected": -490.65087890625,
      "loss": 0.3647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0029460787773132324,
      "rewards/margins": 1.4737709760665894,
      "rewards/rejected": -1.4767169952392578,
      "step": 7695
    },
    {
      "epoch": 1.0070663286169952,
      "grad_norm": 10.295015335083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4559705853462219,
      "logits/rejected": -0.3862583339214325,
      "logps/chosen": -454.3017272949219,
      "logps/rejected": -402.0547790527344,
      "loss": 0.4877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10080836713314056,
      "rewards/margins": 0.8535133004188538,
      "rewards/rejected": -0.9543217420578003,
      "step": 7696
    },
    {
      "epoch": 1.007197186554347,
      "grad_norm": 5.327004432678223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07163995504379272,
      "logits/rejected": -0.09503702819347382,
      "logps/chosen": -608.26953125,
      "logps/rejected": -476.32147216796875,
      "loss": 0.3598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.366403192281723,
      "rewards/margins": 1.4768253564834595,
      "rewards/rejected": -1.110422134399414,
      "step": 7697
    },
    {
      "epoch": 1.0073280444916988,
      "grad_norm": 4.851340293884277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4533821940422058,
      "logits/rejected": -0.4046335220336914,
      "logps/chosen": -393.21710205078125,
      "logps/rejected": -381.9166564941406,
      "loss": 0.3628,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1585145890712738,
      "rewards/margins": 1.0761393308639526,
      "rewards/rejected": -0.9176246523857117,
      "step": 7698
    },
    {
      "epoch": 1.0074589024290506,
      "grad_norm": 11.271646499633789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.284018337726593,
      "logits/rejected": -0.21691498160362244,
      "logps/chosen": -462.2464294433594,
      "logps/rejected": -651.993896484375,
      "loss": 0.7471,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5866573452949524,
      "rewards/margins": 1.8592255115509033,
      "rewards/rejected": -2.445882797241211,
      "step": 7699
    },
    {
      "epoch": 1.0075897603664021,
      "grad_norm": 8.378802299499512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.056970685720443726,
      "logits/rejected": -0.030447449535131454,
      "logps/chosen": -341.91229248046875,
      "logps/rejected": -320.8988952636719,
      "loss": 0.5893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0173963513225317,
      "rewards/margins": 0.554166316986084,
      "rewards/rejected": -0.5367699861526489,
      "step": 7700
    },
    {
      "epoch": 1.007720618303754,
      "grad_norm": 7.705832481384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23174014687538147,
      "logits/rejected": -0.08148571848869324,
      "logps/chosen": -604.1417236328125,
      "logps/rejected": -427.39935302734375,
      "loss": 0.4188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5144439935684204,
      "rewards/margins": 1.1630102396011353,
      "rewards/rejected": -1.6774539947509766,
      "step": 7701
    },
    {
      "epoch": 1.0078514762411057,
      "grad_norm": 9.284268379211426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07217433303594589,
      "logits/rejected": 0.0016987249255180359,
      "logps/chosen": -575.2123413085938,
      "logps/rejected": -410.4275817871094,
      "loss": 0.5998,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05323333293199539,
      "rewards/margins": 0.5320142507553101,
      "rewards/rejected": -0.47878092527389526,
      "step": 7702
    },
    {
      "epoch": 1.0079823341784575,
      "grad_norm": 4.972794055938721,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047448888421058655,
      "logits/rejected": -0.00015583960339426994,
      "logps/chosen": -400.9599304199219,
      "logps/rejected": -356.19354248046875,
      "loss": 0.3564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17298927903175354,
      "rewards/margins": 1.5252267122268677,
      "rewards/rejected": -1.698216199874878,
      "step": 7703
    },
    {
      "epoch": 1.0081131921158093,
      "grad_norm": 5.867617607116699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2219630926847458,
      "logits/rejected": -0.3623899817466736,
      "logps/chosen": -586.9415283203125,
      "logps/rejected": -472.8734130859375,
      "loss": 0.3381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08577348291873932,
      "rewards/margins": 1.539903163909912,
      "rewards/rejected": -1.454129695892334,
      "step": 7704
    },
    {
      "epoch": 1.008244050053161,
      "grad_norm": 8.33155345916748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3021319508552551,
      "logits/rejected": -0.16403736174106598,
      "logps/chosen": -446.27655029296875,
      "logps/rejected": -373.1595458984375,
      "loss": 0.4354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29320794343948364,
      "rewards/margins": 1.06747305393219,
      "rewards/rejected": -1.3606810569763184,
      "step": 7705
    },
    {
      "epoch": 1.0083749079905129,
      "grad_norm": 5.0430192947387695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05069221928715706,
      "logits/rejected": -0.11062876880168915,
      "logps/chosen": -583.5146484375,
      "logps/rejected": -462.2203674316406,
      "loss": 0.3916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05604305863380432,
      "rewards/margins": 1.1118592023849487,
      "rewards/rejected": -1.1679023504257202,
      "step": 7706
    },
    {
      "epoch": 1.0085057659278647,
      "grad_norm": 10.514473915100098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3385784924030304,
      "logits/rejected": -0.28934407234191895,
      "logps/chosen": -597.568359375,
      "logps/rejected": -422.4732360839844,
      "loss": 0.5418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3989777565002441,
      "rewards/margins": 0.7174356579780579,
      "rewards/rejected": -2.1164133548736572,
      "step": 7707
    },
    {
      "epoch": 1.0086366238652162,
      "grad_norm": 9.030745506286621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4290093779563904,
      "logits/rejected": -0.21079346537590027,
      "logps/chosen": -315.9378967285156,
      "logps/rejected": -305.4404602050781,
      "loss": 0.5436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.012710690498352,
      "rewards/margins": 0.7734805941581726,
      "rewards/rejected": -1.7861912250518799,
      "step": 7708
    },
    {
      "epoch": 1.008767481802568,
      "grad_norm": 6.584220886230469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09500909596681595,
      "logits/rejected": -0.0036977692507207394,
      "logps/chosen": -582.0083618164062,
      "logps/rejected": -447.0752258300781,
      "loss": 0.3339,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11742936074733734,
      "rewards/margins": 1.5728466510772705,
      "rewards/rejected": -1.4554171562194824,
      "step": 7709
    },
    {
      "epoch": 1.0088983397399198,
      "grad_norm": 4.326579570770264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23317204415798187,
      "logits/rejected": -0.04013129696249962,
      "logps/chosen": -577.0126342773438,
      "logps/rejected": -386.7614440917969,
      "loss": 0.266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4870780110359192,
      "rewards/margins": 1.7268648147583008,
      "rewards/rejected": -1.2397868633270264,
      "step": 7710
    },
    {
      "epoch": 1.0090291976772716,
      "grad_norm": 10.047219276428223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06922280043363571,
      "logits/rejected": -0.12644566595554352,
      "logps/chosen": -393.3164367675781,
      "logps/rejected": -435.7346496582031,
      "loss": 1.0184,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.036810848861932755,
      "rewards/margins": -0.17964401841163635,
      "rewards/rejected": 0.2164548933506012,
      "step": 7711
    },
    {
      "epoch": 1.0091600556146234,
      "grad_norm": 8.299041748046875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2037677764892578,
      "logits/rejected": 0.2963780164718628,
      "logps/chosen": -479.91729736328125,
      "logps/rejected": -631.010009765625,
      "loss": 0.4475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14236047863960266,
      "rewards/margins": 0.8093878030776978,
      "rewards/rejected": -0.9517483115196228,
      "step": 7712
    },
    {
      "epoch": 1.0092909135519752,
      "grad_norm": 4.295957088470459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2881008982658386,
      "logits/rejected": -0.2809544801712036,
      "logps/chosen": -421.71038818359375,
      "logps/rejected": -356.8785095214844,
      "loss": 0.2388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2924753427505493,
      "rewards/margins": 1.6414883136749268,
      "rewards/rejected": -1.9339635372161865,
      "step": 7713
    },
    {
      "epoch": 1.009421771489327,
      "grad_norm": 6.826639652252197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20993873476982117,
      "logits/rejected": -0.05580057203769684,
      "logps/chosen": -362.7279052734375,
      "logps/rejected": -474.677734375,
      "loss": 0.5299,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21919867396354675,
      "rewards/margins": 0.6215302348136902,
      "rewards/rejected": -0.8407289385795593,
      "step": 7714
    },
    {
      "epoch": 1.0095526294266786,
      "grad_norm": 3.239382266998291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044128887355327606,
      "logits/rejected": -0.11349263042211533,
      "logps/chosen": -754.840576171875,
      "logps/rejected": -552.1910400390625,
      "loss": 0.2549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5921546816825867,
      "rewards/margins": 1.5235345363616943,
      "rewards/rejected": -0.9313796162605286,
      "step": 7715
    },
    {
      "epoch": 1.0096834873640304,
      "grad_norm": 6.349853992462158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07616028189659119,
      "logits/rejected": -0.007254607975482941,
      "logps/chosen": -312.64251708984375,
      "logps/rejected": -375.8188171386719,
      "loss": 0.5128,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21255111694335938,
      "rewards/margins": 0.5923866033554077,
      "rewards/rejected": -0.8049378395080566,
      "step": 7716
    },
    {
      "epoch": 1.0098143453013821,
      "grad_norm": 10.402166366577148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4164302349090576,
      "logits/rejected": 0.011395709589123726,
      "logps/chosen": -401.65911865234375,
      "logps/rejected": -525.3175048828125,
      "loss": 0.464,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45767009258270264,
      "rewards/margins": 1.363187313079834,
      "rewards/rejected": -1.8208575248718262,
      "step": 7717
    },
    {
      "epoch": 1.009945203238734,
      "grad_norm": 4.591363430023193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06451824307441711,
      "logits/rejected": -0.29090890288352966,
      "logps/chosen": -349.84375,
      "logps/rejected": -333.0962219238281,
      "loss": 0.3129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5394318103790283,
      "rewards/margins": 1.4974422454833984,
      "rewards/rejected": -2.036874294281006,
      "step": 7718
    },
    {
      "epoch": 1.0100760611760857,
      "grad_norm": 8.184995651245117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14541344344615936,
      "logits/rejected": -0.0028135711327195168,
      "logps/chosen": -408.662109375,
      "logps/rejected": -515.4088745117188,
      "loss": 0.5969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5836449265480042,
      "rewards/margins": 0.8116934299468994,
      "rewards/rejected": -1.3953381776809692,
      "step": 7719
    },
    {
      "epoch": 1.0102069191134375,
      "grad_norm": 4.890773773193359,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1472296565771103,
      "logits/rejected": 0.25415393710136414,
      "logps/chosen": -499.4023132324219,
      "logps/rejected": -560.1229248046875,
      "loss": 0.3466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2754688560962677,
      "rewards/margins": 1.7053680419921875,
      "rewards/rejected": -1.4298992156982422,
      "step": 7720
    },
    {
      "epoch": 1.0103377770507893,
      "grad_norm": 10.594040870666504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11115814745426178,
      "logits/rejected": 0.09842321276664734,
      "logps/chosen": -511.1744384765625,
      "logps/rejected": -588.88134765625,
      "loss": 0.6116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15486618876457214,
      "rewards/margins": 0.47783875465393066,
      "rewards/rejected": -0.6327049136161804,
      "step": 7721
    },
    {
      "epoch": 1.010468634988141,
      "grad_norm": 7.277425765991211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35402828454971313,
      "logits/rejected": -0.516732931137085,
      "logps/chosen": -529.7646484375,
      "logps/rejected": -384.5638427734375,
      "loss": 0.4642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7777376174926758,
      "rewards/margins": 0.9818083047866821,
      "rewards/rejected": -1.7595460414886475,
      "step": 7722
    },
    {
      "epoch": 1.0105994929254927,
      "grad_norm": 4.00993537902832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014034230262041092,
      "logits/rejected": -0.007969141937792301,
      "logps/chosen": -447.62835693359375,
      "logps/rejected": -511.1328430175781,
      "loss": 0.2757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11332760006189346,
      "rewards/margins": 1.6554374694824219,
      "rewards/rejected": -1.5421099662780762,
      "step": 7723
    },
    {
      "epoch": 1.0107303508628445,
      "grad_norm": 7.11444616317749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2197476029396057,
      "logits/rejected": 0.1051357090473175,
      "logps/chosen": -365.33349609375,
      "logps/rejected": -351.62139892578125,
      "loss": 0.542,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.44748765230178833,
      "rewards/margins": 0.4431217908859253,
      "rewards/rejected": 0.004365861415863037,
      "step": 7724
    },
    {
      "epoch": 1.0108612088001963,
      "grad_norm": 7.55134916305542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16506266593933105,
      "logits/rejected": 0.10290367901325226,
      "logps/chosen": -398.5083312988281,
      "logps/rejected": -533.2179565429688,
      "loss": 0.3963,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42078500986099243,
      "rewards/margins": 1.1607369184494019,
      "rewards/rejected": -1.58152174949646,
      "step": 7725
    },
    {
      "epoch": 1.010992066737548,
      "grad_norm": 18.538543701171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12626171112060547,
      "logits/rejected": -0.20330534875392914,
      "logps/chosen": -650.8838500976562,
      "logps/rejected": -523.0787353515625,
      "loss": 0.6457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20290841162204742,
      "rewards/margins": 0.9964485764503479,
      "rewards/rejected": -0.7935401797294617,
      "step": 7726
    },
    {
      "epoch": 1.0111229246748998,
      "grad_norm": 8.10973834991455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011690709739923477,
      "logits/rejected": -0.13133838772773743,
      "logps/chosen": -481.83441162109375,
      "logps/rejected": -431.7695617675781,
      "loss": 0.5015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5398382544517517,
      "rewards/margins": 0.8502753973007202,
      "rewards/rejected": -1.3901137113571167,
      "step": 7727
    },
    {
      "epoch": 1.0112537826122516,
      "grad_norm": 7.154611110687256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26353365182876587,
      "logits/rejected": -0.2623230814933777,
      "logps/chosen": -529.1934204101562,
      "logps/rejected": -475.54400634765625,
      "loss": 0.4006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0888473391532898,
      "rewards/margins": 0.9504751563072205,
      "rewards/rejected": -1.0393223762512207,
      "step": 7728
    },
    {
      "epoch": 1.0113846405496034,
      "grad_norm": 6.224050521850586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11336458474397659,
      "logits/rejected": 0.004217810928821564,
      "logps/chosen": -482.05047607421875,
      "logps/rejected": -556.5330810546875,
      "loss": 0.3705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5069814920425415,
      "rewards/margins": 1.0591068267822266,
      "rewards/rejected": -1.566088318824768,
      "step": 7729
    },
    {
      "epoch": 1.011515498486955,
      "grad_norm": 8.93092155456543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05529250204563141,
      "logits/rejected": -0.09966161847114563,
      "logps/chosen": -361.25860595703125,
      "logps/rejected": -344.8004455566406,
      "loss": 0.6153,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9222309589385986,
      "rewards/margins": 0.6546005606651306,
      "rewards/rejected": -1.5768314599990845,
      "step": 7730
    },
    {
      "epoch": 1.0116463564243068,
      "grad_norm": 5.40667200088501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21619385480880737,
      "logits/rejected": -0.5189714431762695,
      "logps/chosen": -324.7409362792969,
      "logps/rejected": -268.5225524902344,
      "loss": 0.4871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.357164591550827,
      "rewards/margins": 0.5977292656898499,
      "rewards/rejected": -0.9548938870429993,
      "step": 7731
    },
    {
      "epoch": 1.0117772143616586,
      "grad_norm": 4.5536208152771,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018781092017889023,
      "logits/rejected": -0.1392941176891327,
      "logps/chosen": -628.6614990234375,
      "logps/rejected": -542.0885620117188,
      "loss": 0.233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1097249984741211,
      "rewards/margins": 1.4917035102844238,
      "rewards/rejected": -1.6014286279678345,
      "step": 7732
    },
    {
      "epoch": 1.0119080722990104,
      "grad_norm": 8.429326057434082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028536275029182434,
      "logits/rejected": -0.02805604785680771,
      "logps/chosen": -519.016357421875,
      "logps/rejected": -412.4281311035156,
      "loss": 0.5135,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34897878766059875,
      "rewards/margins": 0.6002041101455688,
      "rewards/rejected": -0.9491829872131348,
      "step": 7733
    },
    {
      "epoch": 1.0120389302363622,
      "grad_norm": 7.0913801193237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4690514802932739,
      "logits/rejected": -0.23373234272003174,
      "logps/chosen": -494.4277038574219,
      "logps/rejected": -552.8465576171875,
      "loss": 0.572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8953639268875122,
      "rewards/margins": 0.42183685302734375,
      "rewards/rejected": -2.3172006607055664,
      "step": 7734
    },
    {
      "epoch": 1.012169788173714,
      "grad_norm": 6.5674357414245605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27370530366897583,
      "logits/rejected": 0.3305295407772064,
      "logps/chosen": -696.411865234375,
      "logps/rejected": -516.334716796875,
      "loss": 0.2371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.58427894115448,
      "rewards/margins": 2.2243082523345947,
      "rewards/rejected": -1.6400294303894043,
      "step": 7735
    },
    {
      "epoch": 1.0123006461110657,
      "grad_norm": 9.074349403381348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1550348997116089,
      "logits/rejected": -0.11099844425916672,
      "logps/chosen": -538.390869140625,
      "logps/rejected": -638.684814453125,
      "loss": 0.5108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8302531838417053,
      "rewards/margins": 1.6372911930084229,
      "rewards/rejected": -2.4675443172454834,
      "step": 7736
    },
    {
      "epoch": 1.0124315040484175,
      "grad_norm": 9.063119888305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04814077541232109,
      "logits/rejected": -0.243287593126297,
      "logps/chosen": -616.4981079101562,
      "logps/rejected": -714.2271728515625,
      "loss": 0.4979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.53873610496521,
      "rewards/margins": 1.2510713338851929,
      "rewards/rejected": -2.7898075580596924,
      "step": 7737
    },
    {
      "epoch": 1.012562361985769,
      "grad_norm": 9.401419639587402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.149808868765831,
      "logits/rejected": 0.06126388534903526,
      "logps/chosen": -587.413330078125,
      "logps/rejected": -457.7579345703125,
      "loss": 0.5829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.411144495010376,
      "rewards/margins": 1.1863925457000732,
      "rewards/rejected": -1.5975370407104492,
      "step": 7738
    },
    {
      "epoch": 1.012693219923121,
      "grad_norm": 7.564774036407471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09270647168159485,
      "logits/rejected": -0.17227768898010254,
      "logps/chosen": -428.44683837890625,
      "logps/rejected": -382.82891845703125,
      "loss": 0.4003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9034942388534546,
      "rewards/margins": 1.3880479335784912,
      "rewards/rejected": -2.2915420532226562,
      "step": 7739
    },
    {
      "epoch": 1.0128240778604727,
      "grad_norm": 7.09135103225708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3463551998138428,
      "logits/rejected": -0.25025638937950134,
      "logps/chosen": -521.580810546875,
      "logps/rejected": -429.50811767578125,
      "loss": 0.5639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26373445987701416,
      "rewards/margins": 0.6062260270118713,
      "rewards/rejected": -0.3424915075302124,
      "step": 7740
    },
    {
      "epoch": 1.0129549357978245,
      "grad_norm": 6.926533222198486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08300213515758514,
      "logits/rejected": 0.07541914284229279,
      "logps/chosen": -397.85400390625,
      "logps/rejected": -396.6959228515625,
      "loss": 0.4685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17079249024391174,
      "rewards/margins": 0.8605704307556152,
      "rewards/rejected": -1.0313630104064941,
      "step": 7741
    },
    {
      "epoch": 1.0130857937351763,
      "grad_norm": 7.802501201629639,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0414910763502121,
      "logits/rejected": -0.1517612189054489,
      "logps/chosen": -496.92974853515625,
      "logps/rejected": -467.4693603515625,
      "loss": 0.4832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7600905895233154,
      "rewards/margins": 1.077599048614502,
      "rewards/rejected": -1.8376896381378174,
      "step": 7742
    },
    {
      "epoch": 1.013216651672528,
      "grad_norm": 4.877883434295654,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09784521907567978,
      "logits/rejected": 0.061929211020469666,
      "logps/chosen": -545.1034545898438,
      "logps/rejected": -456.5301513671875,
      "loss": 0.3803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3628363013267517,
      "rewards/margins": 1.0597662925720215,
      "rewards/rejected": -1.4226024150848389,
      "step": 7743
    },
    {
      "epoch": 1.0133475096098798,
      "grad_norm": 9.8059663772583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2959521412849426,
      "logits/rejected": -0.19371718168258667,
      "logps/chosen": -292.15802001953125,
      "logps/rejected": -427.94219970703125,
      "loss": 0.6552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2575525045394897,
      "rewards/margins": 0.33956968784332275,
      "rewards/rejected": -1.5971221923828125,
      "step": 7744
    },
    {
      "epoch": 1.0134783675472316,
      "grad_norm": 8.989816665649414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19414523243904114,
      "logits/rejected": -0.07511137425899506,
      "logps/chosen": -563.6004028320312,
      "logps/rejected": -519.9981689453125,
      "loss": 0.5393,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14482460916042328,
      "rewards/margins": 1.278889536857605,
      "rewards/rejected": -1.4237141609191895,
      "step": 7745
    },
    {
      "epoch": 1.0136092254845832,
      "grad_norm": 5.019392967224121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11694605648517609,
      "logits/rejected": 0.06944888830184937,
      "logps/chosen": -330.5093688964844,
      "logps/rejected": -409.81207275390625,
      "loss": 0.308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21207170188426971,
      "rewards/margins": 1.3549124002456665,
      "rewards/rejected": -1.5669841766357422,
      "step": 7746
    },
    {
      "epoch": 1.013740083421935,
      "grad_norm": 4.472139835357666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08537065982818604,
      "logits/rejected": -0.09260720759630203,
      "logps/chosen": -359.35821533203125,
      "logps/rejected": -344.75799560546875,
      "loss": 0.2267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26609864830970764,
      "rewards/margins": 1.791357398033142,
      "rewards/rejected": -2.0574560165405273,
      "step": 7747
    },
    {
      "epoch": 1.0138709413592868,
      "grad_norm": 4.738583087921143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4133192300796509,
      "logits/rejected": -0.3731039762496948,
      "logps/chosen": -319.2525939941406,
      "logps/rejected": -313.3892517089844,
      "loss": 0.3436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.397007018327713,
      "rewards/margins": 1.2039668560028076,
      "rewards/rejected": -1.6009736061096191,
      "step": 7748
    },
    {
      "epoch": 1.0140017992966386,
      "grad_norm": 13.800278663635254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29901453852653503,
      "logits/rejected": -0.34397318959236145,
      "logps/chosen": -462.3222961425781,
      "logps/rejected": -467.6006164550781,
      "loss": 1.1702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4701340198516846,
      "rewards/margins": -0.45363447070121765,
      "rewards/rejected": -1.016499400138855,
      "step": 7749
    },
    {
      "epoch": 1.0141326572339904,
      "grad_norm": 6.366208553314209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07185643911361694,
      "logits/rejected": -0.3093857169151306,
      "logps/chosen": -254.23171997070312,
      "logps/rejected": -425.8898010253906,
      "loss": 0.393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12494371831417084,
      "rewards/margins": 2.1662216186523438,
      "rewards/rejected": -2.2911651134490967,
      "step": 7750
    },
    {
      "epoch": 1.0142635151713422,
      "grad_norm": 7.1651291847229,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13952462375164032,
      "logits/rejected": -0.1468362957239151,
      "logps/chosen": -472.33062744140625,
      "logps/rejected": -578.7551879882812,
      "loss": 0.3818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6544142961502075,
      "rewards/margins": 1.7706269025802612,
      "rewards/rejected": -2.4250411987304688,
      "step": 7751
    },
    {
      "epoch": 1.014394373108694,
      "grad_norm": 7.683686256408691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04659359157085419,
      "logits/rejected": -0.16292496025562286,
      "logps/chosen": -352.2448425292969,
      "logps/rejected": -334.1786193847656,
      "loss": 0.3487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9960500597953796,
      "rewards/margins": 1.5805659294128418,
      "rewards/rejected": -2.576616048812866,
      "step": 7752
    },
    {
      "epoch": 1.0145252310460455,
      "grad_norm": 8.822007179260254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2949339747428894,
      "logits/rejected": 0.2220883071422577,
      "logps/chosen": -427.4214782714844,
      "logps/rejected": -457.4164123535156,
      "loss": 0.7002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3049606382846832,
      "rewards/margins": 0.6480011940002441,
      "rewards/rejected": -0.9529616832733154,
      "step": 7753
    },
    {
      "epoch": 1.0146560889833973,
      "grad_norm": 7.616687297821045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08487626910209656,
      "logits/rejected": -0.250076025724411,
      "logps/chosen": -472.5127868652344,
      "logps/rejected": -616.8320922851562,
      "loss": 0.3945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6083081960678101,
      "rewards/margins": 1.1682074069976807,
      "rewards/rejected": -1.7765157222747803,
      "step": 7754
    },
    {
      "epoch": 1.0147869469207491,
      "grad_norm": 9.233147621154785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03273492678999901,
      "logits/rejected": -0.10024382174015045,
      "logps/chosen": -465.6469421386719,
      "logps/rejected": -368.6718444824219,
      "loss": 0.6667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08398135006427765,
      "rewards/margins": 0.43143293261528015,
      "rewards/rejected": -0.3474515676498413,
      "step": 7755
    },
    {
      "epoch": 1.014917804858101,
      "grad_norm": 5.932581901550293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4720139503479004,
      "logits/rejected": 0.3045370876789093,
      "logps/chosen": -312.14569091796875,
      "logps/rejected": -327.884765625,
      "loss": 0.3544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.051461800932884216,
      "rewards/margins": 1.0632412433624268,
      "rewards/rejected": -1.0117794275283813,
      "step": 7756
    },
    {
      "epoch": 1.0150486627954527,
      "grad_norm": 8.012372016906738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21205294132232666,
      "logits/rejected": -0.0060247257351875305,
      "logps/chosen": -426.95794677734375,
      "logps/rejected": -297.15185546875,
      "loss": 0.6983,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16507452726364136,
      "rewards/margins": 0.6464438438415527,
      "rewards/rejected": -0.48136940598487854,
      "step": 7757
    },
    {
      "epoch": 1.0151795207328045,
      "grad_norm": 6.855416774749756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0229039303958416,
      "logits/rejected": -0.18921639025211334,
      "logps/chosen": -530.8228759765625,
      "logps/rejected": -523.3124389648438,
      "loss": 0.3446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3621438443660736,
      "rewards/margins": 1.2192318439483643,
      "rewards/rejected": -1.5813757181167603,
      "step": 7758
    },
    {
      "epoch": 1.0153103786701563,
      "grad_norm": 7.8176703453063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0714193731546402,
      "logits/rejected": -0.13761842250823975,
      "logps/chosen": -514.9605712890625,
      "logps/rejected": -473.4835510253906,
      "loss": 0.5236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06281529366970062,
      "rewards/margins": 0.9653283953666687,
      "rewards/rejected": -1.0281437635421753,
      "step": 7759
    },
    {
      "epoch": 1.015441236607508,
      "grad_norm": 5.556633472442627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2095036804676056,
      "logits/rejected": 0.010099239647388458,
      "logps/chosen": -451.8848876953125,
      "logps/rejected": -446.31329345703125,
      "loss": 0.4212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23295816779136658,
      "rewards/margins": 0.9075163006782532,
      "rewards/rejected": -1.1404744386672974,
      "step": 7760
    },
    {
      "epoch": 1.0155720945448596,
      "grad_norm": 4.164133071899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24715390801429749,
      "logits/rejected": 0.3191604018211365,
      "logps/chosen": -442.6069641113281,
      "logps/rejected": -356.7236328125,
      "loss": 0.1921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.637364387512207,
      "rewards/margins": 1.9942117929458618,
      "rewards/rejected": -2.6315760612487793,
      "step": 7761
    },
    {
      "epoch": 1.0157029524822114,
      "grad_norm": 4.2418532371521,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023583631962537766,
      "logits/rejected": 0.03687851503491402,
      "logps/chosen": -416.25311279296875,
      "logps/rejected": -434.0631103515625,
      "loss": 0.2276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17328664660453796,
      "rewards/margins": 1.8934324979782104,
      "rewards/rejected": -1.7201457023620605,
      "step": 7762
    },
    {
      "epoch": 1.0158338104195632,
      "grad_norm": 7.4651899337768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017134737223386765,
      "logits/rejected": 0.020067669451236725,
      "logps/chosen": -397.4496154785156,
      "logps/rejected": -480.57354736328125,
      "loss": 0.5369,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05064164474606514,
      "rewards/margins": 1.057192087173462,
      "rewards/rejected": -1.107833743095398,
      "step": 7763
    },
    {
      "epoch": 1.015964668356915,
      "grad_norm": 10.854020118713379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1607709378004074,
      "logits/rejected": -0.036476828157901764,
      "logps/chosen": -374.644287109375,
      "logps/rejected": -475.3174743652344,
      "loss": 1.1199,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2298510074615479,
      "rewards/margins": -0.2638216018676758,
      "rewards/rejected": -0.9660294055938721,
      "step": 7764
    },
    {
      "epoch": 1.0160955262942668,
      "grad_norm": 5.7584404945373535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18946011364459991,
      "logits/rejected": 0.11754612624645233,
      "logps/chosen": -462.0801086425781,
      "logps/rejected": -400.4444580078125,
      "loss": 0.3397,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10991573333740234,
      "rewards/margins": 1.2663861513137817,
      "rewards/rejected": -1.3763020038604736,
      "step": 7765
    },
    {
      "epoch": 1.0162263842316186,
      "grad_norm": 12.228705406188965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0668429359793663,
      "logits/rejected": -0.10764911770820618,
      "logps/chosen": -628.0543823242188,
      "logps/rejected": -594.3727416992188,
      "loss": 1.2272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32679253816604614,
      "rewards/margins": -0.24540424346923828,
      "rewards/rejected": -0.08138832449913025,
      "step": 7766
    },
    {
      "epoch": 1.0163572421689704,
      "grad_norm": 6.8111491203308105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2981743812561035,
      "logits/rejected": -0.28744029998779297,
      "logps/chosen": -601.0504150390625,
      "logps/rejected": -454.30804443359375,
      "loss": 0.4353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16127124428749084,
      "rewards/margins": 1.0579224824905396,
      "rewards/rejected": -1.2191938161849976,
      "step": 7767
    },
    {
      "epoch": 1.0164881001063222,
      "grad_norm": 7.0446271896362305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010247059166431427,
      "logits/rejected": 0.043697506189346313,
      "logps/chosen": -370.97833251953125,
      "logps/rejected": -355.2843933105469,
      "loss": 0.7006,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03772889822721481,
      "rewards/margins": 0.6509783267974854,
      "rewards/rejected": -0.6132494211196899,
      "step": 7768
    },
    {
      "epoch": 1.0166189580436737,
      "grad_norm": 7.880913257598877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0366862416267395,
      "logits/rejected": 0.13411523401737213,
      "logps/chosen": -521.494384765625,
      "logps/rejected": -576.4769287109375,
      "loss": 0.3883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7350714802742004,
      "rewards/margins": 1.789418339729309,
      "rewards/rejected": -2.5244898796081543,
      "step": 7769
    },
    {
      "epoch": 1.0167498159810255,
      "grad_norm": 3.448286294937134,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0620412677526474,
      "logits/rejected": -0.011858535930514336,
      "logps/chosen": -706.6015625,
      "logps/rejected": -676.280517578125,
      "loss": 0.1856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.330526351928711,
      "rewards/margins": 1.7434425354003906,
      "rewards/rejected": -0.4129162132740021,
      "step": 7770
    },
    {
      "epoch": 1.0168806739183773,
      "grad_norm": 5.502954959869385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13865593075752258,
      "logits/rejected": -0.1264384686946869,
      "logps/chosen": -395.4121398925781,
      "logps/rejected": -497.8321533203125,
      "loss": 0.3285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4295448660850525,
      "rewards/margins": 1.704350233078003,
      "rewards/rejected": -1.2748053073883057,
      "step": 7771
    },
    {
      "epoch": 1.0170115318557291,
      "grad_norm": 3.8552327156066895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47342774271965027,
      "logits/rejected": -0.2715476155281067,
      "logps/chosen": -335.94293212890625,
      "logps/rejected": -552.5170288085938,
      "loss": 0.3346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5043737888336182,
      "rewards/margins": 1.2646691799163818,
      "rewards/rejected": -1.7690430879592896,
      "step": 7772
    },
    {
      "epoch": 1.017142389793081,
      "grad_norm": 2.104936122894287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004975717514753342,
      "logits/rejected": -0.06265886127948761,
      "logps/chosen": -668.85693359375,
      "logps/rejected": -672.9130859375,
      "loss": 0.0887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35651129484176636,
      "rewards/margins": 3.0889976024627686,
      "rewards/rejected": -2.7324860095977783,
      "step": 7773
    },
    {
      "epoch": 1.0172732477304327,
      "grad_norm": 7.611443519592285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19124969840049744,
      "logits/rejected": -0.10277949273586273,
      "logps/chosen": -497.7806396484375,
      "logps/rejected": -620.864990234375,
      "loss": 0.4685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0013372339308261871,
      "rewards/margins": 1.0035393238067627,
      "rewards/rejected": -1.002202033996582,
      "step": 7774
    },
    {
      "epoch": 1.0174041056677845,
      "grad_norm": 8.937891006469727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03889468312263489,
      "logits/rejected": -0.09373176097869873,
      "logps/chosen": -389.1241760253906,
      "logps/rejected": -371.7430725097656,
      "loss": 0.5746,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20865042507648468,
      "rewards/margins": 0.8354484438896179,
      "rewards/rejected": -1.0440988540649414,
      "step": 7775
    },
    {
      "epoch": 1.017534963605136,
      "grad_norm": 12.350361824035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18446919322013855,
      "logits/rejected": -0.06306222081184387,
      "logps/chosen": -504.8277587890625,
      "logps/rejected": -316.5749816894531,
      "loss": 0.858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5487731695175171,
      "rewards/margins": 0.6425384283065796,
      "rewards/rejected": -1.1913117170333862,
      "step": 7776
    },
    {
      "epoch": 1.0176658215424879,
      "grad_norm": 6.247649192810059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38683196902275085,
      "logits/rejected": 0.1352369338274002,
      "logps/chosen": -448.8059387207031,
      "logps/rejected": -429.4013671875,
      "loss": 0.3968,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2598114013671875,
      "rewards/margins": 0.9375931024551392,
      "rewards/rejected": -1.1974046230316162,
      "step": 7777
    },
    {
      "epoch": 1.0177966794798396,
      "grad_norm": 9.293632507324219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0855177491903305,
      "logits/rejected": 0.024542562663555145,
      "logps/chosen": -494.83160400390625,
      "logps/rejected": -527.5662841796875,
      "loss": 0.5185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10540599375963211,
      "rewards/margins": 0.8757154941558838,
      "rewards/rejected": -0.9811214208602905,
      "step": 7778
    },
    {
      "epoch": 1.0179275374171914,
      "grad_norm": 4.399265289306641,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05981997400522232,
      "logits/rejected": -0.24111858010292053,
      "logps/chosen": -411.4784851074219,
      "logps/rejected": -415.6130676269531,
      "loss": 0.2381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.00228804349899292,
      "rewards/margins": 1.602486491203308,
      "rewards/rejected": -1.60019850730896,
      "step": 7779
    },
    {
      "epoch": 1.0180583953545432,
      "grad_norm": 9.858793258666992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12284841388463974,
      "logits/rejected": 0.25651347637176514,
      "logps/chosen": -653.6415405273438,
      "logps/rejected": -572.2894897460938,
      "loss": 0.6338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.463909775018692,
      "rewards/margins": 0.4978848695755005,
      "rewards/rejected": -0.9617947340011597,
      "step": 7780
    },
    {
      "epoch": 1.018189253291895,
      "grad_norm": 7.9773759841918945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22230178117752075,
      "logits/rejected": -0.02831978350877762,
      "logps/chosen": -412.3887634277344,
      "logps/rejected": -259.6737060546875,
      "loss": 0.6255,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0046478509902954,
      "rewards/margins": 0.2749347686767578,
      "rewards/rejected": -1.2795826196670532,
      "step": 7781
    },
    {
      "epoch": 1.0183201112292468,
      "grad_norm": 11.816997528076172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11237656325101852,
      "logits/rejected": -0.16331565380096436,
      "logps/chosen": -506.5987548828125,
      "logps/rejected": -467.463623046875,
      "loss": 0.7692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0136979818344116,
      "rewards/margins": 0.18342439830303192,
      "rewards/rejected": -1.19712233543396,
      "step": 7782
    },
    {
      "epoch": 1.0184509691665986,
      "grad_norm": 6.733396530151367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20624913275241852,
      "logits/rejected": 0.13620907068252563,
      "logps/chosen": -683.3993530273438,
      "logps/rejected": -615.876953125,
      "loss": 0.3016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6509498953819275,
      "rewards/margins": 2.0491693019866943,
      "rewards/rejected": -1.398219347000122,
      "step": 7783
    },
    {
      "epoch": 1.0185818271039502,
      "grad_norm": 8.245226860046387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05459347367286682,
      "logits/rejected": 0.05859369412064552,
      "logps/chosen": -378.7430725097656,
      "logps/rejected": -433.57366943359375,
      "loss": 0.4534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24306489527225494,
      "rewards/margins": 0.894519031047821,
      "rewards/rejected": -1.13758385181427,
      "step": 7784
    },
    {
      "epoch": 1.018712685041302,
      "grad_norm": 4.6930437088012695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07785215973854065,
      "logits/rejected": -0.037186525762081146,
      "logps/chosen": -348.1258544921875,
      "logps/rejected": -370.14373779296875,
      "loss": 0.3267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22436219453811646,
      "rewards/margins": 1.833749771118164,
      "rewards/rejected": -1.6093873977661133,
      "step": 7785
    },
    {
      "epoch": 1.0188435429786538,
      "grad_norm": 7.1388936042785645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2551910877227783,
      "logits/rejected": -0.27936673164367676,
      "logps/chosen": -397.6867370605469,
      "logps/rejected": -441.8426208496094,
      "loss": 0.4124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11702186614274979,
      "rewards/margins": 0.9419583082199097,
      "rewards/rejected": -0.8249364495277405,
      "step": 7786
    },
    {
      "epoch": 1.0189744009160056,
      "grad_norm": 8.17094898223877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06169953569769859,
      "logits/rejected": -0.1696225255727768,
      "logps/chosen": -397.47076416015625,
      "logps/rejected": -479.0504150390625,
      "loss": 0.3078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5238732099533081,
      "rewards/margins": 1.77641761302948,
      "rewards/rejected": -2.300290584564209,
      "step": 7787
    },
    {
      "epoch": 1.0191052588533573,
      "grad_norm": 8.454011917114258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06625869870185852,
      "logits/rejected": -0.16873623430728912,
      "logps/chosen": -311.45477294921875,
      "logps/rejected": -320.96722412109375,
      "loss": 0.5927,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7037094235420227,
      "rewards/margins": 0.49836447834968567,
      "rewards/rejected": -1.2020739316940308,
      "step": 7788
    },
    {
      "epoch": 1.0192361167907091,
      "grad_norm": 6.874248504638672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21509791910648346,
      "logits/rejected": 0.11864076554775238,
      "logps/chosen": -460.34375,
      "logps/rejected": -378.683349609375,
      "loss": 0.5507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45140188932418823,
      "rewards/margins": 0.46386992931365967,
      "rewards/rejected": -0.9152718186378479,
      "step": 7789
    },
    {
      "epoch": 1.019366974728061,
      "grad_norm": 8.051173210144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17902185022830963,
      "logits/rejected": -0.43819865584373474,
      "logps/chosen": -414.87872314453125,
      "logps/rejected": -392.7330322265625,
      "loss": 0.6842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9494915008544922,
      "rewards/margins": 0.5400848388671875,
      "rewards/rejected": -2.4895761013031006,
      "step": 7790
    },
    {
      "epoch": 1.0194978326654127,
      "grad_norm": 5.527081489562988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053279809653759,
      "logits/rejected": -0.3090988099575043,
      "logps/chosen": -488.89898681640625,
      "logps/rejected": -350.73504638671875,
      "loss": 0.2148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3806636929512024,
      "rewards/margins": 2.4521164894104004,
      "rewards/rejected": -2.0714526176452637,
      "step": 7791
    },
    {
      "epoch": 1.0196286906027643,
      "grad_norm": 8.709516525268555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1628185510635376,
      "logits/rejected": -0.05800515413284302,
      "logps/chosen": -482.4527282714844,
      "logps/rejected": -707.1660766601562,
      "loss": 0.5386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9399732351303101,
      "rewards/margins": 0.7204459309577942,
      "rewards/rejected": -1.660419225692749,
      "step": 7792
    },
    {
      "epoch": 1.019759548540116,
      "grad_norm": 5.621639728546143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3470403254032135,
      "logits/rejected": -0.24590915441513062,
      "logps/chosen": -358.052734375,
      "logps/rejected": -315.81011962890625,
      "loss": 0.3661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.421830952167511,
      "rewards/margins": 1.1263599395751953,
      "rewards/rejected": -0.7045289874076843,
      "step": 7793
    },
    {
      "epoch": 1.0198904064774679,
      "grad_norm": 5.246467113494873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10507720708847046,
      "logits/rejected": -0.1429653912782669,
      "logps/chosen": -600.306884765625,
      "logps/rejected": -543.3623657226562,
      "loss": 0.2883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17605435848236084,
      "rewards/margins": 1.353024959564209,
      "rewards/rejected": -1.1769704818725586,
      "step": 7794
    },
    {
      "epoch": 1.0200212644148197,
      "grad_norm": 12.217108726501465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10091027617454529,
      "logits/rejected": -0.08150745183229446,
      "logps/chosen": -607.6992797851562,
      "logps/rejected": -525.4755859375,
      "loss": 0.9042,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11866013705730438,
      "rewards/margins": 0.6146340370178223,
      "rewards/rejected": -0.4959738254547119,
      "step": 7795
    },
    {
      "epoch": 1.0201521223521715,
      "grad_norm": 6.3448405265808105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16898077726364136,
      "logits/rejected": -0.19907307624816895,
      "logps/chosen": -466.802978515625,
      "logps/rejected": -490.150146484375,
      "loss": 0.3538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012008503079414368,
      "rewards/margins": 1.5490119457244873,
      "rewards/rejected": -1.5370036363601685,
      "step": 7796
    },
    {
      "epoch": 1.0202829802895232,
      "grad_norm": 4.847346782684326,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007340624928474426,
      "logits/rejected": -0.17265480756759644,
      "logps/chosen": -345.50933837890625,
      "logps/rejected": -276.5830993652344,
      "loss": 0.4579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40354210138320923,
      "rewards/margins": 1.1458015441894531,
      "rewards/rejected": -1.5493438243865967,
      "step": 7797
    },
    {
      "epoch": 1.020413838226875,
      "grad_norm": 8.701895713806152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23772718012332916,
      "logits/rejected": 0.19643552601337433,
      "logps/chosen": -527.3233642578125,
      "logps/rejected": -524.755859375,
      "loss": 0.7112,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6804466247558594,
      "rewards/margins": 0.6305549740791321,
      "rewards/rejected": -1.3110015392303467,
      "step": 7798
    },
    {
      "epoch": 1.0205446961642266,
      "grad_norm": 7.453542709350586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013558074831962585,
      "logits/rejected": -0.3631299138069153,
      "logps/chosen": -654.1505126953125,
      "logps/rejected": -532.2210693359375,
      "loss": 0.3495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25627386569976807,
      "rewards/margins": 1.9914945363998413,
      "rewards/rejected": -2.2477684020996094,
      "step": 7799
    },
    {
      "epoch": 1.0206755541015784,
      "grad_norm": 6.855266571044922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.43488022685050964,
      "logits/rejected": 0.14070600271224976,
      "logps/chosen": -353.5495910644531,
      "logps/rejected": -484.53271484375,
      "loss": 0.4746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04786502569913864,
      "rewards/margins": 1.6784641742706299,
      "rewards/rejected": -1.6305992603302002,
      "step": 7800
    },
    {
      "epoch": 1.0208064120389302,
      "grad_norm": 19.340301513671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20302918553352356,
      "logits/rejected": 0.046671923249959946,
      "logps/chosen": -383.29608154296875,
      "logps/rejected": -276.0997009277344,
      "loss": 0.7475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2503210306167603,
      "rewards/margins": 1.3964332342147827,
      "rewards/rejected": -2.646754264831543,
      "step": 7801
    },
    {
      "epoch": 1.020937269976282,
      "grad_norm": 7.406166076660156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03922104090452194,
      "logits/rejected": -0.17445562779903412,
      "logps/chosen": -488.31134033203125,
      "logps/rejected": -780.8582153320312,
      "loss": 0.5311,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8914918899536133,
      "rewards/margins": 0.7717695236206055,
      "rewards/rejected": -1.6632614135742188,
      "step": 7802
    },
    {
      "epoch": 1.0210681279136338,
      "grad_norm": 5.189061641693115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0953875333070755,
      "logits/rejected": -0.1014220267534256,
      "logps/chosen": -543.1671752929688,
      "logps/rejected": -576.29296875,
      "loss": 0.3772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12075309455394745,
      "rewards/margins": 1.1739033460617065,
      "rewards/rejected": -1.0531502962112427,
      "step": 7803
    },
    {
      "epoch": 1.0211989858509856,
      "grad_norm": 10.11437702178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3415636718273163,
      "logits/rejected": -0.15612977743148804,
      "logps/chosen": -406.75347900390625,
      "logps/rejected": -719.4235229492188,
      "loss": 0.8288,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5090821981430054,
      "rewards/margins": 0.10134509205818176,
      "rewards/rejected": -0.6104273200035095,
      "step": 7804
    },
    {
      "epoch": 1.0213298437883374,
      "grad_norm": 8.910825729370117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035040464252233505,
      "logits/rejected": -0.09168922156095505,
      "logps/chosen": -574.0491333007812,
      "logps/rejected": -518.5535278320312,
      "loss": 0.451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01548156701028347,
      "rewards/margins": 1.6235158443450928,
      "rewards/rejected": -1.6389975547790527,
      "step": 7805
    },
    {
      "epoch": 1.0214607017256891,
      "grad_norm": 6.913255214691162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37647509574890137,
      "logits/rejected": -0.25427478551864624,
      "logps/chosen": -402.87213134765625,
      "logps/rejected": -406.4867248535156,
      "loss": 0.4492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7997950315475464,
      "rewards/margins": 0.9581470489501953,
      "rewards/rejected": -0.1583520770072937,
      "step": 7806
    },
    {
      "epoch": 1.0215915596630407,
      "grad_norm": 7.3002166748046875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11299210786819458,
      "logits/rejected": -0.3902077078819275,
      "logps/chosen": -560.479248046875,
      "logps/rejected": -451.15179443359375,
      "loss": 0.4658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.055002212524414e-05,
      "rewards/margins": 0.9012196063995361,
      "rewards/rejected": -0.9012089967727661,
      "step": 7807
    },
    {
      "epoch": 1.0217224176003925,
      "grad_norm": 6.7503767013549805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006512214429676533,
      "logits/rejected": 0.03079688549041748,
      "logps/chosen": -357.7007141113281,
      "logps/rejected": -326.0718688964844,
      "loss": 0.5516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0881563276052475,
      "rewards/margins": 1.3596431016921997,
      "rewards/rejected": -1.2714868783950806,
      "step": 7808
    },
    {
      "epoch": 1.0218532755377443,
      "grad_norm": 9.589580535888672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10652942955493927,
      "logits/rejected": 0.15877579152584076,
      "logps/chosen": -405.8000183105469,
      "logps/rejected": -668.2792358398438,
      "loss": 0.578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2729806900024414,
      "rewards/margins": 1.5147161483764648,
      "rewards/rejected": -2.7876968383789062,
      "step": 7809
    },
    {
      "epoch": 1.021984133475096,
      "grad_norm": 8.91684627532959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05928432568907738,
      "logits/rejected": -0.28247159719467163,
      "logps/chosen": -396.88958740234375,
      "logps/rejected": -558.4354858398438,
      "loss": 0.6117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8362680673599243,
      "rewards/margins": 0.7533750534057617,
      "rewards/rejected": -2.5896430015563965,
      "step": 7810
    },
    {
      "epoch": 1.0221149914124479,
      "grad_norm": 7.903566837310791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01240910030901432,
      "logits/rejected": 0.06582678854465485,
      "logps/chosen": -550.7640991210938,
      "logps/rejected": -458.7459716796875,
      "loss": 0.5434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16202011704444885,
      "rewards/margins": 1.3641469478607178,
      "rewards/rejected": -1.2021267414093018,
      "step": 7811
    },
    {
      "epoch": 1.0222458493497997,
      "grad_norm": 8.110214233398438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22877809405326843,
      "logits/rejected": 0.20002011954784393,
      "logps/chosen": -395.2095642089844,
      "logps/rejected": -476.2506408691406,
      "loss": 0.4001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.026617228984832764,
      "rewards/margins": 1.5663506984710693,
      "rewards/rejected": -1.5929681062698364,
      "step": 7812
    },
    {
      "epoch": 1.0223767072871515,
      "grad_norm": 4.864604949951172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02807357907295227,
      "logits/rejected": -0.0761738047003746,
      "logps/chosen": -502.01239013671875,
      "logps/rejected": -401.3442077636719,
      "loss": 0.3539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2555086016654968,
      "rewards/margins": 1.2526466846466064,
      "rewards/rejected": -0.9971380233764648,
      "step": 7813
    },
    {
      "epoch": 1.0225075652245033,
      "grad_norm": 6.407832622528076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19819165766239166,
      "logits/rejected": -0.07562527060508728,
      "logps/chosen": -577.9697875976562,
      "logps/rejected": -558.08642578125,
      "loss": 0.5622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.571087658405304,
      "rewards/margins": 0.47266101837158203,
      "rewards/rejected": 0.09842666983604431,
      "step": 7814
    },
    {
      "epoch": 1.0226384231618548,
      "grad_norm": 9.006710052490234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1431933045387268,
      "logits/rejected": -0.0035052988678216934,
      "logps/chosen": -442.233154296875,
      "logps/rejected": -512.2355346679688,
      "loss": 0.7178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5213294625282288,
      "rewards/margins": 0.3897401690483093,
      "rewards/rejected": -0.9110697507858276,
      "step": 7815
    },
    {
      "epoch": 1.0227692810992066,
      "grad_norm": 12.124129295349121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24106702208518982,
      "logits/rejected": 0.02969451993703842,
      "logps/chosen": -393.0593566894531,
      "logps/rejected": -442.12786865234375,
      "loss": 0.6216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8312832117080688,
      "rewards/margins": 0.7500917315483093,
      "rewards/rejected": -1.581375002861023,
      "step": 7816
    },
    {
      "epoch": 1.0229001390365584,
      "grad_norm": 5.774175643920898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05492883175611496,
      "logits/rejected": 0.004375746939331293,
      "logps/chosen": -357.80670166015625,
      "logps/rejected": -397.2860412597656,
      "loss": 0.4085,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11465586721897125,
      "rewards/margins": 0.792124330997467,
      "rewards/rejected": -0.6774684190750122,
      "step": 7817
    },
    {
      "epoch": 1.0230309969739102,
      "grad_norm": 16.152606964111328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1535857617855072,
      "logits/rejected": 0.04454922676086426,
      "logps/chosen": -590.320556640625,
      "logps/rejected": -482.09765625,
      "loss": 1.185,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5782464146614075,
      "rewards/margins": -0.28981614112854004,
      "rewards/rejected": -0.2884300947189331,
      "step": 7818
    },
    {
      "epoch": 1.023161854911262,
      "grad_norm": 6.828316688537598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1613912284374237,
      "logits/rejected": -0.06801632046699524,
      "logps/chosen": -415.84881591796875,
      "logps/rejected": -300.9639892578125,
      "loss": 0.5268,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4074394106864929,
      "rewards/margins": 0.817742109298706,
      "rewards/rejected": -0.41030263900756836,
      "step": 7819
    },
    {
      "epoch": 1.0232927128486138,
      "grad_norm": 3.9413015842437744,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11879672855138779,
      "logits/rejected": -0.2852157950401306,
      "logps/chosen": -373.7528076171875,
      "logps/rejected": -395.347900390625,
      "loss": 0.2326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6396152973175049,
      "rewards/margins": 1.6173959970474243,
      "rewards/rejected": -0.9777806997299194,
      "step": 7820
    },
    {
      "epoch": 1.0234235707859656,
      "grad_norm": 10.013760566711426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04074045270681381,
      "logits/rejected": -0.19324423372745514,
      "logps/chosen": -520.0140380859375,
      "logps/rejected": -328.28790283203125,
      "loss": 0.6467,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08349305391311646,
      "rewards/margins": 0.797726035118103,
      "rewards/rejected": -0.8812191486358643,
      "step": 7821
    },
    {
      "epoch": 1.0235544287233171,
      "grad_norm": 5.449825286865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14168663322925568,
      "logits/rejected": -0.09482391923666,
      "logps/chosen": -422.3912658691406,
      "logps/rejected": -543.7854614257812,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46297627687454224,
      "rewards/margins": 2.170010805130005,
      "rewards/rejected": -2.6329872608184814,
      "step": 7822
    },
    {
      "epoch": 1.023685286660669,
      "grad_norm": 8.827363014221191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3573295474052429,
      "logits/rejected": -0.18624427914619446,
      "logps/chosen": -526.0125122070312,
      "logps/rejected": -577.5098876953125,
      "loss": 0.4528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5159433484077454,
      "rewards/margins": 1.233723759651184,
      "rewards/rejected": -1.7496672868728638,
      "step": 7823
    },
    {
      "epoch": 1.0238161445980207,
      "grad_norm": 5.273649215698242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12314863502979279,
      "logits/rejected": 0.07953470200300217,
      "logps/chosen": -515.671142578125,
      "logps/rejected": -549.566162109375,
      "loss": 0.209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.015976712107658386,
      "rewards/margins": 1.9930813312530518,
      "rewards/rejected": -1.977104663848877,
      "step": 7824
    },
    {
      "epoch": 1.0239470025353725,
      "grad_norm": 5.652409553527832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18736425042152405,
      "logits/rejected": -0.2406141608953476,
      "logps/chosen": -485.4881286621094,
      "logps/rejected": -371.7640075683594,
      "loss": 0.4325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19099333882331848,
      "rewards/margins": 0.7470773458480835,
      "rewards/rejected": -0.5560840964317322,
      "step": 7825
    },
    {
      "epoch": 1.0240778604727243,
      "grad_norm": 5.158202171325684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027954570949077606,
      "logits/rejected": -0.01881246268749237,
      "logps/chosen": -624.2727661132812,
      "logps/rejected": -514.37255859375,
      "loss": 0.3277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2479310929775238,
      "rewards/margins": 1.4032411575317383,
      "rewards/rejected": -1.6511722803115845,
      "step": 7826
    },
    {
      "epoch": 1.024208718410076,
      "grad_norm": 4.344261169433594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03540012240409851,
      "logits/rejected": -0.11168313026428223,
      "logps/chosen": -557.60595703125,
      "logps/rejected": -454.578369140625,
      "loss": 0.2556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07009492069482803,
      "rewards/margins": 1.5293265581130981,
      "rewards/rejected": -1.4592314958572388,
      "step": 7827
    },
    {
      "epoch": 1.024339576347428,
      "grad_norm": 6.209204196929932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03423818200826645,
      "logits/rejected": 0.13213863968849182,
      "logps/chosen": -401.01544189453125,
      "logps/rejected": -336.9296875,
      "loss": 0.3545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15897616744041443,
      "rewards/margins": 1.4581878185272217,
      "rewards/rejected": -1.617163896560669,
      "step": 7828
    },
    {
      "epoch": 1.0244704342847797,
      "grad_norm": 11.028549194335938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17816422879695892,
      "logits/rejected": 0.08833543211221695,
      "logps/chosen": -409.8564147949219,
      "logps/rejected": -570.0699462890625,
      "loss": 0.7992,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07365190982818604,
      "rewards/margins": 0.1118960976600647,
      "rewards/rejected": -0.18554799258708954,
      "step": 7829
    },
    {
      "epoch": 1.0246012922221313,
      "grad_norm": 4.004350662231445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15885044634342194,
      "logits/rejected": -0.4623448848724365,
      "logps/chosen": -639.297607421875,
      "logps/rejected": -476.2032775878906,
      "loss": 0.2401,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38176673650741577,
      "rewards/margins": 1.7217553853988647,
      "rewards/rejected": -1.3399887084960938,
      "step": 7830
    },
    {
      "epoch": 1.024732150159483,
      "grad_norm": 8.605775833129883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09299002587795258,
      "logits/rejected": -0.050567615777254105,
      "logps/chosen": -425.6356201171875,
      "logps/rejected": -527.1590576171875,
      "loss": 0.7062,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.689449667930603,
      "rewards/margins": 0.4265558421611786,
      "rewards/rejected": -1.1160054206848145,
      "step": 7831
    },
    {
      "epoch": 1.0248630080968348,
      "grad_norm": 5.259068965911865,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22119355201721191,
      "logits/rejected": 0.1948344111442566,
      "logps/chosen": -496.93377685546875,
      "logps/rejected": -387.300048828125,
      "loss": 0.3845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24436643719673157,
      "rewards/margins": 1.2194126844406128,
      "rewards/rejected": -0.9750461578369141,
      "step": 7832
    },
    {
      "epoch": 1.0249938660341866,
      "grad_norm": 8.51801872253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23128069937229156,
      "logits/rejected": 0.0604279488325119,
      "logps/chosen": -508.22076416015625,
      "logps/rejected": -435.3990173339844,
      "loss": 0.5922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08958913385868073,
      "rewards/margins": 0.7256650924682617,
      "rewards/rejected": -0.6360759139060974,
      "step": 7833
    },
    {
      "epoch": 1.0251247239715384,
      "grad_norm": 4.684774398803711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23683567345142365,
      "logits/rejected": -0.2344473898410797,
      "logps/chosen": -416.194580078125,
      "logps/rejected": -390.75518798828125,
      "loss": 0.3167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6145316958427429,
      "rewards/margins": 1.9863709211349487,
      "rewards/rejected": -1.3718390464782715,
      "step": 7834
    },
    {
      "epoch": 1.0252555819088902,
      "grad_norm": 9.033212661743164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16922840476036072,
      "logits/rejected": 0.25212597846984863,
      "logps/chosen": -401.415283203125,
      "logps/rejected": -414.86016845703125,
      "loss": 0.8558,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5889114141464233,
      "rewards/margins": -0.1326366662979126,
      "rewards/rejected": -0.4562748074531555,
      "step": 7835
    },
    {
      "epoch": 1.025386439846242,
      "grad_norm": 7.288498878479004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0681903064250946,
      "logits/rejected": -0.07803814858198166,
      "logps/chosen": -390.3853454589844,
      "logps/rejected": -266.9727783203125,
      "loss": 0.4418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024300744757056236,
      "rewards/margins": 1.0546939373016357,
      "rewards/rejected": -1.0303930044174194,
      "step": 7836
    },
    {
      "epoch": 1.0255172977835938,
      "grad_norm": 5.633426189422607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22108346223831177,
      "logits/rejected": -0.3584344983100891,
      "logps/chosen": -646.6797485351562,
      "logps/rejected": -464.84661865234375,
      "loss": 0.2677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0018608570098876953,
      "rewards/margins": 1.8844095468521118,
      "rewards/rejected": -1.886270523071289,
      "step": 7837
    },
    {
      "epoch": 1.0256481557209454,
      "grad_norm": 14.896463394165039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2766755521297455,
      "logits/rejected": 0.05837758630514145,
      "logps/chosen": -427.1594543457031,
      "logps/rejected": -358.05987548828125,
      "loss": 0.9741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.212599277496338,
      "rewards/margins": -0.04869970679283142,
      "rewards/rejected": -1.163899540901184,
      "step": 7838
    },
    {
      "epoch": 1.0257790136582972,
      "grad_norm": 7.925650119781494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05555497854948044,
      "logits/rejected": 0.08459222316741943,
      "logps/chosen": -553.2469482421875,
      "logps/rejected": -481.3609924316406,
      "loss": 0.4274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3528936505317688,
      "rewards/margins": 1.3212010860443115,
      "rewards/rejected": -1.6740949153900146,
      "step": 7839
    },
    {
      "epoch": 1.025909871595649,
      "grad_norm": 9.324483871459961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003130229189991951,
      "logits/rejected": -0.09717225283384323,
      "logps/chosen": -406.7772521972656,
      "logps/rejected": -458.82147216796875,
      "loss": 0.719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6016123294830322,
      "rewards/margins": 0.7588402032852173,
      "rewards/rejected": -1.3604525327682495,
      "step": 7840
    },
    {
      "epoch": 1.0260407295330007,
      "grad_norm": 10.253244400024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23946815729141235,
      "logits/rejected": -0.2440292090177536,
      "logps/chosen": -285.9742431640625,
      "logps/rejected": -348.1298828125,
      "loss": 0.6444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28428831696510315,
      "rewards/margins": 1.3121622800827026,
      "rewards/rejected": -1.596450686454773,
      "step": 7841
    },
    {
      "epoch": 1.0261715874703525,
      "grad_norm": 7.120299816131592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18353226780891418,
      "logits/rejected": 0.08034180104732513,
      "logps/chosen": -481.17266845703125,
      "logps/rejected": -478.1837463378906,
      "loss": 0.4775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14379289746284485,
      "rewards/margins": 0.8410397171974182,
      "rewards/rejected": -0.9848325848579407,
      "step": 7842
    },
    {
      "epoch": 1.0263024454077043,
      "grad_norm": 7.0218915939331055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0018039196729660034,
      "logits/rejected": 0.010117953643202782,
      "logps/chosen": -505.5023193359375,
      "logps/rejected": -357.73492431640625,
      "loss": 0.4234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2133277952671051,
      "rewards/margins": 1.1238734722137451,
      "rewards/rejected": -1.3372013568878174,
      "step": 7843
    },
    {
      "epoch": 1.0264333033450561,
      "grad_norm": 8.032783508300781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.310026615858078,
      "logits/rejected": -0.4786924719810486,
      "logps/chosen": -330.8304138183594,
      "logps/rejected": -484.994873046875,
      "loss": 0.5663,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2656912803649902,
      "rewards/margins": 1.0559873580932617,
      "rewards/rejected": -2.321678400039673,
      "step": 7844
    },
    {
      "epoch": 1.0265641612824077,
      "grad_norm": 8.906801223754883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06293035298585892,
      "logits/rejected": -0.19492371380329132,
      "logps/chosen": -528.7571411132812,
      "logps/rejected": -536.4182739257812,
      "loss": 0.5716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3051721453666687,
      "rewards/margins": 1.0060209035873413,
      "rewards/rejected": -0.7008487582206726,
      "step": 7845
    },
    {
      "epoch": 1.0266950192197595,
      "grad_norm": 4.743906021118164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017332304269075394,
      "logits/rejected": -0.2170693427324295,
      "logps/chosen": -546.213623046875,
      "logps/rejected": -451.98529052734375,
      "loss": 0.1685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3696826994419098,
      "rewards/margins": 2.357388973236084,
      "rewards/rejected": -1.9877063035964966,
      "step": 7846
    },
    {
      "epoch": 1.0268258771571113,
      "grad_norm": 3.822326898574829,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10413390398025513,
      "logits/rejected": 0.03312661126255989,
      "logps/chosen": -355.9535827636719,
      "logps/rejected": -443.49993896484375,
      "loss": 0.2917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.116835817694664,
      "rewards/margins": 1.6608872413635254,
      "rewards/rejected": -1.7777228355407715,
      "step": 7847
    },
    {
      "epoch": 1.026956735094463,
      "grad_norm": 6.031589508056641,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25919437408447266,
      "logits/rejected": -0.21557891368865967,
      "logps/chosen": -463.83001708984375,
      "logps/rejected": -457.8952941894531,
      "loss": 0.3661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8200654983520508,
      "rewards/margins": 1.132655143737793,
      "rewards/rejected": -0.3125896453857422,
      "step": 7848
    },
    {
      "epoch": 1.0270875930318148,
      "grad_norm": 6.2947492599487305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06839292496442795,
      "logits/rejected": 0.12865379452705383,
      "logps/chosen": -361.45526123046875,
      "logps/rejected": -383.19610595703125,
      "loss": 0.3611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3859178423881531,
      "rewards/margins": 1.485520601272583,
      "rewards/rejected": -1.0996026992797852,
      "step": 7849
    },
    {
      "epoch": 1.0272184509691666,
      "grad_norm": 6.86135196685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2564137876033783,
      "logits/rejected": -0.2561003565788269,
      "logps/chosen": -415.692138671875,
      "logps/rejected": -388.8063659667969,
      "loss": 0.483,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.059061624109745026,
      "rewards/margins": 0.8029050827026367,
      "rewards/rejected": -0.7438434958457947,
      "step": 7850
    },
    {
      "epoch": 1.0273493089065184,
      "grad_norm": 8.608003616333008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23020489513874054,
      "logits/rejected": -0.1061563566327095,
      "logps/chosen": -452.3163146972656,
      "logps/rejected": -350.4972229003906,
      "loss": 0.4627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5364923477172852,
      "rewards/margins": 1.090836524963379,
      "rewards/rejected": -0.5543441772460938,
      "step": 7851
    },
    {
      "epoch": 1.0274801668438702,
      "grad_norm": 8.159579277038574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08713100105524063,
      "logits/rejected": -0.03464963659644127,
      "logps/chosen": -420.5902404785156,
      "logps/rejected": -438.1873779296875,
      "loss": 0.6295,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.393824964761734,
      "rewards/margins": 0.6016930937767029,
      "rewards/rejected": -0.20786818861961365,
      "step": 7852
    },
    {
      "epoch": 1.0276110247812218,
      "grad_norm": 12.280241012573242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09834305197000504,
      "logits/rejected": -0.0689040869474411,
      "logps/chosen": -469.2095031738281,
      "logps/rejected": -434.0479736328125,
      "loss": 0.918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2104763984680176,
      "rewards/margins": -0.11559998989105225,
      "rewards/rejected": -1.0948764085769653,
      "step": 7853
    },
    {
      "epoch": 1.0277418827185736,
      "grad_norm": 4.502505302429199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20669236779212952,
      "logits/rejected": -0.16929790377616882,
      "logps/chosen": -515.898681640625,
      "logps/rejected": -503.0388488769531,
      "loss": 0.2873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20239083468914032,
      "rewards/margins": 1.4880685806274414,
      "rewards/rejected": -1.2856775522232056,
      "step": 7854
    },
    {
      "epoch": 1.0278727406559254,
      "grad_norm": 3.82961368560791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023317795246839523,
      "logits/rejected": 0.05262262374162674,
      "logps/chosen": -399.39599609375,
      "logps/rejected": -503.78167724609375,
      "loss": 0.1926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09658887982368469,
      "rewards/margins": 2.1291964054107666,
      "rewards/rejected": -2.0326075553894043,
      "step": 7855
    },
    {
      "epoch": 1.0280035985932772,
      "grad_norm": 12.89474868774414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10018744319677353,
      "logits/rejected": -0.35821533203125,
      "logps/chosen": -575.78955078125,
      "logps/rejected": -662.91943359375,
      "loss": 0.6251,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07578851282596588,
      "rewards/margins": 1.1224071979522705,
      "rewards/rejected": -1.1981956958770752,
      "step": 7856
    },
    {
      "epoch": 1.028134456530629,
      "grad_norm": 10.468801498413086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36758753657341003,
      "logits/rejected": -0.25032129883766174,
      "logps/chosen": -492.57061767578125,
      "logps/rejected": -509.94171142578125,
      "loss": 0.6209,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5682371258735657,
      "rewards/margins": 0.5167213678359985,
      "rewards/rejected": -1.0849584341049194,
      "step": 7857
    },
    {
      "epoch": 1.0282653144679808,
      "grad_norm": 3.2459182739257812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07700061053037643,
      "logits/rejected": 0.06259749829769135,
      "logps/chosen": -575.6417236328125,
      "logps/rejected": -468.2745361328125,
      "loss": 0.1371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21351473033428192,
      "rewards/margins": 2.156935214996338,
      "rewards/rejected": -2.3704497814178467,
      "step": 7858
    },
    {
      "epoch": 1.0283961724053325,
      "grad_norm": 5.490516662597656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2704586088657379,
      "logits/rejected": -0.12857365608215332,
      "logps/chosen": -342.087890625,
      "logps/rejected": -391.4931640625,
      "loss": 0.3301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6632868051528931,
      "rewards/margins": 1.151503324508667,
      "rewards/rejected": -0.488216370344162,
      "step": 7859
    },
    {
      "epoch": 1.0285270303426843,
      "grad_norm": 5.831356048583984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13555775582790375,
      "logits/rejected": -0.2124710977077484,
      "logps/chosen": -301.667236328125,
      "logps/rejected": -286.4454650878906,
      "loss": 0.6349,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00413435697555542,
      "rewards/margins": 0.5902239084243774,
      "rewards/rejected": -0.586089551448822,
      "step": 7860
    },
    {
      "epoch": 1.028657888280036,
      "grad_norm": 5.775660991668701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23580825328826904,
      "logits/rejected": -0.04564272612333298,
      "logps/chosen": -515.6419067382812,
      "logps/rejected": -343.2825927734375,
      "loss": 0.4123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.419180691242218,
      "rewards/margins": 1.569169282913208,
      "rewards/rejected": -1.9883499145507812,
      "step": 7861
    },
    {
      "epoch": 1.0287887462173877,
      "grad_norm": 9.354406356811523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.337067186832428,
      "logits/rejected": -0.46937865018844604,
      "logps/chosen": -457.4942626953125,
      "logps/rejected": -379.1400451660156,
      "loss": 0.5235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0405004620552063,
      "rewards/margins": 0.7645586729049683,
      "rewards/rejected": -0.8050590753555298,
      "step": 7862
    },
    {
      "epoch": 1.0289196041547395,
      "grad_norm": 4.690953731536865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3019702732563019,
      "logits/rejected": -0.29480254650115967,
      "logps/chosen": -470.1260986328125,
      "logps/rejected": -555.5582275390625,
      "loss": 0.244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1780162900686264,
      "rewards/margins": 1.7227014303207397,
      "rewards/rejected": -1.5446851253509521,
      "step": 7863
    },
    {
      "epoch": 1.0290504620920913,
      "grad_norm": 7.406283855438232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.386497437953949,
      "logits/rejected": 0.17597946524620056,
      "logps/chosen": -447.2903747558594,
      "logps/rejected": -442.960693359375,
      "loss": 0.368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7290356159210205,
      "rewards/margins": 1.694702386856079,
      "rewards/rejected": -0.9656667113304138,
      "step": 7864
    },
    {
      "epoch": 1.029181320029443,
      "grad_norm": 6.262869358062744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1646983027458191,
      "logits/rejected": -0.24614845216274261,
      "logps/chosen": -442.36163330078125,
      "logps/rejected": -417.3370056152344,
      "loss": 0.3386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7377241849899292,
      "rewards/margins": 1.6628532409667969,
      "rewards/rejected": -0.9251291751861572,
      "step": 7865
    },
    {
      "epoch": 1.0293121779667949,
      "grad_norm": 5.704614162445068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028956428170204163,
      "logits/rejected": -0.027471400797367096,
      "logps/chosen": -554.3064575195312,
      "logps/rejected": -378.5548400878906,
      "loss": 0.37,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.056656837463378906,
      "rewards/margins": 1.7444984912872314,
      "rewards/rejected": -1.8011553287506104,
      "step": 7866
    },
    {
      "epoch": 1.0294430359041467,
      "grad_norm": 7.328380107879639,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05758806690573692,
      "logits/rejected": -0.06843531876802444,
      "logps/chosen": -602.2564697265625,
      "logps/rejected": -599.2517700195312,
      "loss": 0.4797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08245107531547546,
      "rewards/margins": 1.6501797437667847,
      "rewards/rejected": -1.5677287578582764,
      "step": 7867
    },
    {
      "epoch": 1.0295738938414982,
      "grad_norm": 2.992893934249878,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3001507818698883,
      "logits/rejected": -0.19206592440605164,
      "logps/chosen": -459.8753662109375,
      "logps/rejected": -746.3966064453125,
      "loss": 0.1346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6255218982696533,
      "rewards/margins": 2.6352415084838867,
      "rewards/rejected": -2.0097193717956543,
      "step": 7868
    },
    {
      "epoch": 1.02970475177885,
      "grad_norm": 9.806577682495117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27951523661613464,
      "logits/rejected": 0.16459837555885315,
      "logps/chosen": -432.41241455078125,
      "logps/rejected": -471.32733154296875,
      "loss": 0.6231,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3375701904296875,
      "rewards/margins": 0.6872640252113342,
      "rewards/rejected": -1.024834156036377,
      "step": 7869
    },
    {
      "epoch": 1.0298356097162018,
      "grad_norm": 7.883602619171143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07594176381826401,
      "logits/rejected": 0.20437054336071014,
      "logps/chosen": -517.3718872070312,
      "logps/rejected": -480.5565185546875,
      "loss": 0.4895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2995448708534241,
      "rewards/margins": 0.6721481680870056,
      "rewards/rejected": -0.9716930985450745,
      "step": 7870
    },
    {
      "epoch": 1.0299664676535536,
      "grad_norm": 9.792774200439453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4339599311351776,
      "logits/rejected": -0.4553503096103668,
      "logps/chosen": -272.58111572265625,
      "logps/rejected": -325.1197204589844,
      "loss": 0.9848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6198278665542603,
      "rewards/margins": -0.10505926609039307,
      "rewards/rejected": -0.5147685408592224,
      "step": 7871
    },
    {
      "epoch": 1.0300973255909054,
      "grad_norm": 6.560693264007568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3072024881839752,
      "logits/rejected": 0.04512079805135727,
      "logps/chosen": -378.9433898925781,
      "logps/rejected": -564.4598388671875,
      "loss": 0.4431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21578297019004822,
      "rewards/margins": 1.3898358345031738,
      "rewards/rejected": -1.1740530729293823,
      "step": 7872
    },
    {
      "epoch": 1.0302281835282572,
      "grad_norm": 8.489097595214844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20845338702201843,
      "logits/rejected": 0.11601267755031586,
      "logps/chosen": -441.7364501953125,
      "logps/rejected": -370.3628845214844,
      "loss": 0.3892,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6540329456329346,
      "rewards/margins": 1.5625255107879639,
      "rewards/rejected": -0.9084926843643188,
      "step": 7873
    },
    {
      "epoch": 1.030359041465609,
      "grad_norm": 12.456575393676758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08346264809370041,
      "logits/rejected": 0.11739107966423035,
      "logps/chosen": -373.64691162109375,
      "logps/rejected": -380.8166809082031,
      "loss": 0.7227,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30353426933288574,
      "rewards/margins": 0.227056622505188,
      "rewards/rejected": -0.530590832233429,
      "step": 7874
    },
    {
      "epoch": 1.0304898994029608,
      "grad_norm": 7.287161827087402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2510225474834442,
      "logits/rejected": -0.2324361503124237,
      "logps/chosen": -490.0260925292969,
      "logps/rejected": -427.03143310546875,
      "loss": 0.6028,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47838595509529114,
      "rewards/margins": 0.6585524678230286,
      "rewards/rejected": -1.1369385719299316,
      "step": 7875
    },
    {
      "epoch": 1.0306207573403123,
      "grad_norm": 8.223230361938477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12430613487958908,
      "logits/rejected": -0.32608121633529663,
      "logps/chosen": -555.4600830078125,
      "logps/rejected": -644.349365234375,
      "loss": 0.5818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20387153327465057,
      "rewards/margins": 0.5290932059288025,
      "rewards/rejected": -0.7329646944999695,
      "step": 7876
    },
    {
      "epoch": 1.0307516152776641,
      "grad_norm": 5.339005947113037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013611901551485062,
      "logits/rejected": -0.04024767875671387,
      "logps/chosen": -299.86572265625,
      "logps/rejected": -341.54962158203125,
      "loss": 0.3038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3065214157104492,
      "rewards/margins": 1.190269947052002,
      "rewards/rejected": -0.8837486505508423,
      "step": 7877
    },
    {
      "epoch": 1.030882473215016,
      "grad_norm": 7.273698329925537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19362634420394897,
      "logits/rejected": -0.14881794154644012,
      "logps/chosen": -513.8500366210938,
      "logps/rejected": -653.7291259765625,
      "loss": 0.3787,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19901391863822937,
      "rewards/margins": 1.5465565919876099,
      "rewards/rejected": -1.347542643547058,
      "step": 7878
    },
    {
      "epoch": 1.0310133311523677,
      "grad_norm": 7.250523090362549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10623713582754135,
      "logits/rejected": -0.11386498063802719,
      "logps/chosen": -361.5825500488281,
      "logps/rejected": -404.85986328125,
      "loss": 0.4741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22547553479671478,
      "rewards/margins": 0.94917893409729,
      "rewards/rejected": -1.174654483795166,
      "step": 7879
    },
    {
      "epoch": 1.0311441890897195,
      "grad_norm": 16.393163681030273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.036690399050712585,
      "logits/rejected": -0.12687809765338898,
      "logps/chosen": -375.4189758300781,
      "logps/rejected": -419.158447265625,
      "loss": 1.0349,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34062063694000244,
      "rewards/margins": -0.10200750827789307,
      "rewards/rejected": -0.23861315846443176,
      "step": 7880
    },
    {
      "epoch": 1.0312750470270713,
      "grad_norm": 11.013766288757324,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13108223676681519,
      "logits/rejected": 0.041718341410160065,
      "logps/chosen": -458.93560791015625,
      "logps/rejected": -496.01910400390625,
      "loss": 0.8036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.6083601117134094,
      "rewards/margins": 0.4355838894844055,
      "rewards/rejected": 0.1727762222290039,
      "step": 7881
    },
    {
      "epoch": 1.031405904964423,
      "grad_norm": 8.226932525634766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34476256370544434,
      "logits/rejected": -0.15629041194915771,
      "logps/chosen": -375.7019348144531,
      "logps/rejected": -422.5196838378906,
      "loss": 0.4473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5964331030845642,
      "rewards/margins": 1.1031174659729004,
      "rewards/rejected": -0.5066844820976257,
      "step": 7882
    },
    {
      "epoch": 1.0315367629017747,
      "grad_norm": 4.337169170379639,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3656974136829376,
      "logits/rejected": -0.5084589123725891,
      "logps/chosen": -376.31439208984375,
      "logps/rejected": -486.6838073730469,
      "loss": 0.3313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4248884320259094,
      "rewards/margins": 1.5732579231262207,
      "rewards/rejected": -1.998146414756775,
      "step": 7883
    },
    {
      "epoch": 1.0316676208391264,
      "grad_norm": 6.416263103485107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.102182537317276,
      "logits/rejected": -0.07452176511287689,
      "logps/chosen": -491.000244140625,
      "logps/rejected": -408.26019287109375,
      "loss": 0.5544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39535829424858093,
      "rewards/margins": 1.0326117277145386,
      "rewards/rejected": -0.6372534036636353,
      "step": 7884
    },
    {
      "epoch": 1.0317984787764782,
      "grad_norm": 2.2750279903411865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1730102002620697,
      "logits/rejected": -0.2729164659976959,
      "logps/chosen": -464.58624267578125,
      "logps/rejected": -430.61846923828125,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9575964212417603,
      "rewards/margins": 2.4742016792297363,
      "rewards/rejected": -1.516605019569397,
      "step": 7885
    },
    {
      "epoch": 1.03192933671383,
      "grad_norm": 7.561241149902344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2839207053184509,
      "logits/rejected": -0.4168819785118103,
      "logps/chosen": -438.0909118652344,
      "logps/rejected": -395.7507629394531,
      "loss": 0.5584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.432280957698822,
      "rewards/margins": 0.6137180924415588,
      "rewards/rejected": -1.0459990501403809,
      "step": 7886
    },
    {
      "epoch": 1.0320601946511818,
      "grad_norm": 6.8470611572265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25971242785453796,
      "logits/rejected": -0.2235562801361084,
      "logps/chosen": -546.0040893554688,
      "logps/rejected": -470.1057434082031,
      "loss": 0.5135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49983111023902893,
      "rewards/margins": 1.1510326862335205,
      "rewards/rejected": -1.6508638858795166,
      "step": 7887
    },
    {
      "epoch": 1.0321910525885336,
      "grad_norm": 12.78484058380127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05691905692219734,
      "logits/rejected": -0.034414779394865036,
      "logps/chosen": -702.5715942382812,
      "logps/rejected": -570.1189575195312,
      "loss": 0.7404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7495300769805908,
      "rewards/margins": 0.7366187572479248,
      "rewards/rejected": -1.4861488342285156,
      "step": 7888
    },
    {
      "epoch": 1.0323219105258854,
      "grad_norm": 9.35006046295166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008799206465482712,
      "logits/rejected": 0.004480394534766674,
      "logps/chosen": -520.1510009765625,
      "logps/rejected": -524.0341796875,
      "loss": 0.757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09223175048828125,
      "rewards/margins": 0.30629122257232666,
      "rewards/rejected": -0.2140594720840454,
      "step": 7889
    },
    {
      "epoch": 1.0324527684632372,
      "grad_norm": 6.976032733917236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25055912137031555,
      "logits/rejected": -0.10765957832336426,
      "logps/chosen": -363.5113525390625,
      "logps/rejected": -432.4301452636719,
      "loss": 0.564,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22338151931762695,
      "rewards/margins": 0.6357115507125854,
      "rewards/rejected": -0.4123300015926361,
      "step": 7890
    },
    {
      "epoch": 1.0325836264005888,
      "grad_norm": 8.570503234863281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20207394659519196,
      "logits/rejected": -0.22379139065742493,
      "logps/chosen": -340.99945068359375,
      "logps/rejected": -314.65863037109375,
      "loss": 0.5518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6912168264389038,
      "rewards/margins": 0.868969202041626,
      "rewards/rejected": -0.17775237560272217,
      "step": 7891
    },
    {
      "epoch": 1.0327144843379406,
      "grad_norm": 11.090249061584473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0318564809858799,
      "logits/rejected": -0.2462468296289444,
      "logps/chosen": -454.0880126953125,
      "logps/rejected": -471.09332275390625,
      "loss": 0.6402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44070398807525635,
      "rewards/margins": 1.0238691568374634,
      "rewards/rejected": -1.4645731449127197,
      "step": 7892
    },
    {
      "epoch": 1.0328453422752923,
      "grad_norm": 4.000212669372559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29618799686431885,
      "logits/rejected": -0.18749098479747772,
      "logps/chosen": -457.6081237792969,
      "logps/rejected": -383.56524658203125,
      "loss": 0.2627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2897825241088867,
      "rewards/margins": 1.5414639711380005,
      "rewards/rejected": -1.8312464952468872,
      "step": 7893
    },
    {
      "epoch": 1.0329762002126441,
      "grad_norm": 8.44123649597168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05114823952317238,
      "logits/rejected": -0.10792718082666397,
      "logps/chosen": -409.11883544921875,
      "logps/rejected": -440.6330871582031,
      "loss": 0.539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6657372713088989,
      "rewards/margins": 0.6739690899848938,
      "rewards/rejected": -1.3397064208984375,
      "step": 7894
    },
    {
      "epoch": 1.033107058149996,
      "grad_norm": 11.696246147155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05576816201210022,
      "logits/rejected": 0.039666272699832916,
      "logps/chosen": -509.18280029296875,
      "logps/rejected": -553.9899291992188,
      "loss": 0.4859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6291160583496094,
      "rewards/margins": 1.1850546598434448,
      "rewards/rejected": -1.8141707181930542,
      "step": 7895
    },
    {
      "epoch": 1.0332379160873477,
      "grad_norm": 6.160502910614014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014394626021385193,
      "logits/rejected": -0.011797339655458927,
      "logps/chosen": -507.1396179199219,
      "logps/rejected": -499.9861755371094,
      "loss": 0.388,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2858024835586548,
      "rewards/margins": 1.2071523666381836,
      "rewards/rejected": -0.9213499426841736,
      "step": 7896
    },
    {
      "epoch": 1.0333687740246995,
      "grad_norm": 6.922634124755859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3778870105743408,
      "logits/rejected": -0.2641371786594391,
      "logps/chosen": -537.1021728515625,
      "logps/rejected": -741.2862548828125,
      "loss": 0.6235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3649759590625763,
      "rewards/margins": 1.149329662322998,
      "rewards/rejected": -1.514305591583252,
      "step": 7897
    },
    {
      "epoch": 1.0334996319620513,
      "grad_norm": 7.021181583404541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019624684005975723,
      "logits/rejected": -0.0689358189702034,
      "logps/chosen": -525.1424560546875,
      "logps/rejected": -495.13079833984375,
      "loss": 0.6035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22639250755310059,
      "rewards/margins": 0.9906340837478638,
      "rewards/rejected": -1.217026710510254,
      "step": 7898
    },
    {
      "epoch": 1.0336304898994029,
      "grad_norm": 10.707391738891602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2289610654115677,
      "logits/rejected": -0.19278249144554138,
      "logps/chosen": -406.84375,
      "logps/rejected": -378.7105712890625,
      "loss": 0.6831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8053438663482666,
      "rewards/margins": 0.4470134973526001,
      "rewards/rejected": -1.2523572444915771,
      "step": 7899
    },
    {
      "epoch": 1.0337613478367547,
      "grad_norm": 7.591490268707275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11911886930465698,
      "logits/rejected": -0.027907976880669594,
      "logps/chosen": -350.0244140625,
      "logps/rejected": -309.4297180175781,
      "loss": 0.5862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3445141911506653,
      "rewards/margins": 0.5264325737953186,
      "rewards/rejected": -0.8709466457366943,
      "step": 7900
    },
    {
      "epoch": 1.0338922057741065,
      "grad_norm": 6.229912757873535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027269013226032257,
      "logits/rejected": -0.11856415867805481,
      "logps/chosen": -347.5518798828125,
      "logps/rejected": -346.325927734375,
      "loss": 0.4385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.467184841632843,
      "rewards/margins": 1.5983448028564453,
      "rewards/rejected": -2.0655295848846436,
      "step": 7901
    },
    {
      "epoch": 1.0340230637114582,
      "grad_norm": 7.701845169067383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18474383652210236,
      "logits/rejected": -0.18289552628993988,
      "logps/chosen": -569.3375244140625,
      "logps/rejected": -543.256103515625,
      "loss": 0.4902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1892297863960266,
      "rewards/margins": 1.342872977256775,
      "rewards/rejected": -1.1536431312561035,
      "step": 7902
    },
    {
      "epoch": 1.03415392164881,
      "grad_norm": 7.399711608886719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09698213636875153,
      "logits/rejected": -0.511782169342041,
      "logps/chosen": -338.51263427734375,
      "logps/rejected": -374.6874084472656,
      "loss": 0.5092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6826149821281433,
      "rewards/margins": 2.255671739578247,
      "rewards/rejected": -2.938286781311035,
      "step": 7903
    },
    {
      "epoch": 1.0342847795861618,
      "grad_norm": 6.221460819244385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07646121084690094,
      "logits/rejected": 0.06921997666358948,
      "logps/chosen": -385.87664794921875,
      "logps/rejected": -485.0018005371094,
      "loss": 0.3642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2762705087661743,
      "rewards/margins": 1.3701295852661133,
      "rewards/rejected": -1.6464000940322876,
      "step": 7904
    },
    {
      "epoch": 1.0344156375235136,
      "grad_norm": 7.273050785064697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11368868499994278,
      "logits/rejected": 0.14280544221401215,
      "logps/chosen": -379.4486999511719,
      "logps/rejected": -441.19732666015625,
      "loss": 0.4976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0044120848178863525,
      "rewards/margins": 0.8786449432373047,
      "rewards/rejected": -0.8742328882217407,
      "step": 7905
    },
    {
      "epoch": 1.0345464954608654,
      "grad_norm": 5.193214416503906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022187963128089905,
      "logits/rejected": 0.0997767299413681,
      "logps/chosen": -363.42608642578125,
      "logps/rejected": -288.64605712890625,
      "loss": 0.3668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3438936471939087,
      "rewards/margins": 1.2526073455810547,
      "rewards/rejected": -1.5965009927749634,
      "step": 7906
    },
    {
      "epoch": 1.034677353398217,
      "grad_norm": 10.30942153930664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10556665062904358,
      "logits/rejected": -0.024511363357305527,
      "logps/chosen": -456.24029541015625,
      "logps/rejected": -439.98583984375,
      "loss": 0.6443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03241714835166931,
      "rewards/margins": 0.6311088800430298,
      "rewards/rejected": -0.6635260581970215,
      "step": 7907
    },
    {
      "epoch": 1.0348082113355688,
      "grad_norm": 10.67182445526123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05523926019668579,
      "logits/rejected": 0.06809383630752563,
      "logps/chosen": -563.3740844726562,
      "logps/rejected": -542.763427734375,
      "loss": 0.7439,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.36204642057418823,
      "rewards/margins": 0.27911511063575745,
      "rewards/rejected": 0.08293130993843079,
      "step": 7908
    },
    {
      "epoch": 1.0349390692729206,
      "grad_norm": 8.36074447631836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05396509915590286,
      "logits/rejected": 0.013757050037384033,
      "logps/chosen": -410.2410583496094,
      "logps/rejected": -393.28497314453125,
      "loss": 0.5718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8147453665733337,
      "rewards/margins": 0.8129168748855591,
      "rewards/rejected": -1.6276623010635376,
      "step": 7909
    },
    {
      "epoch": 1.0350699272102724,
      "grad_norm": 10.131613731384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08453474938869476,
      "logits/rejected": -0.3503333330154419,
      "logps/chosen": -546.6846923828125,
      "logps/rejected": -440.5220642089844,
      "loss": 0.7632,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.65508633852005,
      "rewards/margins": 0.20577988028526306,
      "rewards/rejected": -0.8608661890029907,
      "step": 7910
    },
    {
      "epoch": 1.0352007851476241,
      "grad_norm": 9.870387077331543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2555999755859375,
      "logits/rejected": 0.08380082249641418,
      "logps/chosen": -422.7195739746094,
      "logps/rejected": -607.0372924804688,
      "loss": 0.8328,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.8581249117851257,
      "rewards/margins": -0.09771688282489777,
      "rewards/rejected": -0.7604081034660339,
      "step": 7911
    },
    {
      "epoch": 1.035331643084976,
      "grad_norm": 7.037151336669922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5002983808517456,
      "logits/rejected": -0.39712947607040405,
      "logps/chosen": -467.33734130859375,
      "logps/rejected": -352.4491882324219,
      "loss": 0.471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49340495467185974,
      "rewards/margins": 0.6247313022613525,
      "rewards/rejected": -1.1181362867355347,
      "step": 7912
    },
    {
      "epoch": 1.0354625010223277,
      "grad_norm": 6.3586859703063965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11088106036186218,
      "logits/rejected": 0.297182559967041,
      "logps/chosen": -532.1725463867188,
      "logps/rejected": -391.992431640625,
      "loss": 0.485,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7405414581298828,
      "rewards/margins": 0.7082108855247498,
      "rewards/rejected": -1.4487524032592773,
      "step": 7913
    },
    {
      "epoch": 1.0355933589596793,
      "grad_norm": 9.106040000915527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2953125536441803,
      "logits/rejected": 0.1079474464058876,
      "logps/chosen": -388.7719421386719,
      "logps/rejected": -523.5050048828125,
      "loss": 0.6736,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2774363160133362,
      "rewards/margins": 1.766674280166626,
      "rewards/rejected": -2.0441105365753174,
      "step": 7914
    },
    {
      "epoch": 1.035724216897031,
      "grad_norm": 4.589694976806641,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01480066031217575,
      "logits/rejected": -0.14326980710029602,
      "logps/chosen": -490.59857177734375,
      "logps/rejected": -331.17486572265625,
      "loss": 0.2236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1800115704536438,
      "rewards/margins": 1.9154608249664307,
      "rewards/rejected": -1.7354493141174316,
      "step": 7915
    },
    {
      "epoch": 1.0358550748343829,
      "grad_norm": 4.5850629806518555,
      "learning_rate": 5e-06,
      "logits/chosen": 6.274878978729248e-05,
      "logits/rejected": 0.1117139384150505,
      "logps/chosen": -451.05279541015625,
      "logps/rejected": -428.783935546875,
      "loss": 0.4169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3887006938457489,
      "rewards/margins": 1.1523728370666504,
      "rewards/rejected": -0.7636722326278687,
      "step": 7916
    },
    {
      "epoch": 1.0359859327717347,
      "grad_norm": 7.239260196685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0798703134059906,
      "logits/rejected": -0.08850237727165222,
      "logps/chosen": -689.790283203125,
      "logps/rejected": -635.103759765625,
      "loss": 0.5518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4892190992832184,
      "rewards/margins": 0.8297631740570068,
      "rewards/rejected": -0.34054407477378845,
      "step": 7917
    },
    {
      "epoch": 1.0361167907090865,
      "grad_norm": 4.757336616516113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18756474554538727,
      "logits/rejected": -0.2911423444747925,
      "logps/chosen": -470.3343811035156,
      "logps/rejected": -508.52398681640625,
      "loss": 0.2998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3851892352104187,
      "rewards/margins": 1.2598412036895752,
      "rewards/rejected": -0.8746519088745117,
      "step": 7918
    },
    {
      "epoch": 1.0362476486464383,
      "grad_norm": 6.069225788116455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06320548057556152,
      "logits/rejected": 0.064669169485569,
      "logps/chosen": -405.08642578125,
      "logps/rejected": -472.0407409667969,
      "loss": 0.346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.011824220418930054,
      "rewards/margins": 1.1447066068649292,
      "rewards/rejected": -1.156530737876892,
      "step": 7919
    },
    {
      "epoch": 1.03637850658379,
      "grad_norm": 8.061403274536133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024065641686320305,
      "logits/rejected": 0.0720604658126831,
      "logps/chosen": -455.74951171875,
      "logps/rejected": -319.39508056640625,
      "loss": 0.4629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01823580265045166,
      "rewards/margins": 1.3128751516342163,
      "rewards/rejected": -1.331110954284668,
      "step": 7920
    },
    {
      "epoch": 1.0365093645211418,
      "grad_norm": 6.609951496124268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24951313436031342,
      "logits/rejected": -0.38120946288108826,
      "logps/chosen": -444.76092529296875,
      "logps/rejected": -441.8636779785156,
      "loss": 0.4395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32302725315093994,
      "rewards/margins": 1.464058518409729,
      "rewards/rejected": -1.787085771560669,
      "step": 7921
    },
    {
      "epoch": 1.0366402224584934,
      "grad_norm": 8.096649169921875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055762819945812225,
      "logits/rejected": 0.193794846534729,
      "logps/chosen": -522.8909912109375,
      "logps/rejected": -548.58837890625,
      "loss": 0.5331,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48765790462493896,
      "rewards/margins": 1.2952529191970825,
      "rewards/rejected": -1.7829108238220215,
      "step": 7922
    },
    {
      "epoch": 1.0367710803958452,
      "grad_norm": 6.683338642120361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05462343618273735,
      "logits/rejected": -0.1511654257774353,
      "logps/chosen": -640.0381469726562,
      "logps/rejected": -449.3978576660156,
      "loss": 0.38,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8452816009521484,
      "rewards/margins": 1.6644283533096313,
      "rewards/rejected": -0.8191468119621277,
      "step": 7923
    },
    {
      "epoch": 1.036901938333197,
      "grad_norm": 6.676602840423584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1313440203666687,
      "logits/rejected": -0.19036747515201569,
      "logps/chosen": -583.4942626953125,
      "logps/rejected": -559.4942016601562,
      "loss": 0.3979,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07252772152423859,
      "rewards/margins": 1.6142157316207886,
      "rewards/rejected": -1.5416879653930664,
      "step": 7924
    },
    {
      "epoch": 1.0370327962705488,
      "grad_norm": 6.481491565704346,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004048310220241547,
      "logits/rejected": -0.160796657204628,
      "logps/chosen": -440.52813720703125,
      "logps/rejected": -356.7110595703125,
      "loss": 0.2927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9060672521591187,
      "rewards/margins": 1.6584876775741577,
      "rewards/rejected": -0.7524204254150391,
      "step": 7925
    },
    {
      "epoch": 1.0371636542079006,
      "grad_norm": 5.690451622009277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12960228323936462,
      "logits/rejected": -0.11547647416591644,
      "logps/chosen": -412.225341796875,
      "logps/rejected": -363.1166076660156,
      "loss": 0.3681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7951382398605347,
      "rewards/margins": 1.031693696975708,
      "rewards/rejected": -0.23655551671981812,
      "step": 7926
    },
    {
      "epoch": 1.0372945121452524,
      "grad_norm": 6.365859031677246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04922175407409668,
      "logits/rejected": -0.0949479341506958,
      "logps/chosen": -554.45703125,
      "logps/rejected": -447.6082763671875,
      "loss": 0.3921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1825830638408661,
      "rewards/margins": 0.9945726990699768,
      "rewards/rejected": -1.1771557331085205,
      "step": 7927
    },
    {
      "epoch": 1.0374253700826042,
      "grad_norm": 3.5616068840026855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05948754400014877,
      "logits/rejected": -0.2128557562828064,
      "logps/chosen": -357.20416259765625,
      "logps/rejected": -460.83782958984375,
      "loss": 0.2142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34803780913352966,
      "rewards/margins": 2.698711395263672,
      "rewards/rejected": -2.3506734371185303,
      "step": 7928
    },
    {
      "epoch": 1.0375562280199557,
      "grad_norm": 5.285675525665283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046855829656124115,
      "logits/rejected": 0.020690657198429108,
      "logps/chosen": -377.15350341796875,
      "logps/rejected": -430.3835144042969,
      "loss": 0.2423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17029057443141937,
      "rewards/margins": 1.597635269165039,
      "rewards/rejected": -1.4273446798324585,
      "step": 7929
    },
    {
      "epoch": 1.0376870859573075,
      "grad_norm": 6.53230094909668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.041861455887556076,
      "logits/rejected": -0.040262699127197266,
      "logps/chosen": -439.7435607910156,
      "logps/rejected": -337.6613464355469,
      "loss": 0.5112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6799230575561523,
      "rewards/margins": 0.8442150354385376,
      "rewards/rejected": -0.16429194808006287,
      "step": 7930
    },
    {
      "epoch": 1.0378179438946593,
      "grad_norm": 6.523906707763672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023730680346488953,
      "logits/rejected": -0.02164686843752861,
      "logps/chosen": -436.47442626953125,
      "logps/rejected": -464.4622802734375,
      "loss": 0.4244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2281583696603775,
      "rewards/margins": 0.9025008082389832,
      "rewards/rejected": -1.1306591033935547,
      "step": 7931
    },
    {
      "epoch": 1.037948801832011,
      "grad_norm": 6.135051727294922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004476169124245644,
      "logits/rejected": 0.12907160818576813,
      "logps/chosen": -483.2087097167969,
      "logps/rejected": -440.9042053222656,
      "loss": 0.5436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.409501850605011,
      "rewards/margins": 0.5134568810462952,
      "rewards/rejected": -0.10395503789186478,
      "step": 7932
    },
    {
      "epoch": 1.038079659769363,
      "grad_norm": 6.498019218444824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16337159276008606,
      "logits/rejected": -0.08751194924116135,
      "logps/chosen": -289.234375,
      "logps/rejected": -359.8659973144531,
      "loss": 0.3621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8699568510055542,
      "rewards/margins": 1.254913330078125,
      "rewards/rejected": -2.1248703002929688,
      "step": 7933
    },
    {
      "epoch": 1.0382105177067147,
      "grad_norm": 5.2517619132995605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04374074190855026,
      "logits/rejected": -0.13591670989990234,
      "logps/chosen": -445.45513916015625,
      "logps/rejected": -582.8101806640625,
      "loss": 0.2978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.384281724691391,
      "rewards/margins": 1.8331258296966553,
      "rewards/rejected": -2.217407703399658,
      "step": 7934
    },
    {
      "epoch": 1.0383413756440665,
      "grad_norm": 8.166497230529785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19567976891994476,
      "logits/rejected": -0.21346859633922577,
      "logps/chosen": -543.8582763671875,
      "logps/rejected": -596.0947265625,
      "loss": 0.4968,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10564950108528137,
      "rewards/margins": 1.2035189867019653,
      "rewards/rejected": -1.3091686964035034,
      "step": 7935
    },
    {
      "epoch": 1.0384722335814183,
      "grad_norm": 8.469878196716309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21412497758865356,
      "logits/rejected": -0.13735757768154144,
      "logps/chosen": -668.8536376953125,
      "logps/rejected": -538.2701416015625,
      "loss": 0.5403,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26629066467285156,
      "rewards/margins": 0.9236114025115967,
      "rewards/rejected": -0.6573207974433899,
      "step": 7936
    },
    {
      "epoch": 1.0386030915187698,
      "grad_norm": 8.423110961914062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20529912412166595,
      "logits/rejected": 0.06643183529376984,
      "logps/chosen": -680.6204833984375,
      "logps/rejected": -798.4248657226562,
      "loss": 0.5931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.29398787021636963,
      "rewards/margins": 0.5523473024368286,
      "rewards/rejected": -0.25835955142974854,
      "step": 7937
    },
    {
      "epoch": 1.0387339494561216,
      "grad_norm": 5.896403789520264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04393336549401283,
      "logits/rejected": -0.062171049416065216,
      "logps/chosen": -346.77001953125,
      "logps/rejected": -469.2447204589844,
      "loss": 0.2899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3303411602973938,
      "rewards/margins": 1.7809698581695557,
      "rewards/rejected": -2.1113109588623047,
      "step": 7938
    },
    {
      "epoch": 1.0388648073934734,
      "grad_norm": 2.3527815341949463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2252543866634369,
      "logits/rejected": -0.2630404531955719,
      "logps/chosen": -583.671875,
      "logps/rejected": -562.5026245117188,
      "loss": 0.0968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7701424360275269,
      "rewards/margins": 3.121569871902466,
      "rewards/rejected": -2.3514275550842285,
      "step": 7939
    },
    {
      "epoch": 1.0389956653308252,
      "grad_norm": 6.0355143547058105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10481321066617966,
      "logits/rejected": 0.10701604187488556,
      "logps/chosen": -555.6390380859375,
      "logps/rejected": -494.0146484375,
      "loss": 0.3152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.45486241579055786,
      "rewards/margins": 2.0225095748901367,
      "rewards/rejected": -1.5676473379135132,
      "step": 7940
    },
    {
      "epoch": 1.039126523268177,
      "grad_norm": 6.6148552894592285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15080460906028748,
      "logits/rejected": -0.011628832668066025,
      "logps/chosen": -387.92950439453125,
      "logps/rejected": -538.9156494140625,
      "loss": 0.4169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16678543388843536,
      "rewards/margins": 0.8069874048233032,
      "rewards/rejected": -0.6402019262313843,
      "step": 7941
    },
    {
      "epoch": 1.0392573812055288,
      "grad_norm": 5.334842681884766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.32107651233673096,
      "logits/rejected": 0.0999557226896286,
      "logps/chosen": -355.6733093261719,
      "logps/rejected": -341.8736877441406,
      "loss": 0.3174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36685964465141296,
      "rewards/margins": 1.264275074005127,
      "rewards/rejected": -1.6311346292495728,
      "step": 7942
    },
    {
      "epoch": 1.0393882391428806,
      "grad_norm": 8.182062149047852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17267754673957825,
      "logits/rejected": -0.18771177530288696,
      "logps/chosen": -336.148193359375,
      "logps/rejected": -390.67041015625,
      "loss": 0.5401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9469480514526367,
      "rewards/margins": 0.660138726234436,
      "rewards/rejected": -1.6070868968963623,
      "step": 7943
    },
    {
      "epoch": 1.0395190970802324,
      "grad_norm": 6.1895599365234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24700650572776794,
      "logits/rejected": -0.34512701630592346,
      "logps/chosen": -408.0561828613281,
      "logps/rejected": -388.22113037109375,
      "loss": 0.2916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09568672627210617,
      "rewards/margins": 1.3491508960723877,
      "rewards/rejected": -1.2534642219543457,
      "step": 7944
    },
    {
      "epoch": 1.039649955017584,
      "grad_norm": 8.919849395751953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21959209442138672,
      "logits/rejected": -0.02597765624523163,
      "logps/chosen": -452.2056884765625,
      "logps/rejected": -322.37548828125,
      "loss": 0.7001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7744721174240112,
      "rewards/margins": 1.2310950756072998,
      "rewards/rejected": -0.45662301778793335,
      "step": 7945
    },
    {
      "epoch": 1.0397808129549357,
      "grad_norm": 7.259485244750977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25406721234321594,
      "logits/rejected": -0.2979917526245117,
      "logps/chosen": -440.3924865722656,
      "logps/rejected": -494.35198974609375,
      "loss": 0.3718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22269251942634583,
      "rewards/margins": 2.303619623184204,
      "rewards/rejected": -2.0809268951416016,
      "step": 7946
    },
    {
      "epoch": 1.0399116708922875,
      "grad_norm": 7.693587779998779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011209545657038689,
      "logits/rejected": -0.14939281344413757,
      "logps/chosen": -299.0733947753906,
      "logps/rejected": -319.07232666015625,
      "loss": 0.5832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4379047751426697,
      "rewards/margins": 0.36499154567718506,
      "rewards/rejected": -0.80289626121521,
      "step": 7947
    },
    {
      "epoch": 1.0400425288296393,
      "grad_norm": 4.355897426605225,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08021244406700134,
      "logits/rejected": 0.0824681967496872,
      "logps/chosen": -434.9866027832031,
      "logps/rejected": -473.7957458496094,
      "loss": 0.2967,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1550661325454712,
      "rewards/margins": 1.3598178625106812,
      "rewards/rejected": -1.2047518491744995,
      "step": 7948
    },
    {
      "epoch": 1.0401733867669911,
      "grad_norm": 4.636840343475342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08464496582746506,
      "logits/rejected": -0.009449228644371033,
      "logps/chosen": -454.85601806640625,
      "logps/rejected": -536.225830078125,
      "loss": 0.2121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5165534019470215,
      "rewards/margins": 1.9207175970077515,
      "rewards/rejected": -1.4041640758514404,
      "step": 7949
    },
    {
      "epoch": 1.040304244704343,
      "grad_norm": 8.99256420135498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20752400159835815,
      "logits/rejected": 0.010618306696414948,
      "logps/chosen": -482.9111328125,
      "logps/rejected": -397.468505859375,
      "loss": 0.4911,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.037202268838882446,
      "rewards/margins": 0.9901352524757385,
      "rewards/rejected": -1.0273374319076538,
      "step": 7950
    },
    {
      "epoch": 1.0404351026416947,
      "grad_norm": 6.41642951965332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032972775399684906,
      "logits/rejected": -0.1055038794875145,
      "logps/chosen": -399.03564453125,
      "logps/rejected": -450.715087890625,
      "loss": 0.2976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6528388857841492,
      "rewards/margins": 1.4480164051055908,
      "rewards/rejected": -2.1008553504943848,
      "step": 7951
    },
    {
      "epoch": 1.0405659605790463,
      "grad_norm": 7.4618024826049805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30687397718429565,
      "logits/rejected": 0.1517457664012909,
      "logps/chosen": -523.680419921875,
      "logps/rejected": -412.1169128417969,
      "loss": 0.5257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8169730305671692,
      "rewards/margins": 1.683670997619629,
      "rewards/rejected": -2.5006439685821533,
      "step": 7952
    },
    {
      "epoch": 1.040696818516398,
      "grad_norm": 7.056053161621094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0959860235452652,
      "logits/rejected": -0.24137482047080994,
      "logps/chosen": -452.0830383300781,
      "logps/rejected": -442.8210144042969,
      "loss": 0.4336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6939584612846375,
      "rewards/margins": 1.0601580142974854,
      "rewards/rejected": -1.7541165351867676,
      "step": 7953
    },
    {
      "epoch": 1.0408276764537499,
      "grad_norm": 7.000873565673828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08938349783420563,
      "logits/rejected": -0.09241656213998795,
      "logps/chosen": -537.1624145507812,
      "logps/rejected": -423.7925109863281,
      "loss": 0.3809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4838840365409851,
      "rewards/margins": 1.1690131425857544,
      "rewards/rejected": -0.6851291656494141,
      "step": 7954
    },
    {
      "epoch": 1.0409585343911016,
      "grad_norm": 6.658606052398682,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3845794200897217,
      "logits/rejected": -0.11463884264230728,
      "logps/chosen": -295.40887451171875,
      "logps/rejected": -263.8376770019531,
      "loss": 0.5004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11236956715583801,
      "rewards/margins": 0.5570627450942993,
      "rewards/rejected": -0.6694322824478149,
      "step": 7955
    },
    {
      "epoch": 1.0410893923284534,
      "grad_norm": 12.125550270080566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11513431370258331,
      "logits/rejected": 0.1011136919260025,
      "logps/chosen": -834.7406005859375,
      "logps/rejected": -706.466552734375,
      "loss": 0.7593,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01919744908809662,
      "rewards/margins": 0.44435402750968933,
      "rewards/rejected": -0.4251565933227539,
      "step": 7956
    },
    {
      "epoch": 1.0412202502658052,
      "grad_norm": 4.8275322914123535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0802687481045723,
      "logits/rejected": 0.05074325203895569,
      "logps/chosen": -309.9662780761719,
      "logps/rejected": -380.266845703125,
      "loss": 0.2254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4710756242275238,
      "rewards/margins": 1.626063346862793,
      "rewards/rejected": -1.1549876928329468,
      "step": 7957
    },
    {
      "epoch": 1.041351108203157,
      "grad_norm": 4.949923992156982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11674071848392487,
      "logits/rejected": -0.3013867437839508,
      "logps/chosen": -639.1044311523438,
      "logps/rejected": -450.81402587890625,
      "loss": 0.1956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36670321226119995,
      "rewards/margins": 1.9913560152053833,
      "rewards/rejected": -2.3580594062805176,
      "step": 7958
    },
    {
      "epoch": 1.0414819661405088,
      "grad_norm": 5.265179634094238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0018488839268684387,
      "logits/rejected": 0.1205117404460907,
      "logps/chosen": -347.5732421875,
      "logps/rejected": -594.5750732421875,
      "loss": 0.2871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8105834126472473,
      "rewards/margins": 1.3265185356140137,
      "rewards/rejected": -2.137101888656616,
      "step": 7959
    },
    {
      "epoch": 1.0416128240778604,
      "grad_norm": 8.067571640014648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.35338008403778076,
      "logits/rejected": 0.04359262436628342,
      "logps/chosen": -665.4210205078125,
      "logps/rejected": -488.59686279296875,
      "loss": 0.5913,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.194405198097229,
      "rewards/margins": 0.31421396136283875,
      "rewards/rejected": -0.5086191296577454,
      "step": 7960
    },
    {
      "epoch": 1.0417436820152122,
      "grad_norm": 4.6241135597229,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05279404670000076,
      "logits/rejected": -0.1331094652414322,
      "logps/chosen": -375.8014831542969,
      "logps/rejected": -297.4322814941406,
      "loss": 0.2298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4399889409542084,
      "rewards/margins": 1.987302541732788,
      "rewards/rejected": -1.5473136901855469,
      "step": 7961
    },
    {
      "epoch": 1.041874539952564,
      "grad_norm": 5.596261978149414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1423935890197754,
      "logits/rejected": -0.21764227747917175,
      "logps/chosen": -487.998779296875,
      "logps/rejected": -334.270263671875,
      "loss": 0.4148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.130116268992424,
      "rewards/margins": 1.059498906135559,
      "rewards/rejected": -0.9293824434280396,
      "step": 7962
    },
    {
      "epoch": 1.0420053978899158,
      "grad_norm": 12.476645469665527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04779674485325813,
      "logits/rejected": -0.061873797327280045,
      "logps/chosen": -490.0843200683594,
      "logps/rejected": -416.9637145996094,
      "loss": 0.6429,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5155776739120483,
      "rewards/margins": 0.42375510931015015,
      "rewards/rejected": -1.9393327236175537,
      "step": 7963
    },
    {
      "epoch": 1.0421362558272675,
      "grad_norm": 6.797678470611572,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13168758153915405,
      "logits/rejected": 0.21556204557418823,
      "logps/chosen": -482.83331298828125,
      "logps/rejected": -538.05908203125,
      "loss": 0.368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6048423647880554,
      "rewards/margins": 1.3796756267547607,
      "rewards/rejected": -1.984518051147461,
      "step": 7964
    },
    {
      "epoch": 1.0422671137646193,
      "grad_norm": 9.821094512939453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0032518021762371063,
      "logits/rejected": -0.025706473737955093,
      "logps/chosen": -530.8475341796875,
      "logps/rejected": -385.73175048828125,
      "loss": 0.544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20474550127983093,
      "rewards/margins": 0.8262894153594971,
      "rewards/rejected": -1.0310349464416504,
      "step": 7965
    },
    {
      "epoch": 1.0423979717019711,
      "grad_norm": 6.478853702545166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18537867069244385,
      "logits/rejected": -0.3172965347766876,
      "logps/chosen": -326.28057861328125,
      "logps/rejected": -273.57196044921875,
      "loss": 0.3718,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37455978989601135,
      "rewards/margins": 1.2804532051086426,
      "rewards/rejected": -1.6550129652023315,
      "step": 7966
    },
    {
      "epoch": 1.042528829639323,
      "grad_norm": 10.42634391784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12872816622257233,
      "logits/rejected": -0.028205372393131256,
      "logps/chosen": -346.94390869140625,
      "logps/rejected": -399.41290283203125,
      "loss": 1.1016,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1540759801864624,
      "rewards/margins": -0.33519986271858215,
      "rewards/rejected": -0.8188762068748474,
      "step": 7967
    },
    {
      "epoch": 1.0426596875766745,
      "grad_norm": 10.510869026184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005284026265144348,
      "logits/rejected": 0.1794382631778717,
      "logps/chosen": -412.11932373046875,
      "logps/rejected": -559.4111328125,
      "loss": 0.842,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7395854592323303,
      "rewards/margins": -0.09863339364528656,
      "rewards/rejected": -0.6409521102905273,
      "step": 7968
    },
    {
      "epoch": 1.0427905455140263,
      "grad_norm": 6.306065082550049,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18228165805339813,
      "logits/rejected": -0.09046140313148499,
      "logps/chosen": -300.5936584472656,
      "logps/rejected": -323.16015625,
      "loss": 0.4418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29367855191230774,
      "rewards/margins": 0.8756818175315857,
      "rewards/rejected": -1.1693602800369263,
      "step": 7969
    },
    {
      "epoch": 1.042921403451378,
      "grad_norm": 7.344356060028076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1529475897550583,
      "logits/rejected": -0.21966737508773804,
      "logps/chosen": -344.04107666015625,
      "logps/rejected": -396.5771179199219,
      "loss": 0.5052,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -9.745359420776367e-05,
      "rewards/margins": 1.0284626483917236,
      "rewards/rejected": -1.0285600423812866,
      "step": 7970
    },
    {
      "epoch": 1.0430522613887299,
      "grad_norm": 3.385887384414673,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12046122550964355,
      "logits/rejected": -0.2551730275154114,
      "logps/chosen": -274.0810546875,
      "logps/rejected": -297.67498779296875,
      "loss": 0.168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27613943815231323,
      "rewards/margins": 1.9077084064483643,
      "rewards/rejected": -2.1838479042053223,
      "step": 7971
    },
    {
      "epoch": 1.0431831193260817,
      "grad_norm": 10.370892524719238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05340184271335602,
      "logits/rejected": 0.13701462745666504,
      "logps/chosen": -516.9555053710938,
      "logps/rejected": -401.7372741699219,
      "loss": 0.623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7346299886703491,
      "rewards/margins": 0.551098644733429,
      "rewards/rejected": -1.2857286930084229,
      "step": 7972
    },
    {
      "epoch": 1.0433139772634334,
      "grad_norm": 9.080703735351562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08026903867721558,
      "logits/rejected": 0.27498483657836914,
      "logps/chosen": -456.7114562988281,
      "logps/rejected": -424.9232482910156,
      "loss": 0.4889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22206705808639526,
      "rewards/margins": 0.9853861331939697,
      "rewards/rejected": -1.2074531316757202,
      "step": 7973
    },
    {
      "epoch": 1.0434448352007852,
      "grad_norm": 8.27214241027832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05633695796132088,
      "logits/rejected": -0.21773405373096466,
      "logps/chosen": -382.1473083496094,
      "logps/rejected": -336.6502990722656,
      "loss": 0.5148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1259300857782364,
      "rewards/margins": 0.7800309658050537,
      "rewards/rejected": -0.9059610366821289,
      "step": 7974
    },
    {
      "epoch": 1.0435756931381368,
      "grad_norm": 9.549290657043457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04676403850317001,
      "logits/rejected": 0.20915503799915314,
      "logps/chosen": -390.91583251953125,
      "logps/rejected": -364.08953857421875,
      "loss": 0.6597,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6654351949691772,
      "rewards/margins": 0.7351863384246826,
      "rewards/rejected": -1.4006216526031494,
      "step": 7975
    },
    {
      "epoch": 1.0437065510754886,
      "grad_norm": 9.594660758972168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07442561537027359,
      "logits/rejected": -0.1370791792869568,
      "logps/chosen": -542.6077880859375,
      "logps/rejected": -396.116455078125,
      "loss": 0.7064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1864301562309265,
      "rewards/margins": 0.7654274106025696,
      "rewards/rejected": -0.9518575072288513,
      "step": 7976
    },
    {
      "epoch": 1.0438374090128404,
      "grad_norm": 8.061504364013672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06678243726491928,
      "logits/rejected": 0.18818947672843933,
      "logps/chosen": -367.3321838378906,
      "logps/rejected": -558.3323364257812,
      "loss": 0.6093,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4379124045372009,
      "rewards/margins": 0.6760231852531433,
      "rewards/rejected": -1.1139353513717651,
      "step": 7977
    },
    {
      "epoch": 1.0439682669501922,
      "grad_norm": 11.247323989868164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2106773555278778,
      "logits/rejected": 0.015813440084457397,
      "logps/chosen": -404.7952575683594,
      "logps/rejected": -456.06005859375,
      "loss": 0.5464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.28264319896698,
      "rewards/margins": 1.408066987991333,
      "rewards/rejected": -2.6907100677490234,
      "step": 7978
    },
    {
      "epoch": 1.044099124887544,
      "grad_norm": 7.5744500160217285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023476343601942062,
      "logits/rejected": 0.08834455907344818,
      "logps/chosen": -451.0331726074219,
      "logps/rejected": -516.18603515625,
      "loss": 0.4362,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6723639965057373,
      "rewards/margins": 1.1514915227890015,
      "rewards/rejected": -1.8238555192947388,
      "step": 7979
    },
    {
      "epoch": 1.0442299828248958,
      "grad_norm": 6.134620189666748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16688048839569092,
      "logits/rejected": -0.07286525517702103,
      "logps/chosen": -379.6995544433594,
      "logps/rejected": -365.43072509765625,
      "loss": 0.3771,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.633470892906189,
      "rewards/margins": 0.9810585975646973,
      "rewards/rejected": -1.6145294904708862,
      "step": 7980
    },
    {
      "epoch": 1.0443608407622476,
      "grad_norm": 5.612274646759033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24877548217773438,
      "logits/rejected": 0.11191822588443756,
      "logps/chosen": -292.82928466796875,
      "logps/rejected": -291.08099365234375,
      "loss": 0.3348,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23864687979221344,
      "rewards/margins": 1.6114907264709473,
      "rewards/rejected": -1.85013747215271,
      "step": 7981
    },
    {
      "epoch": 1.0444916986995993,
      "grad_norm": 6.356250286102295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31809908151626587,
      "logits/rejected": 0.18710491061210632,
      "logps/chosen": -411.2467041015625,
      "logps/rejected": -544.1842041015625,
      "loss": 0.3344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16954001784324646,
      "rewards/margins": 1.4091119766235352,
      "rewards/rejected": -1.2395719289779663,
      "step": 7982
    },
    {
      "epoch": 1.044622556636951,
      "grad_norm": 7.693617820739746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06904079020023346,
      "logits/rejected": 0.13900922238826752,
      "logps/chosen": -397.1325378417969,
      "logps/rejected": -344.28448486328125,
      "loss": 0.5015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9406136274337769,
      "rewards/margins": 0.5941319465637207,
      "rewards/rejected": -1.534745693206787,
      "step": 7983
    },
    {
      "epoch": 1.0447534145743027,
      "grad_norm": 4.97658634185791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15168452262878418,
      "logits/rejected": 0.01245754212141037,
      "logps/chosen": -457.3246765136719,
      "logps/rejected": -484.689208984375,
      "loss": 0.2884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08133049309253693,
      "rewards/margins": 1.374168038368225,
      "rewards/rejected": -1.2928376197814941,
      "step": 7984
    },
    {
      "epoch": 1.0448842725116545,
      "grad_norm": 8.787238121032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16409292817115784,
      "logits/rejected": -0.049453895539045334,
      "logps/chosen": -310.64337158203125,
      "logps/rejected": -383.5968322753906,
      "loss": 0.7894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4597430229187012,
      "rewards/margins": 0.5254818201065063,
      "rewards/rejected": -1.985224723815918,
      "step": 7985
    },
    {
      "epoch": 1.0450151304490063,
      "grad_norm": 8.264307022094727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0027457736432552338,
      "logits/rejected": -0.09749321639537811,
      "logps/chosen": -455.0216064453125,
      "logps/rejected": -389.52581787109375,
      "loss": 0.5129,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008259952068328857,
      "rewards/margins": 0.8473589420318604,
      "rewards/rejected": -0.8556188941001892,
      "step": 7986
    },
    {
      "epoch": 1.045145988386358,
      "grad_norm": 5.244297504425049,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07898791134357452,
      "logits/rejected": 0.09176511317491531,
      "logps/chosen": -465.8447570800781,
      "logps/rejected": -493.8106384277344,
      "loss": 0.2721,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4877590239048004,
      "rewards/margins": 1.6069047451019287,
      "rewards/rejected": -1.1191456317901611,
      "step": 7987
    },
    {
      "epoch": 1.0452768463237099,
      "grad_norm": 5.9034247398376465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18045438826084137,
      "logits/rejected": -0.08360850065946579,
      "logps/chosen": -486.8985290527344,
      "logps/rejected": -382.6060791015625,
      "loss": 0.3971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33879444003105164,
      "rewards/margins": 1.0240503549575806,
      "rewards/rejected": -0.6852558255195618,
      "step": 7988
    },
    {
      "epoch": 1.0454077042610617,
      "grad_norm": 7.49272346496582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17013734579086304,
      "logits/rejected": -0.2604270577430725,
      "logps/chosen": -391.7437744140625,
      "logps/rejected": -403.0552673339844,
      "loss": 0.5979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2680755853652954,
      "rewards/margins": 1.1567943096160889,
      "rewards/rejected": -1.4248698949813843,
      "step": 7989
    },
    {
      "epoch": 1.0455385621984132,
      "grad_norm": 4.677754878997803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019380636513233185,
      "logits/rejected": 0.20948311686515808,
      "logps/chosen": -530.3710327148438,
      "logps/rejected": -448.0840759277344,
      "loss": 0.2053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6126070618629456,
      "rewards/margins": 1.8003970384597778,
      "rewards/rejected": -1.187790036201477,
      "step": 7990
    },
    {
      "epoch": 1.045669420135765,
      "grad_norm": 7.303651809692383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01714540272951126,
      "logits/rejected": -0.04567369073629379,
      "logps/chosen": -389.30804443359375,
      "logps/rejected": -367.4333190917969,
      "loss": 0.5274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.956426739692688,
      "rewards/margins": 0.9054436683654785,
      "rewards/rejected": -1.861870527267456,
      "step": 7991
    },
    {
      "epoch": 1.0458002780731168,
      "grad_norm": 7.656610488891602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16784882545471191,
      "logits/rejected": 0.08131664246320724,
      "logps/chosen": -326.37567138671875,
      "logps/rejected": -342.7764892578125,
      "loss": 0.4697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23522454500198364,
      "rewards/margins": 0.9914548397064209,
      "rewards/rejected": -0.756230354309082,
      "step": 7992
    },
    {
      "epoch": 1.0459311360104686,
      "grad_norm": 7.18083381652832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10250365734100342,
      "logits/rejected": 0.0025370940566062927,
      "logps/chosen": -476.8001403808594,
      "logps/rejected": -476.7012939453125,
      "loss": 0.4321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3386375606060028,
      "rewards/margins": 1.562989592552185,
      "rewards/rejected": -1.9016269445419312,
      "step": 7993
    },
    {
      "epoch": 1.0460619939478204,
      "grad_norm": 5.484365940093994,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1844875067472458,
      "logits/rejected": -0.105068638920784,
      "logps/chosen": -504.21826171875,
      "logps/rejected": -372.260986328125,
      "loss": 0.3383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09586639702320099,
      "rewards/margins": 1.331843614578247,
      "rewards/rejected": -1.2359771728515625,
      "step": 7994
    },
    {
      "epoch": 1.0461928518851722,
      "grad_norm": 6.8321123123168945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13753333687782288,
      "logits/rejected": 0.04112674295902252,
      "logps/chosen": -411.3602600097656,
      "logps/rejected": -519.0303344726562,
      "loss": 0.5559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43110087513923645,
      "rewards/margins": 0.7423895001411438,
      "rewards/rejected": -1.173490285873413,
      "step": 7995
    },
    {
      "epoch": 1.046323709822524,
      "grad_norm": 5.603799343109131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20739732682704926,
      "logits/rejected": 0.10492213070392609,
      "logps/chosen": -430.03717041015625,
      "logps/rejected": -385.3020324707031,
      "loss": 0.342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5201549530029297,
      "rewards/margins": 1.4176807403564453,
      "rewards/rejected": -0.8975257873535156,
      "step": 7996
    },
    {
      "epoch": 1.0464545677598758,
      "grad_norm": 4.835295677185059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040108174085617065,
      "logits/rejected": 0.07299032062292099,
      "logps/chosen": -607.5363159179688,
      "logps/rejected": -491.75494384765625,
      "loss": 0.2201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6993112564086914,
      "rewards/margins": 1.7273398637771606,
      "rewards/rejected": -2.4266514778137207,
      "step": 7997
    },
    {
      "epoch": 1.0465854256972273,
      "grad_norm": 7.035870552062988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10722804069519043,
      "logits/rejected": 0.2539105713367462,
      "logps/chosen": -346.3503723144531,
      "logps/rejected": -351.569580078125,
      "loss": 0.4594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09142113476991653,
      "rewards/margins": 1.3199529647827148,
      "rewards/rejected": -1.2285317182540894,
      "step": 7998
    },
    {
      "epoch": 1.0467162836345791,
      "grad_norm": 4.893979549407959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12966959178447723,
      "logits/rejected": 0.019289489835500717,
      "logps/chosen": -585.0047607421875,
      "logps/rejected": -360.196044921875,
      "loss": 0.2415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1176450252532959,
      "rewards/margins": 1.754280686378479,
      "rewards/rejected": -1.636635661125183,
      "step": 7999
    },
    {
      "epoch": 1.046847141571931,
      "grad_norm": 4.186567783355713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11000367254018784,
      "logits/rejected": -0.12241347134113312,
      "logps/chosen": -515.11376953125,
      "logps/rejected": -427.2802734375,
      "loss": 0.2387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.659125804901123,
      "rewards/margins": 1.612577199935913,
      "rewards/rejected": -0.9534511566162109,
      "step": 8000
    },
    {
      "epoch": 1.0469779995092827,
      "grad_norm": 7.89134407043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0579027384519577,
      "logits/rejected": 0.20969367027282715,
      "logps/chosen": -308.7718200683594,
      "logps/rejected": -443.3899230957031,
      "loss": 0.4796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1836700439453125,
      "rewards/margins": 0.9854664206504822,
      "rewards/rejected": -0.8017963767051697,
      "step": 8001
    },
    {
      "epoch": 1.0471088574466345,
      "grad_norm": 6.136472225189209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06862400472164154,
      "logits/rejected": 0.009849529713392258,
      "logps/chosen": -522.994384765625,
      "logps/rejected": -353.025390625,
      "loss": 0.2993,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.005356639623641968,
      "rewards/margins": 1.7378361225128174,
      "rewards/rejected": -1.7431927919387817,
      "step": 8002
    },
    {
      "epoch": 1.0472397153839863,
      "grad_norm": 4.304943084716797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.317548006772995,
      "logits/rejected": -0.4912075698375702,
      "logps/chosen": -476.0510559082031,
      "logps/rejected": -413.3937072753906,
      "loss": 0.2181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.677491307258606,
      "rewards/margins": 2.318850040435791,
      "rewards/rejected": -1.6413588523864746,
      "step": 8003
    },
    {
      "epoch": 1.047370573321338,
      "grad_norm": 6.700288772583008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11782003194093704,
      "logits/rejected": 0.04803144931793213,
      "logps/chosen": -425.33966064453125,
      "logps/rejected": -348.62115478515625,
      "loss": 0.4291,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11164550483226776,
      "rewards/margins": 0.946497917175293,
      "rewards/rejected": -0.8348524570465088,
      "step": 8004
    },
    {
      "epoch": 1.0475014312586899,
      "grad_norm": 12.275018692016602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14118047058582306,
      "logits/rejected": -0.030525077134370804,
      "logps/chosen": -356.3570556640625,
      "logps/rejected": -467.19464111328125,
      "loss": 0.9398,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7304195761680603,
      "rewards/margins": -0.17325729131698608,
      "rewards/rejected": -0.5571622848510742,
      "step": 8005
    },
    {
      "epoch": 1.0476322891960415,
      "grad_norm": 5.873433589935303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04284335672855377,
      "logits/rejected": -0.05797986686229706,
      "logps/chosen": -625.9288940429688,
      "logps/rejected": -528.4742431640625,
      "loss": 0.3829,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11808034777641296,
      "rewards/margins": 1.1938438415527344,
      "rewards/rejected": -1.075763463973999,
      "step": 8006
    },
    {
      "epoch": 1.0477631471333932,
      "grad_norm": 5.330011367797852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10024361312389374,
      "logits/rejected": 0.06498914957046509,
      "logps/chosen": -445.2268371582031,
      "logps/rejected": -378.1833801269531,
      "loss": 0.3705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41341322660446167,
      "rewards/margins": 1.468200445175171,
      "rewards/rejected": -1.881613850593567,
      "step": 8007
    },
    {
      "epoch": 1.047894005070745,
      "grad_norm": 6.015901565551758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14113694429397583,
      "logits/rejected": 0.07090489566326141,
      "logps/chosen": -341.09332275390625,
      "logps/rejected": -466.1663818359375,
      "loss": 0.4437,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32597893476486206,
      "rewards/margins": 0.9704851508140564,
      "rewards/rejected": -1.296464204788208,
      "step": 8008
    },
    {
      "epoch": 1.0480248630080968,
      "grad_norm": 3.3493611812591553,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0457511804997921,
      "logits/rejected": -0.13771562278270721,
      "logps/chosen": -378.1379089355469,
      "logps/rejected": -439.724365234375,
      "loss": 0.294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2446201741695404,
      "rewards/margins": 1.559079885482788,
      "rewards/rejected": -1.3144594430923462,
      "step": 8009
    },
    {
      "epoch": 1.0481557209454486,
      "grad_norm": 8.906545639038086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.128318652510643,
      "logits/rejected": 0.03415703773498535,
      "logps/chosen": -324.7915954589844,
      "logps/rejected": -408.59503173828125,
      "loss": 0.6049,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7437820434570312,
      "rewards/margins": 0.6377055644989014,
      "rewards/rejected": -1.3814876079559326,
      "step": 8010
    },
    {
      "epoch": 1.0482865788828004,
      "grad_norm": 7.238662242889404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18756243586540222,
      "logits/rejected": -0.20768606662750244,
      "logps/chosen": -484.54058837890625,
      "logps/rejected": -596.7490844726562,
      "loss": 0.4838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0510274171829224,
      "rewards/margins": 0.6069848537445068,
      "rewards/rejected": -1.6580123901367188,
      "step": 8011
    },
    {
      "epoch": 1.0484174368201522,
      "grad_norm": 8.108330726623535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3547165095806122,
      "logits/rejected": -0.19001588225364685,
      "logps/chosen": -437.86932373046875,
      "logps/rejected": -385.44915771484375,
      "loss": 0.4647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6015130877494812,
      "rewards/margins": 0.9373830556869507,
      "rewards/rejected": -1.5388962030410767,
      "step": 8012
    },
    {
      "epoch": 1.048548294757504,
      "grad_norm": 5.0092644691467285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04438554495573044,
      "logits/rejected": -0.07340556383132935,
      "logps/chosen": -616.49658203125,
      "logps/rejected": -367.4130859375,
      "loss": 0.29,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2786872982978821,
      "rewards/margins": 1.7886743545532227,
      "rewards/rejected": -1.5099869966506958,
      "step": 8013
    },
    {
      "epoch": 1.0486791526948556,
      "grad_norm": 7.764276504516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05629026144742966,
      "logits/rejected": -0.21475067734718323,
      "logps/chosen": -383.561279296875,
      "logps/rejected": -569.649658203125,
      "loss": 0.3543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05438346043229103,
      "rewards/margins": 2.468291759490967,
      "rewards/rejected": -2.5226755142211914,
      "step": 8014
    },
    {
      "epoch": 1.0488100106322074,
      "grad_norm": 10.844131469726562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20258566737174988,
      "logits/rejected": -0.1755220741033554,
      "logps/chosen": -538.3411865234375,
      "logps/rejected": -353.3369445800781,
      "loss": 0.6388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21952363848686218,
      "rewards/margins": 1.0304036140441895,
      "rewards/rejected": -0.8108799457550049,
      "step": 8015
    },
    {
      "epoch": 1.0489408685695591,
      "grad_norm": 4.769049644470215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4996551275253296,
      "logits/rejected": -0.5133869647979736,
      "logps/chosen": -805.0723876953125,
      "logps/rejected": -417.7627868652344,
      "loss": 0.2215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3154037594795227,
      "rewards/margins": 1.7236720323562622,
      "rewards/rejected": -1.4082683324813843,
      "step": 8016
    },
    {
      "epoch": 1.049071726506911,
      "grad_norm": 9.960960388183594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22618353366851807,
      "logits/rejected": -0.2175631821155548,
      "logps/chosen": -479.11627197265625,
      "logps/rejected": -452.66070556640625,
      "loss": 0.6586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2362508773803711,
      "rewards/margins": 0.4377700686454773,
      "rewards/rejected": -0.6740209460258484,
      "step": 8017
    },
    {
      "epoch": 1.0492025844442627,
      "grad_norm": 6.463727951049805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00272374227643013,
      "logits/rejected": 0.03177379071712494,
      "logps/chosen": -635.9616088867188,
      "logps/rejected": -515.9417724609375,
      "loss": 0.3984,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04922601580619812,
      "rewards/margins": 1.447489619255066,
      "rewards/rejected": -1.4967155456542969,
      "step": 8018
    },
    {
      "epoch": 1.0493334423816145,
      "grad_norm": 7.207879066467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14430463314056396,
      "logits/rejected": -0.30150333046913147,
      "logps/chosen": -426.626708984375,
      "logps/rejected": -412.0497131347656,
      "loss": 0.4335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8624467849731445,
      "rewards/margins": 1.0641305446624756,
      "rewards/rejected": -1.9265772104263306,
      "step": 8019
    },
    {
      "epoch": 1.0494643003189663,
      "grad_norm": 7.646512985229492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09815476834774017,
      "logits/rejected": 0.08582685887813568,
      "logps/chosen": -366.7939758300781,
      "logps/rejected": -415.84478759765625,
      "loss": 0.4739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.657483696937561,
      "rewards/margins": 0.7749845385551453,
      "rewards/rejected": -1.4324681758880615,
      "step": 8020
    },
    {
      "epoch": 1.0495951582563179,
      "grad_norm": 6.098303318023682,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11686748266220093,
      "logits/rejected": -0.204665869474411,
      "logps/chosen": -312.433349609375,
      "logps/rejected": -317.5220642089844,
      "loss": 0.4751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8651368021965027,
      "rewards/margins": 0.7019424438476562,
      "rewards/rejected": -1.5670791864395142,
      "step": 8021
    },
    {
      "epoch": 1.0497260161936697,
      "grad_norm": 9.519482612609863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20818838477134705,
      "logits/rejected": 0.06805359572172165,
      "logps/chosen": -717.224365234375,
      "logps/rejected": -472.44830322265625,
      "loss": 0.5368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08314406126737595,
      "rewards/margins": 0.9031943082809448,
      "rewards/rejected": -0.8200502395629883,
      "step": 8022
    },
    {
      "epoch": 1.0498568741310215,
      "grad_norm": 8.170845031738281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04511116445064545,
      "logits/rejected": -0.02947039157152176,
      "logps/chosen": -497.62884521484375,
      "logps/rejected": -462.2577819824219,
      "loss": 0.4384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2934928834438324,
      "rewards/margins": 0.9958095550537109,
      "rewards/rejected": -0.7023166418075562,
      "step": 8023
    },
    {
      "epoch": 1.0499877320683733,
      "grad_norm": 7.721179962158203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11044516414403915,
      "logits/rejected": -0.19150671362876892,
      "logps/chosen": -462.01397705078125,
      "logps/rejected": -516.1663818359375,
      "loss": 0.3456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.878524899482727,
      "rewards/margins": 1.54275643825531,
      "rewards/rejected": -2.421281337738037,
      "step": 8024
    },
    {
      "epoch": 1.050118590005725,
      "grad_norm": 4.54469108581543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15157409012317657,
      "logits/rejected": -0.006784670054912567,
      "logps/chosen": -372.9944152832031,
      "logps/rejected": -354.50555419921875,
      "loss": 0.1652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8585548400878906,
      "rewards/margins": 3.3531742095947266,
      "rewards/rejected": -2.494619131088257,
      "step": 8025
    },
    {
      "epoch": 1.0502494479430768,
      "grad_norm": 9.968880653381348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27385082840919495,
      "logits/rejected": -0.23416569828987122,
      "logps/chosen": -295.1047668457031,
      "logps/rejected": -396.0703125,
      "loss": 0.4304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6456509828567505,
      "rewards/margins": 1.4338099956512451,
      "rewards/rejected": -2.079460859298706,
      "step": 8026
    },
    {
      "epoch": 1.0503803058804286,
      "grad_norm": 5.052858829498291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09538151323795319,
      "logits/rejected": 0.008344825357198715,
      "logps/chosen": -593.0447998046875,
      "logps/rejected": -697.4436645507812,
      "loss": 0.1775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7973262667655945,
      "rewards/margins": 2.355051040649414,
      "rewards/rejected": -1.5577248334884644,
      "step": 8027
    },
    {
      "epoch": 1.0505111638177804,
      "grad_norm": 7.7544121742248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03531785309314728,
      "logits/rejected": -0.02465926855802536,
      "logps/chosen": -492.6062927246094,
      "logps/rejected": -372.9068298339844,
      "loss": 0.4587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24878959357738495,
      "rewards/margins": 0.9436325430870056,
      "rewards/rejected": -0.6948428750038147,
      "step": 8028
    },
    {
      "epoch": 1.050642021755132,
      "grad_norm": 5.4481635093688965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24119004607200623,
      "logits/rejected": 0.11354392766952515,
      "logps/chosen": -533.5831298828125,
      "logps/rejected": -398.0886535644531,
      "loss": 0.2214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08094008266925812,
      "rewards/margins": 2.921043872833252,
      "rewards/rejected": -2.8401038646698,
      "step": 8029
    },
    {
      "epoch": 1.0507728796924838,
      "grad_norm": 5.551161766052246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03262106329202652,
      "logits/rejected": -0.04439961910247803,
      "logps/chosen": -306.7262268066406,
      "logps/rejected": -344.28167724609375,
      "loss": 0.2739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6027012467384338,
      "rewards/margins": 1.9208534955978394,
      "rewards/rejected": -2.523554563522339,
      "step": 8030
    },
    {
      "epoch": 1.0509037376298356,
      "grad_norm": 7.04072904586792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46394801139831543,
      "logits/rejected": -0.42275160551071167,
      "logps/chosen": -272.28887939453125,
      "logps/rejected": -288.7215576171875,
      "loss": 0.4731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4352806508541107,
      "rewards/margins": 0.9947630167007446,
      "rewards/rejected": -1.4300435781478882,
      "step": 8031
    },
    {
      "epoch": 1.0510345955671874,
      "grad_norm": 4.87750244140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3625708222389221,
      "logits/rejected": -0.14589758217334747,
      "logps/chosen": -429.3694152832031,
      "logps/rejected": -507.12969970703125,
      "loss": 0.2853,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9182748794555664,
      "rewards/margins": 1.9926565885543823,
      "rewards/rejected": -1.074381709098816,
      "step": 8032
    },
    {
      "epoch": 1.0511654535045392,
      "grad_norm": 5.711055755615234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21716170012950897,
      "logits/rejected": -0.12310580909252167,
      "logps/chosen": -525.328125,
      "logps/rejected": -387.4588317871094,
      "loss": 0.2912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16547852754592896,
      "rewards/margins": 1.375766396522522,
      "rewards/rejected": -1.2102879285812378,
      "step": 8033
    },
    {
      "epoch": 1.051296311441891,
      "grad_norm": 3.9378881454467773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14443868398666382,
      "logits/rejected": 0.006254091858863831,
      "logps/chosen": -413.0001220703125,
      "logps/rejected": -410.259033203125,
      "loss": 0.2576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23915578424930573,
      "rewards/margins": 1.5375926494598389,
      "rewards/rejected": -1.7767482995986938,
      "step": 8034
    },
    {
      "epoch": 1.0514271693792427,
      "grad_norm": 8.194058418273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09361199289560318,
      "logits/rejected": -0.0928492397069931,
      "logps/chosen": -457.82769775390625,
      "logps/rejected": -424.814453125,
      "loss": 0.5463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5842952728271484,
      "rewards/margins": 1.2877273559570312,
      "rewards/rejected": -1.8720226287841797,
      "step": 8035
    },
    {
      "epoch": 1.0515580273165943,
      "grad_norm": 8.971075057983398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.059716030955314636,
      "logits/rejected": 0.04962165653705597,
      "logps/chosen": -560.832763671875,
      "logps/rejected": -586.9539184570312,
      "loss": 0.6991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33558189868927,
      "rewards/margins": 1.2306010723114014,
      "rewards/rejected": -0.8950191736221313,
      "step": 8036
    },
    {
      "epoch": 1.051688885253946,
      "grad_norm": 10.645554542541504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10411962121725082,
      "logits/rejected": 0.021422721445560455,
      "logps/chosen": -501.4745178222656,
      "logps/rejected": -351.2893371582031,
      "loss": 0.7005,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6990981698036194,
      "rewards/margins": 0.6525285243988037,
      "rewards/rejected": -1.3516267538070679,
      "step": 8037
    },
    {
      "epoch": 1.051819743191298,
      "grad_norm": 3.59733510017395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15146318078041077,
      "logits/rejected": 0.19784489274024963,
      "logps/chosen": -357.9021301269531,
      "logps/rejected": -364.8707275390625,
      "loss": 0.353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28755152225494385,
      "rewards/margins": 1.0924935340881348,
      "rewards/rejected": -0.8049420118331909,
      "step": 8038
    },
    {
      "epoch": 1.0519506011286497,
      "grad_norm": 5.562874794006348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07371015846729279,
      "logits/rejected": -0.06357252597808838,
      "logps/chosen": -414.9724426269531,
      "logps/rejected": -600.4150390625,
      "loss": 0.3472,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0046582221984863,
      "rewards/margins": 1.5029209852218628,
      "rewards/rejected": -2.5075793266296387,
      "step": 8039
    },
    {
      "epoch": 1.0520814590660015,
      "grad_norm": 9.967549324035645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1064397394657135,
      "logits/rejected": -0.274111807346344,
      "logps/chosen": -556.4002075195312,
      "logps/rejected": -424.275146484375,
      "loss": 0.4295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11353664100170135,
      "rewards/margins": 1.792708396911621,
      "rewards/rejected": -1.9062447547912598,
      "step": 8040
    },
    {
      "epoch": 1.0522123170033533,
      "grad_norm": 4.489132404327393,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16049957275390625,
      "logits/rejected": 0.10845224559307098,
      "logps/chosen": -313.9296569824219,
      "logps/rejected": -319.2103271484375,
      "loss": 0.2679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1544143557548523,
      "rewards/margins": 1.3813709020614624,
      "rewards/rejected": -1.2269566059112549,
      "step": 8041
    },
    {
      "epoch": 1.052343174940705,
      "grad_norm": 5.28557014465332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017186351120471954,
      "logits/rejected": -0.0383838452398777,
      "logps/chosen": -434.00927734375,
      "logps/rejected": -520.79296875,
      "loss": 0.2248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2666168212890625,
      "rewards/margins": 2.3882029056549072,
      "rewards/rejected": -2.121586322784424,
      "step": 8042
    },
    {
      "epoch": 1.0524740328780569,
      "grad_norm": 8.601481437683105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1859455555677414,
      "logits/rejected": 0.22501935064792633,
      "logps/chosen": -609.2149047851562,
      "logps/rejected": -728.2744750976562,
      "loss": 0.5841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02422599494457245,
      "rewards/margins": 1.9545804262161255,
      "rewards/rejected": -1.9303542375564575,
      "step": 8043
    },
    {
      "epoch": 1.0526048908154084,
      "grad_norm": 9.487570762634277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18395528197288513,
      "logits/rejected": -0.14764012396335602,
      "logps/chosen": -617.0521240234375,
      "logps/rejected": -624.322509765625,
      "loss": 0.541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21763570606708527,
      "rewards/margins": 1.1879829168319702,
      "rewards/rejected": -1.4056187868118286,
      "step": 8044
    },
    {
      "epoch": 1.0527357487527602,
      "grad_norm": 8.657235145568848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13285110890865326,
      "logits/rejected": -0.06825815886259079,
      "logps/chosen": -471.200927734375,
      "logps/rejected": -672.5460815429688,
      "loss": 0.4913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3689665198326111,
      "rewards/margins": 1.042311429977417,
      "rewards/rejected": -1.4112777709960938,
      "step": 8045
    },
    {
      "epoch": 1.052866606690112,
      "grad_norm": 8.322260856628418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.156732439994812,
      "logits/rejected": -0.21046602725982666,
      "logps/chosen": -293.7257995605469,
      "logps/rejected": -437.7151184082031,
      "loss": 0.4545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3022514283657074,
      "rewards/margins": 1.4945404529571533,
      "rewards/rejected": -1.1922889947891235,
      "step": 8046
    },
    {
      "epoch": 1.0529974646274638,
      "grad_norm": 6.7000813484191895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10154145956039429,
      "logits/rejected": -0.008961163461208344,
      "logps/chosen": -384.1710205078125,
      "logps/rejected": -390.6895751953125,
      "loss": 0.5677,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2730626165866852,
      "rewards/margins": 0.9043008685112,
      "rewards/rejected": -0.6312382221221924,
      "step": 8047
    },
    {
      "epoch": 1.0531283225648156,
      "grad_norm": 5.369562149047852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0016274694353342056,
      "logits/rejected": -0.2848730683326721,
      "logps/chosen": -346.0970458984375,
      "logps/rejected": -339.45355224609375,
      "loss": 0.3143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2381061613559723,
      "rewards/margins": 2.324904441833496,
      "rewards/rejected": -2.5630104541778564,
      "step": 8048
    },
    {
      "epoch": 1.0532591805021674,
      "grad_norm": 8.796385765075684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00836605578660965,
      "logits/rejected": -0.07270855456590652,
      "logps/chosen": -419.57965087890625,
      "logps/rejected": -452.2489318847656,
      "loss": 0.5853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14086362719535828,
      "rewards/margins": 1.317948341369629,
      "rewards/rejected": -1.45881187915802,
      "step": 8049
    },
    {
      "epoch": 1.0533900384395192,
      "grad_norm": 8.822296142578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22298747301101685,
      "logits/rejected": -0.26792195439338684,
      "logps/chosen": -518.5490112304688,
      "logps/rejected": -624.14697265625,
      "loss": 0.5221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7913181781768799,
      "rewards/margins": 1.0489698648452759,
      "rewards/rejected": -1.8402882814407349,
      "step": 8050
    },
    {
      "epoch": 1.053520896376871,
      "grad_norm": 5.05142068862915,
      "learning_rate": 5e-06,
      "logits/chosen": 0.127236008644104,
      "logits/rejected": 0.11803700029850006,
      "logps/chosen": -541.9404296875,
      "logps/rejected": -435.3921203613281,
      "loss": 0.3041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8102526664733887,
      "rewards/margins": 1.5529289245605469,
      "rewards/rejected": -0.7426763772964478,
      "step": 8051
    },
    {
      "epoch": 1.0536517543142225,
      "grad_norm": 12.144122123718262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02979559451341629,
      "logits/rejected": 0.3824329674243927,
      "logps/chosen": -342.8598937988281,
      "logps/rejected": -438.9958801269531,
      "loss": 0.5812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13604286313056946,
      "rewards/margins": 1.4246242046356201,
      "rewards/rejected": -1.5606670379638672,
      "step": 8052
    },
    {
      "epoch": 1.0537826122515743,
      "grad_norm": 4.77502965927124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3686162829399109,
      "logits/rejected": 0.11264179646968842,
      "logps/chosen": -269.7996520996094,
      "logps/rejected": -320.0442810058594,
      "loss": 0.2437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42112138867378235,
      "rewards/margins": 1.7212178707122803,
      "rewards/rejected": -2.1423394680023193,
      "step": 8053
    },
    {
      "epoch": 1.0539134701889261,
      "grad_norm": 8.989578247070312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1298343539237976,
      "logits/rejected": -0.15319626033306122,
      "logps/chosen": -432.1883850097656,
      "logps/rejected": -294.54315185546875,
      "loss": 0.5462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1336195170879364,
      "rewards/margins": 1.7954108715057373,
      "rewards/rejected": -1.661791205406189,
      "step": 8054
    },
    {
      "epoch": 1.054044328126278,
      "grad_norm": 5.304556369781494,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10223853588104248,
      "logits/rejected": 0.00012878328561782837,
      "logps/chosen": -481.8624267578125,
      "logps/rejected": -369.27545166015625,
      "loss": 0.2578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8016502261161804,
      "rewards/margins": 1.7295578718185425,
      "rewards/rejected": -2.5312082767486572,
      "step": 8055
    },
    {
      "epoch": 1.0541751860636297,
      "grad_norm": 8.920342445373535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10596691071987152,
      "logits/rejected": 0.13438493013381958,
      "logps/chosen": -433.10821533203125,
      "logps/rejected": -428.9844970703125,
      "loss": 0.3419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2357061505317688,
      "rewards/margins": 1.3582532405853271,
      "rewards/rejected": -1.5939594507217407,
      "step": 8056
    },
    {
      "epoch": 1.0543060440009815,
      "grad_norm": 10.330195426940918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2602611184120178,
      "logits/rejected": 0.0951630026102066,
      "logps/chosen": -456.0101013183594,
      "logps/rejected": -369.24066162109375,
      "loss": 0.6498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21169301867485046,
      "rewards/margins": 0.4047696590423584,
      "rewards/rejected": -0.6164627075195312,
      "step": 8057
    },
    {
      "epoch": 1.0544369019383333,
      "grad_norm": 8.724531173706055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00591779500246048,
      "logits/rejected": 0.053012531250715256,
      "logps/chosen": -475.2773132324219,
      "logps/rejected": -562.9010620117188,
      "loss": 0.464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18585115671157837,
      "rewards/margins": 1.7211627960205078,
      "rewards/rejected": -1.5353116989135742,
      "step": 8058
    },
    {
      "epoch": 1.054567759875685,
      "grad_norm": 7.690154552459717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28357404470443726,
      "logits/rejected": -0.05418357253074646,
      "logps/chosen": -338.52520751953125,
      "logps/rejected": -400.6421813964844,
      "loss": 0.5369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.054780393838882446,
      "rewards/margins": 0.9518033266067505,
      "rewards/rejected": -1.0065838098526,
      "step": 8059
    },
    {
      "epoch": 1.0546986178130366,
      "grad_norm": 7.750525951385498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1886894702911377,
      "logits/rejected": -0.20312625169754028,
      "logps/chosen": -397.322509765625,
      "logps/rejected": -498.3089904785156,
      "loss": 0.4876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.28497087955474854,
      "rewards/margins": 1.0236499309539795,
      "rewards/rejected": -0.738679051399231,
      "step": 8060
    },
    {
      "epoch": 1.0548294757503884,
      "grad_norm": 7.268904209136963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2032938450574875,
      "logits/rejected": -0.06768150627613068,
      "logps/chosen": -303.1707458496094,
      "logps/rejected": -320.81475830078125,
      "loss": 0.6773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7282631397247314,
      "rewards/margins": 1.0385297536849976,
      "rewards/rejected": -1.7667930126190186,
      "step": 8061
    },
    {
      "epoch": 1.0549603336877402,
      "grad_norm": 8.412256240844727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20656992495059967,
      "logits/rejected": 0.036342017352581024,
      "logps/chosen": -492.3844909667969,
      "logps/rejected": -551.4571533203125,
      "loss": 0.4409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.30502548813819885,
      "rewards/margins": 0.7092176675796509,
      "rewards/rejected": -0.4041922092437744,
      "step": 8062
    },
    {
      "epoch": 1.055091191625092,
      "grad_norm": 10.990421295166016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06630465388298035,
      "logits/rejected": -0.013617726974189281,
      "logps/chosen": -487.9173583984375,
      "logps/rejected": -435.1372375488281,
      "loss": 0.8676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7726534605026245,
      "rewards/margins": 0.5105495452880859,
      "rewards/rejected": -1.283203125,
      "step": 8063
    },
    {
      "epoch": 1.0552220495624438,
      "grad_norm": 10.463281631469727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09395358711481094,
      "logits/rejected": 0.011806413531303406,
      "logps/chosen": -558.6651611328125,
      "logps/rejected": -467.6982727050781,
      "loss": 0.67,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.42480945587158203,
      "rewards/margins": 0.40542253851890564,
      "rewards/rejected": -0.8302320241928101,
      "step": 8064
    },
    {
      "epoch": 1.0553529074997956,
      "grad_norm": 8.255476951599121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10082542896270752,
      "logits/rejected": -0.07455060631036758,
      "logps/chosen": -404.4051818847656,
      "logps/rejected": -371.2901916503906,
      "loss": 0.4249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1177359819412231,
      "rewards/margins": 0.7767207622528076,
      "rewards/rejected": -1.8944569826126099,
      "step": 8065
    },
    {
      "epoch": 1.0554837654371474,
      "grad_norm": 5.808228969573975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07193425297737122,
      "logits/rejected": -0.36231034994125366,
      "logps/chosen": -463.3464660644531,
      "logps/rejected": -446.8818359375,
      "loss": 0.2601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21253015100955963,
      "rewards/margins": 2.3314507007598877,
      "rewards/rejected": -2.5439810752868652,
      "step": 8066
    },
    {
      "epoch": 1.055614623374499,
      "grad_norm": 8.65563678741455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11707166582345963,
      "logits/rejected": -0.18225517868995667,
      "logps/chosen": -401.5802307128906,
      "logps/rejected": -348.2853698730469,
      "loss": 0.593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9031972885131836,
      "rewards/margins": 0.6177091598510742,
      "rewards/rejected": -1.5209064483642578,
      "step": 8067
    },
    {
      "epoch": 1.0557454813118508,
      "grad_norm": 3.8298630714416504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13531331717967987,
      "logits/rejected": -0.2933499217033386,
      "logps/chosen": -491.8688659667969,
      "logps/rejected": -448.1114501953125,
      "loss": 0.2737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.422046422958374,
      "rewards/margins": 1.791507601737976,
      "rewards/rejected": -1.3694610595703125,
      "step": 8068
    },
    {
      "epoch": 1.0558763392492025,
      "grad_norm": 19.200902938842773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2187357395887375,
      "logits/rejected": 0.1282806694507599,
      "logps/chosen": -548.9677734375,
      "logps/rejected": -366.32464599609375,
      "loss": 0.6782,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3406999707221985,
      "rewards/margins": 0.6283538937568665,
      "rewards/rejected": -0.9690538644790649,
      "step": 8069
    },
    {
      "epoch": 1.0560071971865543,
      "grad_norm": 10.624632835388184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22021882236003876,
      "logits/rejected": 0.2339361310005188,
      "logps/chosen": -379.26312255859375,
      "logps/rejected": -392.94696044921875,
      "loss": 0.7361,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.282518595457077,
      "rewards/margins": 0.8063482642173767,
      "rewards/rejected": -1.0888668298721313,
      "step": 8070
    },
    {
      "epoch": 1.0561380551239061,
      "grad_norm": 7.100608825683594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4432276785373688,
      "logits/rejected": 0.31909918785095215,
      "logps/chosen": -424.0869140625,
      "logps/rejected": -383.52783203125,
      "loss": 0.4927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21885302662849426,
      "rewards/margins": 0.8123594522476196,
      "rewards/rejected": -0.593506395816803,
      "step": 8071
    },
    {
      "epoch": 1.056268913061258,
      "grad_norm": 10.384833335876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10116663575172424,
      "logits/rejected": -0.08645933866500854,
      "logps/chosen": -581.865234375,
      "logps/rejected": -504.70001220703125,
      "loss": 0.6939,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3465898931026459,
      "rewards/margins": 0.2664545178413391,
      "rewards/rejected": -0.6130443811416626,
      "step": 8072
    },
    {
      "epoch": 1.0563997709986097,
      "grad_norm": 11.111794471740723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24709293246269226,
      "logits/rejected": -0.0161159448325634,
      "logps/chosen": -479.8074951171875,
      "logps/rejected": -555.9971923828125,
      "loss": 0.6397,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4636804759502411,
      "rewards/margins": 0.28517818450927734,
      "rewards/rejected": -0.7488586902618408,
      "step": 8073
    },
    {
      "epoch": 1.0565306289359615,
      "grad_norm": 10.172113418579102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3003522455692291,
      "logits/rejected": -0.37949106097221375,
      "logps/chosen": -368.9613952636719,
      "logps/rejected": -380.13751220703125,
      "loss": 0.6963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8795983791351318,
      "rewards/margins": 0.8053920865058899,
      "rewards/rejected": -1.6849905252456665,
      "step": 8074
    },
    {
      "epoch": 1.056661486873313,
      "grad_norm": 4.866845607757568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04683250933885574,
      "logits/rejected": 0.012686237692832947,
      "logps/chosen": -479.000732421875,
      "logps/rejected": -494.1247253417969,
      "loss": 0.4085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1666593700647354,
      "rewards/margins": 1.0396950244903564,
      "rewards/rejected": -0.8730356693267822,
      "step": 8075
    },
    {
      "epoch": 1.0567923448106649,
      "grad_norm": 6.949973106384277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3179771900177002,
      "logits/rejected": 0.15383189916610718,
      "logps/chosen": -548.1445922851562,
      "logps/rejected": -390.69573974609375,
      "loss": 0.2237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.007832519710063934,
      "rewards/margins": 2.1291732788085938,
      "rewards/rejected": -2.1370060443878174,
      "step": 8076
    },
    {
      "epoch": 1.0569232027480167,
      "grad_norm": 7.734500885009766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04197785630822182,
      "logits/rejected": -0.3143506646156311,
      "logps/chosen": -668.9985961914062,
      "logps/rejected": -532.91259765625,
      "loss": 0.5117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021156691014766693,
      "rewards/margins": 0.6171243190765381,
      "rewards/rejected": -0.6382810473442078,
      "step": 8077
    },
    {
      "epoch": 1.0570540606853684,
      "grad_norm": 10.490431785583496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011984661221504211,
      "logits/rejected": -0.2561474144458771,
      "logps/chosen": -518.379150390625,
      "logps/rejected": -561.384521484375,
      "loss": 0.7287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6234222650527954,
      "rewards/margins": 1.2123807668685913,
      "rewards/rejected": -1.8358030319213867,
      "step": 8078
    },
    {
      "epoch": 1.0571849186227202,
      "grad_norm": 5.316216945648193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0961870327591896,
      "logits/rejected": -0.19247764348983765,
      "logps/chosen": -398.2770080566406,
      "logps/rejected": -347.75885009765625,
      "loss": 0.3709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07174035906791687,
      "rewards/margins": 1.451380968093872,
      "rewards/rejected": -1.3796405792236328,
      "step": 8079
    },
    {
      "epoch": 1.057315776560072,
      "grad_norm": 5.355294704437256,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06685639172792435,
      "logits/rejected": -0.36916399002075195,
      "logps/chosen": -465.6102600097656,
      "logps/rejected": -592.7099609375,
      "loss": 0.2836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5413408279418945,
      "rewards/margins": 1.9506728649139404,
      "rewards/rejected": -2.492013692855835,
      "step": 8080
    },
    {
      "epoch": 1.0574466344974238,
      "grad_norm": 13.635344505310059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3858877420425415,
      "logits/rejected": -0.15206748247146606,
      "logps/chosen": -259.9146728515625,
      "logps/rejected": -276.60809326171875,
      "loss": 1.0168,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.11545713990926743,
      "rewards/margins": -0.4095243811607361,
      "rewards/rejected": 0.2940671741962433,
      "step": 8081
    },
    {
      "epoch": 1.0575774924347754,
      "grad_norm": 5.332919597625732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.142358660697937,
      "logits/rejected": -0.12440724670886993,
      "logps/chosen": -275.4407958984375,
      "logps/rejected": -249.4898681640625,
      "loss": 0.4844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.281485378742218,
      "rewards/margins": 0.9331039786338806,
      "rewards/rejected": -0.6516185998916626,
      "step": 8082
    },
    {
      "epoch": 1.0577083503721272,
      "grad_norm": 11.60004997253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18066193163394928,
      "logits/rejected": 0.30678409337997437,
      "logps/chosen": -519.3863525390625,
      "logps/rejected": -451.286376953125,
      "loss": 0.7206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02588462084531784,
      "rewards/margins": 0.2768310308456421,
      "rewards/rejected": -0.25094640254974365,
      "step": 8083
    },
    {
      "epoch": 1.057839208309479,
      "grad_norm": 5.7807698249816895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03694959357380867,
      "logits/rejected": -0.1642923504114151,
      "logps/chosen": -433.3838195800781,
      "logps/rejected": -412.5142822265625,
      "loss": 0.3848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1203306913375854,
      "rewards/margins": 1.170072078704834,
      "rewards/rejected": -2.290402889251709,
      "step": 8084
    },
    {
      "epoch": 1.0579700662468308,
      "grad_norm": 7.345766544342041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07257646322250366,
      "logits/rejected": 0.08445815742015839,
      "logps/chosen": -409.0501403808594,
      "logps/rejected": -557.3218383789062,
      "loss": 0.4808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05929070711135864,
      "rewards/margins": 0.661585807800293,
      "rewards/rejected": -0.6022951006889343,
      "step": 8085
    },
    {
      "epoch": 1.0581009241841826,
      "grad_norm": 9.76107120513916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2436213493347168,
      "logits/rejected": 0.02199813723564148,
      "logps/chosen": -489.6559143066406,
      "logps/rejected": -471.571533203125,
      "loss": 0.7509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.508984386920929,
      "rewards/margins": 0.5603115558624268,
      "rewards/rejected": -0.051327139139175415,
      "step": 8086
    },
    {
      "epoch": 1.0582317821215343,
      "grad_norm": 8.108001708984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08777877688407898,
      "logits/rejected": -0.009806163609027863,
      "logps/chosen": -396.79888916015625,
      "logps/rejected": -419.3434143066406,
      "loss": 0.717,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7475918531417847,
      "rewards/margins": 0.42786240577697754,
      "rewards/rejected": -1.1754541397094727,
      "step": 8087
    },
    {
      "epoch": 1.0583626400588861,
      "grad_norm": 5.317657947540283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18615689873695374,
      "logits/rejected": -0.32390010356903076,
      "logps/chosen": -691.6188354492188,
      "logps/rejected": -476.7912902832031,
      "loss": 0.2951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0804166793823242,
      "rewards/margins": 1.392361044883728,
      "rewards/rejected": -0.3119443953037262,
      "step": 8088
    },
    {
      "epoch": 1.058493497996238,
      "grad_norm": 4.130785942077637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19479495286941528,
      "logits/rejected": -0.266363263130188,
      "logps/chosen": -724.5762939453125,
      "logps/rejected": -515.4176025390625,
      "loss": 0.1776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9346899390220642,
      "rewards/margins": 2.3403568267822266,
      "rewards/rejected": -1.4056669473648071,
      "step": 8089
    },
    {
      "epoch": 1.0586243559335895,
      "grad_norm": 7.876460075378418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07088561356067657,
      "logits/rejected": -0.3624945282936096,
      "logps/chosen": -547.8441162109375,
      "logps/rejected": -498.5450744628906,
      "loss": 0.4665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18336430191993713,
      "rewards/margins": 1.9426263570785522,
      "rewards/rejected": -1.7592623233795166,
      "step": 8090
    },
    {
      "epoch": 1.0587552138709413,
      "grad_norm": 8.555721282958984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1100223958492279,
      "logits/rejected": -0.4072885513305664,
      "logps/chosen": -417.1591796875,
      "logps/rejected": -423.46441650390625,
      "loss": 0.5985,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5033011436462402,
      "rewards/margins": 0.5345213413238525,
      "rewards/rejected": -1.0378226041793823,
      "step": 8091
    },
    {
      "epoch": 1.058886071808293,
      "grad_norm": 10.310859680175781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009353997185826302,
      "logits/rejected": -0.06784488260746002,
      "logps/chosen": -542.837890625,
      "logps/rejected": -448.71453857421875,
      "loss": 0.7901,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.47007596492767334,
      "rewards/margins": -0.04207974672317505,
      "rewards/rejected": -0.4279962480068207,
      "step": 8092
    },
    {
      "epoch": 1.0590169297456449,
      "grad_norm": 7.178246021270752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015103194862604141,
      "logits/rejected": -0.09114094078540802,
      "logps/chosen": -344.2265319824219,
      "logps/rejected": -434.0738525390625,
      "loss": 0.4999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37912559509277344,
      "rewards/margins": 1.0722038745880127,
      "rewards/rejected": -1.4513295888900757,
      "step": 8093
    },
    {
      "epoch": 1.0591477876829967,
      "grad_norm": 7.939696788787842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2543707489967346,
      "logits/rejected": -0.24097608029842377,
      "logps/chosen": -532.1661987304688,
      "logps/rejected": -480.3707580566406,
      "loss": 0.4952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2469324767589569,
      "rewards/margins": 1.0627727508544922,
      "rewards/rejected": -1.309705138206482,
      "step": 8094
    },
    {
      "epoch": 1.0592786456203485,
      "grad_norm": 5.505468845367432,
      "learning_rate": 5e-06,
      "logits/chosen": 0.28120797872543335,
      "logits/rejected": 0.27725863456726074,
      "logps/chosen": -595.0574951171875,
      "logps/rejected": -555.206787109375,
      "loss": 0.2617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36782675981521606,
      "rewards/margins": 1.4249050617218018,
      "rewards/rejected": -1.792731761932373,
      "step": 8095
    },
    {
      "epoch": 1.0594095035577002,
      "grad_norm": 7.415423393249512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.045970618724823,
      "logits/rejected": -0.1931094527244568,
      "logps/chosen": -472.74176025390625,
      "logps/rejected": -446.0352478027344,
      "loss": 0.5148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1549680531024933,
      "rewards/margins": 1.2417049407958984,
      "rewards/rejected": -1.0867369174957275,
      "step": 8096
    },
    {
      "epoch": 1.059540361495052,
      "grad_norm": 10.717713356018066,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11156593263149261,
      "logits/rejected": 0.21668758988380432,
      "logps/chosen": -447.93341064453125,
      "logps/rejected": -442.4827880859375,
      "loss": 0.8041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.025947745889425278,
      "rewards/margins": 0.4981662333011627,
      "rewards/rejected": -0.47221848368644714,
      "step": 8097
    },
    {
      "epoch": 1.0596712194324036,
      "grad_norm": 6.864725589752197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09541548043489456,
      "logits/rejected": -0.20505978167057037,
      "logps/chosen": -472.3202819824219,
      "logps/rejected": -388.8291015625,
      "loss": 0.3513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.034723907709121704,
      "rewards/margins": 1.5809656381607056,
      "rewards/rejected": -1.6156895160675049,
      "step": 8098
    },
    {
      "epoch": 1.0598020773697554,
      "grad_norm": 3.6683456897735596,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1091490387916565,
      "logits/rejected": -0.27676939964294434,
      "logps/chosen": -701.95556640625,
      "logps/rejected": -454.4115295410156,
      "loss": 0.1628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9397072196006775,
      "rewards/margins": 2.313072443008423,
      "rewards/rejected": -1.3733655214309692,
      "step": 8099
    },
    {
      "epoch": 1.0599329353071072,
      "grad_norm": 6.46639347076416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13254809379577637,
      "logits/rejected": -0.07695883512496948,
      "logps/chosen": -477.73431396484375,
      "logps/rejected": -427.844482421875,
      "loss": 0.282,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4134542644023895,
      "rewards/margins": 2.121551275253296,
      "rewards/rejected": -1.7080968618392944,
      "step": 8100
    },
    {
      "epoch": 1.060063793244459,
      "grad_norm": 11.190030097961426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18115049600601196,
      "logits/rejected": 0.12933599948883057,
      "logps/chosen": -647.3530883789062,
      "logps/rejected": -569.0653686523438,
      "loss": 0.8197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05968398600816727,
      "rewards/margins": 0.8939254879951477,
      "rewards/rejected": -0.8342414498329163,
      "step": 8101
    },
    {
      "epoch": 1.0601946511818108,
      "grad_norm": 2.7274041175842285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3464387357234955,
      "logits/rejected": 0.2486579418182373,
      "logps/chosen": -585.45361328125,
      "logps/rejected": -454.1118469238281,
      "loss": 0.1432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6861745119094849,
      "rewards/margins": 2.094860315322876,
      "rewards/rejected": -1.4086859226226807,
      "step": 8102
    },
    {
      "epoch": 1.0603255091191626,
      "grad_norm": 6.879090309143066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3294856548309326,
      "logits/rejected": -0.17205393314361572,
      "logps/chosen": -479.47979736328125,
      "logps/rejected": -394.2674560546875,
      "loss": 0.4896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.031957611441612244,
      "rewards/margins": 0.8989810943603516,
      "rewards/rejected": -0.930938720703125,
      "step": 8103
    },
    {
      "epoch": 1.0604563670565144,
      "grad_norm": 8.936750411987305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08160679787397385,
      "logits/rejected": 0.03366623818874359,
      "logps/chosen": -354.4971923828125,
      "logps/rejected": -485.9801940917969,
      "loss": 0.5796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2855531573295593,
      "rewards/margins": 0.7171931266784668,
      "rewards/rejected": -1.0027462244033813,
      "step": 8104
    },
    {
      "epoch": 1.060587224993866,
      "grad_norm": 5.056279182434082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02557569183409214,
      "logits/rejected": -0.12061462551355362,
      "logps/chosen": -448.11004638671875,
      "logps/rejected": -408.9495544433594,
      "loss": 0.2765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03125438094139099,
      "rewards/margins": 1.8872761726379395,
      "rewards/rejected": -1.9185304641723633,
      "step": 8105
    },
    {
      "epoch": 1.0607180829312177,
      "grad_norm": 6.222844123840332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2402823567390442,
      "logits/rejected": -0.134848952293396,
      "logps/chosen": -439.8944091796875,
      "logps/rejected": -480.3573303222656,
      "loss": 0.404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007312774658203125,
      "rewards/margins": 1.2686617374420166,
      "rewards/rejected": -1.2613489627838135,
      "step": 8106
    },
    {
      "epoch": 1.0608489408685695,
      "grad_norm": 10.397087097167969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15094304084777832,
      "logits/rejected": -0.18447405099868774,
      "logps/chosen": -381.4181823730469,
      "logps/rejected": -353.2254638671875,
      "loss": 0.7832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5424745678901672,
      "rewards/margins": 0.25008416175842285,
      "rewards/rejected": -0.7925586700439453,
      "step": 8107
    },
    {
      "epoch": 1.0609797988059213,
      "grad_norm": 6.927640438079834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0752641037106514,
      "logits/rejected": -0.14369440078735352,
      "logps/chosen": -420.2435302734375,
      "logps/rejected": -362.7892761230469,
      "loss": 0.5764,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022809401154518127,
      "rewards/margins": 0.803685188293457,
      "rewards/rejected": -0.7808758020401001,
      "step": 8108
    },
    {
      "epoch": 1.061110656743273,
      "grad_norm": 6.572815418243408,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23466238379478455,
      "logits/rejected": 0.10437532514333725,
      "logps/chosen": -466.11639404296875,
      "logps/rejected": -516.93701171875,
      "loss": 0.3427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33420103788375854,
      "rewards/margins": 1.5300639867782593,
      "rewards/rejected": -1.1958630084991455,
      "step": 8109
    },
    {
      "epoch": 1.0612415146806249,
      "grad_norm": 5.079742431640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13217037916183472,
      "logits/rejected": -0.14388935267925262,
      "logps/chosen": -647.80810546875,
      "logps/rejected": -648.3698120117188,
      "loss": 0.1881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16238920390605927,
      "rewards/margins": 2.613825798034668,
      "rewards/rejected": -2.776214838027954,
      "step": 8110
    },
    {
      "epoch": 1.0613723726179767,
      "grad_norm": 3.5562691688537598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16250020265579224,
      "logits/rejected": 0.20396870374679565,
      "logps/chosen": -497.2886962890625,
      "logps/rejected": -427.63671875,
      "loss": 0.1429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.721468448638916,
      "rewards/margins": 2.1773829460144043,
      "rewards/rejected": -1.4559142589569092,
      "step": 8111
    },
    {
      "epoch": 1.0615032305553285,
      "grad_norm": 7.769159317016602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0959700345993042,
      "logits/rejected": 0.06587827950716019,
      "logps/chosen": -587.7313232421875,
      "logps/rejected": -532.5455322265625,
      "loss": 0.392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11237454414367676,
      "rewards/margins": 1.1627931594848633,
      "rewards/rejected": -1.27516770362854,
      "step": 8112
    },
    {
      "epoch": 1.06163408849268,
      "grad_norm": 19.484703063964844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3369852304458618,
      "logits/rejected": -0.19772738218307495,
      "logps/chosen": -621.8239135742188,
      "logps/rejected": -369.1256103515625,
      "loss": 1.2808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2903728485107422,
      "rewards/margins": -0.38897478580474854,
      "rewards/rejected": 0.09860191494226456,
      "step": 8113
    },
    {
      "epoch": 1.0617649464300318,
      "grad_norm": 6.1438117027282715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04021775722503662,
      "logits/rejected": 0.06649930775165558,
      "logps/chosen": -458.40814208984375,
      "logps/rejected": -497.4532470703125,
      "loss": 0.3224,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16843698918819427,
      "rewards/margins": 1.614703893661499,
      "rewards/rejected": -1.783140778541565,
      "step": 8114
    },
    {
      "epoch": 1.0618958043673836,
      "grad_norm": 4.639324188232422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011930901557207108,
      "logits/rejected": 0.030048467218875885,
      "logps/chosen": -462.6825256347656,
      "logps/rejected": -391.852294921875,
      "loss": 0.2802,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5724117755889893,
      "rewards/margins": 1.285659909248352,
      "rewards/rejected": -0.713248074054718,
      "step": 8115
    },
    {
      "epoch": 1.0620266623047354,
      "grad_norm": 10.123489379882812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.128396138548851,
      "logits/rejected": -0.07484492659568787,
      "logps/chosen": -383.00933837890625,
      "logps/rejected": -484.0177001953125,
      "loss": 0.7324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.31282657384872437,
      "rewards/margins": 0.4745986759662628,
      "rewards/rejected": -0.7874252796173096,
      "step": 8116
    },
    {
      "epoch": 1.0621575202420872,
      "grad_norm": 7.505177974700928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24658769369125366,
      "logits/rejected": -0.10593188554048538,
      "logps/chosen": -450.8049011230469,
      "logps/rejected": -415.2693786621094,
      "loss": 0.5015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05509074032306671,
      "rewards/margins": 0.8895851373672485,
      "rewards/rejected": -0.9446758031845093,
      "step": 8117
    },
    {
      "epoch": 1.062288378179439,
      "grad_norm": 6.118770599365234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1854720264673233,
      "logits/rejected": 0.06239945441484451,
      "logps/chosen": -712.5101928710938,
      "logps/rejected": -590.076171875,
      "loss": 0.2966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.027495957911014557,
      "rewards/margins": 1.5744096040725708,
      "rewards/rejected": -1.6019055843353271,
      "step": 8118
    },
    {
      "epoch": 1.0624192361167908,
      "grad_norm": 13.107686996459961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029672540724277496,
      "logits/rejected": 0.148075670003891,
      "logps/chosen": -439.9222106933594,
      "logps/rejected": -536.6580200195312,
      "loss": 0.7902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6627119183540344,
      "rewards/margins": 0.462768018245697,
      "rewards/rejected": -1.1254799365997314,
      "step": 8119
    },
    {
      "epoch": 1.0625500940541426,
      "grad_norm": 5.811021327972412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026896752417087555,
      "logits/rejected": 0.12489195913076401,
      "logps/chosen": -611.6339721679688,
      "logps/rejected": -444.76129150390625,
      "loss": 0.326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5105419158935547,
      "rewards/margins": 1.6217193603515625,
      "rewards/rejected": -2.132261276245117,
      "step": 8120
    },
    {
      "epoch": 1.0626809519914941,
      "grad_norm": 5.600641250610352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33088406920433044,
      "logits/rejected": -0.38820868730545044,
      "logps/chosen": -487.51702880859375,
      "logps/rejected": -394.5501403808594,
      "loss": 0.2681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6844831705093384,
      "rewards/margins": 1.7123394012451172,
      "rewards/rejected": -1.0278559923171997,
      "step": 8121
    },
    {
      "epoch": 1.062811809928846,
      "grad_norm": 8.040436744689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1726481020450592,
      "logits/rejected": -0.136109858751297,
      "logps/chosen": -489.02349853515625,
      "logps/rejected": -478.95672607421875,
      "loss": 0.5658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.593970537185669,
      "rewards/margins": 1.043853759765625,
      "rewards/rejected": -1.637824296951294,
      "step": 8122
    },
    {
      "epoch": 1.0629426678661977,
      "grad_norm": 5.29371452331543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030705034732818604,
      "logits/rejected": -0.23647403717041016,
      "logps/chosen": -412.47796630859375,
      "logps/rejected": -457.48297119140625,
      "loss": 0.2989,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.296103298664093,
      "rewards/margins": 1.5801385641098022,
      "rewards/rejected": -1.2840354442596436,
      "step": 8123
    },
    {
      "epoch": 1.0630735258035495,
      "grad_norm": 6.418209552764893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060172367841005325,
      "logits/rejected": -0.052678339183330536,
      "logps/chosen": -624.0017700195312,
      "logps/rejected": -530.7000732421875,
      "loss": 0.4107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07848357409238815,
      "rewards/margins": 1.8060814142227173,
      "rewards/rejected": -1.7275978326797485,
      "step": 8124
    },
    {
      "epoch": 1.0632043837409013,
      "grad_norm": 9.282937049865723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47805947065353394,
      "logits/rejected": -0.3450579047203064,
      "logps/chosen": -523.9542236328125,
      "logps/rejected": -421.5676574707031,
      "loss": 0.629,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.32778775691986084,
      "rewards/margins": 0.6691131591796875,
      "rewards/rejected": -0.3413253724575043,
      "step": 8125
    },
    {
      "epoch": 1.063335241678253,
      "grad_norm": 12.771681785583496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14920514822006226,
      "logits/rejected": 0.21146807074546814,
      "logps/chosen": -542.2259521484375,
      "logps/rejected": -572.8477172851562,
      "loss": 1.0118,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45771175622940063,
      "rewards/margins": 0.20106855034828186,
      "rewards/rejected": -0.6587803363800049,
      "step": 8126
    },
    {
      "epoch": 1.063466099615605,
      "grad_norm": 5.937736511230469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004929515067487955,
      "logits/rejected": -0.11689262092113495,
      "logps/chosen": -349.3359375,
      "logps/rejected": -423.12249755859375,
      "loss": 0.4399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1688276529312134,
      "rewards/margins": 1.2308149337768555,
      "rewards/rejected": -2.3996427059173584,
      "step": 8127
    },
    {
      "epoch": 1.0635969575529565,
      "grad_norm": 4.183967590332031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2232832908630371,
      "logits/rejected": -0.3068484663963318,
      "logps/chosen": -692.0576782226562,
      "logps/rejected": -500.3714904785156,
      "loss": 0.1828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9165100455284119,
      "rewards/margins": 2.08809232711792,
      "rewards/rejected": -1.1715823411941528,
      "step": 8128
    },
    {
      "epoch": 1.0637278154903083,
      "grad_norm": 5.288680076599121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012516912072896957,
      "logits/rejected": 0.04354572296142578,
      "logps/chosen": -628.0222778320312,
      "logps/rejected": -535.7951049804688,
      "loss": 0.4021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.442217081785202,
      "rewards/margins": 1.1697369813919067,
      "rewards/rejected": -0.7275199890136719,
      "step": 8129
    },
    {
      "epoch": 1.06385867342766,
      "grad_norm": 11.176068305969238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34769392013549805,
      "logits/rejected": -0.006785847246646881,
      "logps/chosen": -495.0761413574219,
      "logps/rejected": -477.97064208984375,
      "loss": 0.6381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5205211639404297,
      "rewards/margins": 1.0962460041046143,
      "rewards/rejected": -1.6167670488357544,
      "step": 8130
    },
    {
      "epoch": 1.0639895313650118,
      "grad_norm": 6.222158908843994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29910922050476074,
      "logits/rejected": -0.21766559779644012,
      "logps/chosen": -360.7779235839844,
      "logps/rejected": -359.2353515625,
      "loss": 0.4132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5305332541465759,
      "rewards/margins": 1.0688161849975586,
      "rewards/rejected": -1.5993494987487793,
      "step": 8131
    },
    {
      "epoch": 1.0641203893023636,
      "grad_norm": 4.756670951843262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021136965602636337,
      "logits/rejected": -0.11417866498231888,
      "logps/chosen": -726.2010498046875,
      "logps/rejected": -486.03131103515625,
      "loss": 0.188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2937201261520386,
      "rewards/margins": 2.069279193878174,
      "rewards/rejected": -2.362999200820923,
      "step": 8132
    },
    {
      "epoch": 1.0642512472397154,
      "grad_norm": 9.074151039123535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43396317958831787,
      "logits/rejected": -0.40486735105514526,
      "logps/chosen": -500.06707763671875,
      "logps/rejected": -442.209716796875,
      "loss": 0.5239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.013219058513641357,
      "rewards/margins": 0.8958625197410583,
      "rewards/rejected": -0.9090816378593445,
      "step": 8133
    },
    {
      "epoch": 1.0643821051770672,
      "grad_norm": 2.975484609603882,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009812467731535435,
      "logits/rejected": -0.2843218445777893,
      "logps/chosen": -517.6077270507812,
      "logps/rejected": -351.2613525390625,
      "loss": 0.2279,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.308933824300766,
      "rewards/margins": 1.82639479637146,
      "rewards/rejected": -2.135328769683838,
      "step": 8134
    },
    {
      "epoch": 1.064512963114419,
      "grad_norm": 6.697392463684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3759608864784241,
      "logits/rejected": -0.3922128975391388,
      "logps/chosen": -261.8507995605469,
      "logps/rejected": -265.05511474609375,
      "loss": 0.5137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1625609397888184,
      "rewards/margins": 1.1424658298492432,
      "rewards/rejected": -2.3050267696380615,
      "step": 8135
    },
    {
      "epoch": 1.0646438210517706,
      "grad_norm": 8.810283660888672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11601124703884125,
      "logits/rejected": 0.003557693213224411,
      "logps/chosen": -583.1954956054688,
      "logps/rejected": -648.9136962890625,
      "loss": 0.4853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07923849672079086,
      "rewards/margins": 0.868880033493042,
      "rewards/rejected": -0.9481186270713806,
      "step": 8136
    },
    {
      "epoch": 1.0647746789891224,
      "grad_norm": 25.09295654296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021388402208685875,
      "logits/rejected": -0.03727426379919052,
      "logps/chosen": -470.6114196777344,
      "logps/rejected": -512.1301879882812,
      "loss": 0.6173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5967662334442139,
      "rewards/margins": 0.42218634486198425,
      "rewards/rejected": -1.018952488899231,
      "step": 8137
    },
    {
      "epoch": 1.0649055369264742,
      "grad_norm": 7.870599746704102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1796623170375824,
      "logits/rejected": -0.11561084538698196,
      "logps/chosen": -535.1032104492188,
      "logps/rejected": -542.1162719726562,
      "loss": 0.5087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2649037837982178,
      "rewards/margins": 0.8873482346534729,
      "rewards/rejected": -2.152252197265625,
      "step": 8138
    },
    {
      "epoch": 1.065036394863826,
      "grad_norm": 14.139200210571289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3047237694263458,
      "logits/rejected": -0.3201092779636383,
      "logps/chosen": -336.8943786621094,
      "logps/rejected": -429.6649475097656,
      "loss": 0.6946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3526450395584106,
      "rewards/margins": 0.7481116652488708,
      "rewards/rejected": -2.1007566452026367,
      "step": 8139
    },
    {
      "epoch": 1.0651672528011777,
      "grad_norm": 6.41041898727417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18579672276973724,
      "logits/rejected": -0.18361040949821472,
      "logps/chosen": -312.58380126953125,
      "logps/rejected": -345.4050598144531,
      "loss": 0.2865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1657482087612152,
      "rewards/margins": 3.038236618041992,
      "rewards/rejected": -2.872488498687744,
      "step": 8140
    },
    {
      "epoch": 1.0652981107385295,
      "grad_norm": 10.581038475036621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29282745718955994,
      "logits/rejected": -0.13243626058101654,
      "logps/chosen": -230.38998413085938,
      "logps/rejected": -261.8997497558594,
      "loss": 0.8948,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3064796924591064,
      "rewards/margins": 0.1455150842666626,
      "rewards/rejected": -1.451994776725769,
      "step": 8141
    },
    {
      "epoch": 1.0654289686758813,
      "grad_norm": 6.3674139976501465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08330090343952179,
      "logits/rejected": -0.09597387164831161,
      "logps/chosen": -467.0658874511719,
      "logps/rejected": -511.56298828125,
      "loss": 0.4433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5566413998603821,
      "rewards/margins": 0.8748580813407898,
      "rewards/rejected": -1.4314996004104614,
      "step": 8142
    },
    {
      "epoch": 1.065559826613233,
      "grad_norm": 8.183637619018555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06793250143527985,
      "logits/rejected": 0.14574559032917023,
      "logps/chosen": -308.1624755859375,
      "logps/rejected": -342.40960693359375,
      "loss": 0.5264,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.047775086015462875,
      "rewards/margins": 0.7855631113052368,
      "rewards/rejected": -0.8333381414413452,
      "step": 8143
    },
    {
      "epoch": 1.0656906845505847,
      "grad_norm": 8.300874710083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03274018317461014,
      "logits/rejected": -0.11447837948799133,
      "logps/chosen": -610.9564208984375,
      "logps/rejected": -538.56396484375,
      "loss": 0.4827,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06714703142642975,
      "rewards/margins": 1.178354263305664,
      "rewards/rejected": -1.1112072467803955,
      "step": 8144
    },
    {
      "epoch": 1.0658215424879365,
      "grad_norm": 6.441849708557129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22724099457263947,
      "logits/rejected": -0.08770635724067688,
      "logps/chosen": -406.977783203125,
      "logps/rejected": -446.66192626953125,
      "loss": 0.3115,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.029204178601503372,
      "rewards/margins": 1.9164047241210938,
      "rewards/rejected": -1.9456087350845337,
      "step": 8145
    },
    {
      "epoch": 1.0659524004252883,
      "grad_norm": 5.665526390075684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024948397651314735,
      "logits/rejected": -0.09582482278347015,
      "logps/chosen": -540.5664672851562,
      "logps/rejected": -472.242919921875,
      "loss": 0.3514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9041388034820557,
      "rewards/margins": 2.4237265586853027,
      "rewards/rejected": -1.5195876359939575,
      "step": 8146
    },
    {
      "epoch": 1.06608325836264,
      "grad_norm": 9.1719970703125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1876002997159958,
      "logits/rejected": -0.10474789142608643,
      "logps/chosen": -293.46539306640625,
      "logps/rejected": -385.200927734375,
      "loss": 0.8092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8138704299926758,
      "rewards/margins": 0.5415889620780945,
      "rewards/rejected": -1.3554593324661255,
      "step": 8147
    },
    {
      "epoch": 1.0662141162999919,
      "grad_norm": 4.947286605834961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2144831418991089,
      "logits/rejected": -0.003617292270064354,
      "logps/chosen": -414.18658447265625,
      "logps/rejected": -405.3543701171875,
      "loss": 0.2605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6463983058929443,
      "rewards/margins": 1.3883450031280518,
      "rewards/rejected": -2.034743309020996,
      "step": 8148
    },
    {
      "epoch": 1.0663449742373436,
      "grad_norm": 5.5268354415893555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24539002776145935,
      "logits/rejected": -0.09895575791597366,
      "logps/chosen": -473.55279541015625,
      "logps/rejected": -715.0602416992188,
      "loss": 0.3552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5953361988067627,
      "rewards/margins": 0.9729120135307312,
      "rewards/rejected": -1.5682482719421387,
      "step": 8149
    },
    {
      "epoch": 1.0664758321746954,
      "grad_norm": 8.843514442443848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22899362444877625,
      "logits/rejected": 0.1511203944683075,
      "logps/chosen": -540.7310180664062,
      "logps/rejected": -571.4319458007812,
      "loss": 0.4995,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5731322169303894,
      "rewards/margins": 1.0945212841033936,
      "rewards/rejected": -1.6676535606384277,
      "step": 8150
    },
    {
      "epoch": 1.066606690112047,
      "grad_norm": 4.013608455657959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08687649667263031,
      "logits/rejected": -0.018286891281604767,
      "logps/chosen": -633.7523193359375,
      "logps/rejected": -526.1690673828125,
      "loss": 0.2366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6803123950958252,
      "rewards/margins": 1.4948618412017822,
      "rewards/rejected": -0.814549446105957,
      "step": 8151
    },
    {
      "epoch": 1.0667375480493988,
      "grad_norm": 16.52344512939453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21146643161773682,
      "logits/rejected": 0.12154823541641235,
      "logps/chosen": -596.8394775390625,
      "logps/rejected": -597.08642578125,
      "loss": 1.1476,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6860953569412231,
      "rewards/margins": 0.5996721982955933,
      "rewards/rejected": -2.2857675552368164,
      "step": 8152
    },
    {
      "epoch": 1.0668684059867506,
      "grad_norm": 4.668502330780029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08589291572570801,
      "logits/rejected": 0.09926407039165497,
      "logps/chosen": -624.0609130859375,
      "logps/rejected": -544.8948974609375,
      "loss": 0.2201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5095596313476562,
      "rewards/margins": 2.0765480995178223,
      "rewards/rejected": -2.5861077308654785,
      "step": 8153
    },
    {
      "epoch": 1.0669992639241024,
      "grad_norm": 6.799182415008545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15759018063545227,
      "logits/rejected": -0.22824926674365997,
      "logps/chosen": -360.1416931152344,
      "logps/rejected": -395.1912841796875,
      "loss": 0.5138,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1090526580810547,
      "rewards/margins": 0.8993785381317139,
      "rewards/rejected": -3.0084311962127686,
      "step": 8154
    },
    {
      "epoch": 1.0671301218614542,
      "grad_norm": 7.914076328277588,
      "learning_rate": 5e-06,
      "logits/chosen": -0.293221652507782,
      "logits/rejected": -0.1808982491493225,
      "logps/chosen": -332.5389404296875,
      "logps/rejected": -396.4370422363281,
      "loss": 0.4694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0975791215896606,
      "rewards/margins": 0.8038400411605835,
      "rewards/rejected": -1.9014192819595337,
      "step": 8155
    },
    {
      "epoch": 1.067260979798806,
      "grad_norm": 7.519237041473389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14984357357025146,
      "logits/rejected": -0.15841494500637054,
      "logps/chosen": -517.701416015625,
      "logps/rejected": -480.2591552734375,
      "loss": 0.3742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4934206008911133,
      "rewards/margins": 1.3237931728363037,
      "rewards/rejected": -1.8172138929367065,
      "step": 8156
    },
    {
      "epoch": 1.0673918377361578,
      "grad_norm": 6.698535442352295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013797976076602936,
      "logits/rejected": -0.262208491563797,
      "logps/chosen": -396.01434326171875,
      "logps/rejected": -399.799072265625,
      "loss": 0.3823,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3185032904148102,
      "rewards/margins": 1.575718641281128,
      "rewards/rejected": -1.2572152614593506,
      "step": 8157
    },
    {
      "epoch": 1.0675226956735095,
      "grad_norm": 5.909481525421143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22067132592201233,
      "logits/rejected": -0.07441128045320511,
      "logps/chosen": -471.46856689453125,
      "logps/rejected": -423.8011169433594,
      "loss": 0.3805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2030478715896606,
      "rewards/margins": 1.137776255607605,
      "rewards/rejected": -2.3408238887786865,
      "step": 8158
    },
    {
      "epoch": 1.0676535536108611,
      "grad_norm": 9.331134796142578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1443295180797577,
      "logits/rejected": -0.08945783972740173,
      "logps/chosen": -357.1119079589844,
      "logps/rejected": -367.9081726074219,
      "loss": 0.7428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9389195442199707,
      "rewards/margins": 0.19933459162712097,
      "rewards/rejected": -2.138254165649414,
      "step": 8159
    },
    {
      "epoch": 1.067784411548213,
      "grad_norm": 6.9511494636535645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06017765775322914,
      "logits/rejected": -0.013688217848539352,
      "logps/chosen": -469.6340637207031,
      "logps/rejected": -525.976318359375,
      "loss": 0.4439,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.452998548746109,
      "rewards/margins": 1.1209614276885986,
      "rewards/rejected": -1.5739598274230957,
      "step": 8160
    },
    {
      "epoch": 1.0679152694855647,
      "grad_norm": 8.48622989654541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08482104539871216,
      "logits/rejected": -0.11859256029129028,
      "logps/chosen": -616.0553588867188,
      "logps/rejected": -656.6324462890625,
      "loss": 0.4842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7967159152030945,
      "rewards/margins": 1.4373643398284912,
      "rewards/rejected": -2.2340803146362305,
      "step": 8161
    },
    {
      "epoch": 1.0680461274229165,
      "grad_norm": 5.936823844909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007157556712627411,
      "logits/rejected": -0.3571755886077881,
      "logps/chosen": -485.021728515625,
      "logps/rejected": -530.5747680664062,
      "loss": 0.2977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3548799753189087,
      "rewards/margins": 1.5676318407058716,
      "rewards/rejected": -1.9225119352340698,
      "step": 8162
    },
    {
      "epoch": 1.0681769853602683,
      "grad_norm": 10.380600929260254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03229055553674698,
      "logits/rejected": -0.021575788035988808,
      "logps/chosen": -392.7222900390625,
      "logps/rejected": -381.97491455078125,
      "loss": 0.632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2870087623596191,
      "rewards/margins": 0.5649243593215942,
      "rewards/rejected": -1.8519331216812134,
      "step": 8163
    },
    {
      "epoch": 1.06830784329762,
      "grad_norm": 7.132689952850342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15054668486118317,
      "logits/rejected": -0.2071475088596344,
      "logps/chosen": -431.00225830078125,
      "logps/rejected": -405.326904296875,
      "loss": 0.4301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4667820930480957,
      "rewards/margins": 0.9280835390090942,
      "rewards/rejected": -2.3948657512664795,
      "step": 8164
    },
    {
      "epoch": 1.0684387012349719,
      "grad_norm": 5.526618480682373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21763870120048523,
      "logits/rejected": -0.16146764159202576,
      "logps/chosen": -380.689453125,
      "logps/rejected": -466.8492431640625,
      "loss": 0.2777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0358121395111084,
      "rewards/margins": 2.771120071411133,
      "rewards/rejected": -3.806932210922241,
      "step": 8165
    },
    {
      "epoch": 1.0685695591723237,
      "grad_norm": 6.8715996742248535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05213311314582825,
      "logits/rejected": 0.09049420058727264,
      "logps/chosen": -480.4278564453125,
      "logps/rejected": -474.315185546875,
      "loss": 0.5452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6374891996383667,
      "rewards/margins": 1.2828178405761719,
      "rewards/rejected": -1.9203071594238281,
      "step": 8166
    },
    {
      "epoch": 1.0687004171096752,
      "grad_norm": 6.040695667266846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08851297199726105,
      "logits/rejected": -0.033584389835596085,
      "logps/chosen": -414.021728515625,
      "logps/rejected": -425.7380065917969,
      "loss": 0.3415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9201012849807739,
      "rewards/margins": 1.1366299390792847,
      "rewards/rejected": -2.0567312240600586,
      "step": 8167
    },
    {
      "epoch": 1.068831275047027,
      "grad_norm": 7.348216533660889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4066913425922394,
      "logits/rejected": -0.11921210587024689,
      "logps/chosen": -640.1412963867188,
      "logps/rejected": -615.8616333007812,
      "loss": 0.3118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.186513066291809,
      "rewards/margins": 1.4809385538101196,
      "rewards/rejected": -2.6674513816833496,
      "step": 8168
    },
    {
      "epoch": 1.0689621329843788,
      "grad_norm": 11.005459785461426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40184295177459717,
      "logits/rejected": -0.4412336051464081,
      "logps/chosen": -395.32171630859375,
      "logps/rejected": -431.6096496582031,
      "loss": 0.6245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9634132981300354,
      "rewards/margins": 0.6269963979721069,
      "rewards/rejected": -1.5904096364974976,
      "step": 8169
    },
    {
      "epoch": 1.0690929909217306,
      "grad_norm": 9.468427658081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31328457593917847,
      "logits/rejected": -0.2218325436115265,
      "logps/chosen": -399.36016845703125,
      "logps/rejected": -389.7656555175781,
      "loss": 0.6311,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.255763292312622,
      "rewards/margins": 0.6735714673995972,
      "rewards/rejected": -1.9293346405029297,
      "step": 8170
    },
    {
      "epoch": 1.0692238488590824,
      "grad_norm": 8.65765380859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11608922481536865,
      "logits/rejected": -0.12109176069498062,
      "logps/chosen": -395.6585693359375,
      "logps/rejected": -377.086669921875,
      "loss": 0.4314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4093654155731201,
      "rewards/margins": 1.4762169122695923,
      "rewards/rejected": -2.885582208633423,
      "step": 8171
    },
    {
      "epoch": 1.0693547067964342,
      "grad_norm": 8.715254783630371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0015013557858765125,
      "logits/rejected": -0.18990087509155273,
      "logps/chosen": -374.4226379394531,
      "logps/rejected": -442.9077453613281,
      "loss": 0.6975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4735890626907349,
      "rewards/margins": 0.5407040119171143,
      "rewards/rejected": -2.0142931938171387,
      "step": 8172
    },
    {
      "epoch": 1.069485564733786,
      "grad_norm": 11.677071571350098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1521364003419876,
      "logits/rejected": -0.09649954736232758,
      "logps/chosen": -516.634765625,
      "logps/rejected": -469.457763671875,
      "loss": 1.0793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.28839075565338135,
      "rewards/margins": 0.5768489837646484,
      "rewards/rejected": -0.28845828771591187,
      "step": 8173
    },
    {
      "epoch": 1.0696164226711375,
      "grad_norm": 4.985620021820068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15840628743171692,
      "logits/rejected": -0.26946139335632324,
      "logps/chosen": -293.2002258300781,
      "logps/rejected": -372.87457275390625,
      "loss": 0.2946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1727091073989868,
      "rewards/margins": 1.4782606363296509,
      "rewards/rejected": -2.6509697437286377,
      "step": 8174
    },
    {
      "epoch": 1.0697472806084893,
      "grad_norm": 4.27156400680542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14690786600112915,
      "logits/rejected": 0.023451481014490128,
      "logps/chosen": -618.8746337890625,
      "logps/rejected": -608.5006713867188,
      "loss": 0.249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5424487590789795,
      "rewards/margins": 2.1351022720336914,
      "rewards/rejected": -2.677551031112671,
      "step": 8175
    },
    {
      "epoch": 1.0698781385458411,
      "grad_norm": 3.5701828002929688,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1250317394733429,
      "logits/rejected": -0.16804704070091248,
      "logps/chosen": -573.0795288085938,
      "logps/rejected": -582.5433959960938,
      "loss": 0.1915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7757545709609985,
      "rewards/margins": 2.313887119293213,
      "rewards/rejected": -3.089641571044922,
      "step": 8176
    },
    {
      "epoch": 1.070008996483193,
      "grad_norm": 6.750437259674072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12387610971927643,
      "logits/rejected": -0.12364616245031357,
      "logps/chosen": -542.6527709960938,
      "logps/rejected": -451.2022705078125,
      "loss": 0.3489,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6274299621582031,
      "rewards/margins": 1.2270504236221313,
      "rewards/rejected": -1.8544803857803345,
      "step": 8177
    },
    {
      "epoch": 1.0701398544205447,
      "grad_norm": 4.046553611755371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013009650632739067,
      "logits/rejected": 0.030555665493011475,
      "logps/chosen": -397.968505859375,
      "logps/rejected": -433.516357421875,
      "loss": 0.2911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8083592057228088,
      "rewards/margins": 1.5896934270858765,
      "rewards/rejected": -2.39805269241333,
      "step": 8178
    },
    {
      "epoch": 1.0702707123578965,
      "grad_norm": 10.719795227050781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0021998025476932526,
      "logits/rejected": -0.02564743533730507,
      "logps/chosen": -526.5335693359375,
      "logps/rejected": -543.4144287109375,
      "loss": 0.6782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.045639991760254,
      "rewards/margins": 0.739517331123352,
      "rewards/rejected": -1.7851572036743164,
      "step": 8179
    },
    {
      "epoch": 1.0704015702952483,
      "grad_norm": 7.044010162353516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12487207353115082,
      "logits/rejected": -0.25629472732543945,
      "logps/chosen": -430.32977294921875,
      "logps/rejected": -378.57568359375,
      "loss": 0.5347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8939113616943359,
      "rewards/margins": 0.7891556620597839,
      "rewards/rejected": -1.683066964149475,
      "step": 8180
    },
    {
      "epoch": 1.0705324282326,
      "grad_norm": 5.757213115692139,
      "learning_rate": 5e-06,
      "logits/chosen": -0.103949636220932,
      "logits/rejected": -0.2602766752243042,
      "logps/chosen": -384.8692626953125,
      "logps/rejected": -294.544677734375,
      "loss": 0.4799,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2774289846420288,
      "rewards/margins": 1.0400420427322388,
      "rewards/rejected": -1.3174710273742676,
      "step": 8181
    },
    {
      "epoch": 1.0706632861699517,
      "grad_norm": 9.26860237121582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26708197593688965,
      "logits/rejected": 0.07115761935710907,
      "logps/chosen": -524.4205322265625,
      "logps/rejected": -423.40966796875,
      "loss": 0.5726,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0197525024414062,
      "rewards/margins": 0.7661736011505127,
      "rewards/rejected": -1.785926103591919,
      "step": 8182
    },
    {
      "epoch": 1.0707941441073034,
      "grad_norm": 7.187891483306885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1732272356748581,
      "logits/rejected": 0.010062430053949356,
      "logps/chosen": -464.06036376953125,
      "logps/rejected": -452.2890319824219,
      "loss": 0.3295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15874652564525604,
      "rewards/margins": 1.1950716972351074,
      "rewards/rejected": -1.3538181781768799,
      "step": 8183
    },
    {
      "epoch": 1.0709250020446552,
      "grad_norm": 10.813457489013672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04518839716911316,
      "logits/rejected": -0.09254185855388641,
      "logps/chosen": -435.3721008300781,
      "logps/rejected": -394.3233947753906,
      "loss": 0.3892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29273301362991333,
      "rewards/margins": 1.8374944925308228,
      "rewards/rejected": -2.130227565765381,
      "step": 8184
    },
    {
      "epoch": 1.071055859982007,
      "grad_norm": 2.8896148204803467,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2482268065214157,
      "logits/rejected": 0.03339027240872383,
      "logps/chosen": -455.478271484375,
      "logps/rejected": -469.5760498046875,
      "loss": 0.2033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8459317088127136,
      "rewards/margins": 2.2052435874938965,
      "rewards/rejected": -3.051175355911255,
      "step": 8185
    },
    {
      "epoch": 1.0711867179193588,
      "grad_norm": 3.927896738052368,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2256554365158081,
      "logits/rejected": -0.29890602827072144,
      "logps/chosen": -327.5128479003906,
      "logps/rejected": -400.8352966308594,
      "loss": 0.2761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45371878147125244,
      "rewards/margins": 1.4290677309036255,
      "rewards/rejected": -1.882786512374878,
      "step": 8186
    },
    {
      "epoch": 1.0713175758567106,
      "grad_norm": 8.269952774047852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43376290798187256,
      "logits/rejected": -0.4112814962863922,
      "logps/chosen": -374.3557434082031,
      "logps/rejected": -325.69171142578125,
      "loss": 0.5418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3362900018692017,
      "rewards/margins": 0.8549093008041382,
      "rewards/rejected": -2.19119930267334,
      "step": 8187
    },
    {
      "epoch": 1.0714484337940624,
      "grad_norm": 10.892987251281738,
      "learning_rate": 5e-06,
      "logits/chosen": 0.061118580400943756,
      "logits/rejected": -0.038308121263980865,
      "logps/chosen": -321.21014404296875,
      "logps/rejected": -308.46685791015625,
      "loss": 0.7656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2474725246429443,
      "rewards/margins": 0.7503339052200317,
      "rewards/rejected": -1.997806429862976,
      "step": 8188
    },
    {
      "epoch": 1.071579291731414,
      "grad_norm": 13.45110034942627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11834774166345596,
      "logits/rejected": -0.11443545669317245,
      "logps/chosen": -500.9093322753906,
      "logps/rejected": -468.3970031738281,
      "loss": 0.8248,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0027869939804077,
      "rewards/margins": 0.5063113570213318,
      "rewards/rejected": -1.5090984106063843,
      "step": 8189
    },
    {
      "epoch": 1.0717101496687658,
      "grad_norm": 7.138100624084473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06774662435054779,
      "logits/rejected": -0.21938997507095337,
      "logps/chosen": -352.0191955566406,
      "logps/rejected": -345.0267639160156,
      "loss": 0.4096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0251946449279785,
      "rewards/margins": 1.3912193775177002,
      "rewards/rejected": -2.4164140224456787,
      "step": 8190
    },
    {
      "epoch": 1.0718410076061176,
      "grad_norm": 13.032256126403809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38442209362983704,
      "logits/rejected": -0.5909663438796997,
      "logps/chosen": -444.58892822265625,
      "logps/rejected": -382.6772155761719,
      "loss": 0.7187,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9998788833618164,
      "rewards/margins": 0.5753293633460999,
      "rewards/rejected": -1.575208306312561,
      "step": 8191
    },
    {
      "epoch": 1.0719718655434693,
      "grad_norm": 6.623802185058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07910699397325516,
      "logits/rejected": -0.1183047890663147,
      "logps/chosen": -304.4378967285156,
      "logps/rejected": -352.6976623535156,
      "loss": 0.5538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2129006385803223,
      "rewards/margins": 0.4759116768836975,
      "rewards/rejected": -1.688812255859375,
      "step": 8192
    },
    {
      "epoch": 1.0721027234808211,
      "grad_norm": 8.918176651000977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24164682626724243,
      "logits/rejected": -0.29506757855415344,
      "logps/chosen": -343.6812744140625,
      "logps/rejected": -394.0123596191406,
      "loss": 0.4872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.071587324142456,
      "rewards/margins": 0.9510475397109985,
      "rewards/rejected": -2.022634744644165,
      "step": 8193
    },
    {
      "epoch": 1.072233581418173,
      "grad_norm": 6.799073219299316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07753921300172806,
      "logits/rejected": -0.17117567360401154,
      "logps/chosen": -509.5559387207031,
      "logps/rejected": -549.8858642578125,
      "loss": 0.4728,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.004909887909889221,
      "rewards/margins": 1.7285798788070679,
      "rewards/rejected": -1.7334898710250854,
      "step": 8194
    },
    {
      "epoch": 1.0723644393555247,
      "grad_norm": 6.362125396728516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3433419167995453,
      "logits/rejected": -0.4694982171058655,
      "logps/chosen": -385.66143798828125,
      "logps/rejected": -330.7034912109375,
      "loss": 0.4086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4917735457420349,
      "rewards/margins": 1.210481882095337,
      "rewards/rejected": -1.7022554874420166,
      "step": 8195
    },
    {
      "epoch": 1.0724952972928765,
      "grad_norm": 3.271597146987915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11689309030771255,
      "logits/rejected": 0.008105956017971039,
      "logps/chosen": -603.232421875,
      "logps/rejected": -502.32110595703125,
      "loss": 0.1922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0653344988822937,
      "rewards/margins": 1.648442029953003,
      "rewards/rejected": -1.583107590675354,
      "step": 8196
    },
    {
      "epoch": 1.072626155230228,
      "grad_norm": 8.462845802307129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10531528294086456,
      "logits/rejected": -0.05299971252679825,
      "logps/chosen": -414.18817138671875,
      "logps/rejected": -497.5650939941406,
      "loss": 0.554,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8595274090766907,
      "rewards/margins": 1.0537636280059814,
      "rewards/rejected": -1.9132909774780273,
      "step": 8197
    },
    {
      "epoch": 1.0727570131675799,
      "grad_norm": 8.645286560058594,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018903691321611404,
      "logits/rejected": -0.07198292762041092,
      "logps/chosen": -503.87298583984375,
      "logps/rejected": -450.9266357421875,
      "loss": 0.5913,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3683408498764038,
      "rewards/margins": 0.8522716760635376,
      "rewards/rejected": -1.2206124067306519,
      "step": 8198
    },
    {
      "epoch": 1.0728878711049317,
      "grad_norm": 8.690207481384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15619169175624847,
      "logits/rejected": -0.15445978939533234,
      "logps/chosen": -405.5689392089844,
      "logps/rejected": -553.2587890625,
      "loss": 0.4134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45697328448295593,
      "rewards/margins": 1.269141435623169,
      "rewards/rejected": -1.7261146306991577,
      "step": 8199
    },
    {
      "epoch": 1.0730187290422835,
      "grad_norm": 4.5725321769714355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18354780972003937,
      "logits/rejected": 0.010452529415488243,
      "logps/chosen": -396.7367858886719,
      "logps/rejected": -261.3326416015625,
      "loss": 0.3059,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.42716336250305176,
      "rewards/margins": 1.8798402547836304,
      "rewards/rejected": -1.452676773071289,
      "step": 8200
    },
    {
      "epoch": 1.0731495869796353,
      "grad_norm": 8.625005722045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23416867852210999,
      "logits/rejected": -0.26387834548950195,
      "logps/chosen": -429.91790771484375,
      "logps/rejected": -481.25927734375,
      "loss": 0.4905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1608631610870361,
      "rewards/margins": 1.0042585134506226,
      "rewards/rejected": -2.165121555328369,
      "step": 8201
    },
    {
      "epoch": 1.073280444916987,
      "grad_norm": 3.6651525497436523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06727461516857147,
      "logits/rejected": 0.05516352504491806,
      "logps/chosen": -365.45623779296875,
      "logps/rejected": -378.3961181640625,
      "loss": 0.1787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5571886301040649,
      "rewards/margins": 1.9840397834777832,
      "rewards/rejected": -1.4268511533737183,
      "step": 8202
    },
    {
      "epoch": 1.0734113028543388,
      "grad_norm": 7.879477024078369,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035069502890110016,
      "logits/rejected": -0.3870451748371124,
      "logps/chosen": -758.6519775390625,
      "logps/rejected": -382.96697998046875,
      "loss": 0.4034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.020279675722122192,
      "rewards/margins": 2.2481071949005127,
      "rewards/rejected": -2.227827548980713,
      "step": 8203
    },
    {
      "epoch": 1.0735421607916906,
      "grad_norm": 6.835793972015381,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18901339173316956,
      "logits/rejected": -0.04450669884681702,
      "logps/chosen": -257.45623779296875,
      "logps/rejected": -323.8920593261719,
      "loss": 0.4576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6890932321548462,
      "rewards/margins": 1.345941424369812,
      "rewards/rejected": -2.035034656524658,
      "step": 8204
    },
    {
      "epoch": 1.0736730187290422,
      "grad_norm": 7.138750076293945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26428312063217163,
      "logits/rejected": -0.010219644755125046,
      "logps/chosen": -376.34576416015625,
      "logps/rejected": -256.61126708984375,
      "loss": 0.4719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7338459491729736,
      "rewards/margins": 0.8398696184158325,
      "rewards/rejected": -1.5737156867980957,
      "step": 8205
    },
    {
      "epoch": 1.073803876666394,
      "grad_norm": 6.850896835327148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029001392424106598,
      "logits/rejected": 0.02007666975259781,
      "logps/chosen": -347.7757568359375,
      "logps/rejected": -397.34405517578125,
      "loss": 0.4756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2318670153617859,
      "rewards/margins": 0.9680851697921753,
      "rewards/rejected": -1.199952244758606,
      "step": 8206
    },
    {
      "epoch": 1.0739347346037458,
      "grad_norm": 5.417362689971924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4130856990814209,
      "logits/rejected": -0.34821856021881104,
      "logps/chosen": -507.1918029785156,
      "logps/rejected": -496.0321960449219,
      "loss": 0.3647,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06264057755470276,
      "rewards/margins": 1.417097568511963,
      "rewards/rejected": -1.354457139968872,
      "step": 8207
    },
    {
      "epoch": 1.0740655925410976,
      "grad_norm": 6.459020614624023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27096325159072876,
      "logits/rejected": -0.15468469262123108,
      "logps/chosen": -533.5730590820312,
      "logps/rejected": -523.4983520507812,
      "loss": 0.3065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1104932576417923,
      "rewards/margins": 1.6799098253250122,
      "rewards/rejected": -1.5694166421890259,
      "step": 8208
    },
    {
      "epoch": 1.0741964504784494,
      "grad_norm": 9.514826774597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12364603579044342,
      "logits/rejected": -0.27213045954704285,
      "logps/chosen": -536.44189453125,
      "logps/rejected": -416.9721984863281,
      "loss": 0.5754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0216680765151978,
      "rewards/margins": 0.6840049624443054,
      "rewards/rejected": -1.7056729793548584,
      "step": 8209
    },
    {
      "epoch": 1.0743273084158012,
      "grad_norm": 8.82286548614502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08378688991069794,
      "logits/rejected": -0.2573322653770447,
      "logps/chosen": -524.31884765625,
      "logps/rejected": -437.5669250488281,
      "loss": 0.4404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8231313228607178,
      "rewards/margins": 1.0271990299224854,
      "rewards/rejected": -1.8503304719924927,
      "step": 8210
    },
    {
      "epoch": 1.074458166353153,
      "grad_norm": 6.756829261779785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026226934045553207,
      "logits/rejected": -0.05860068276524544,
      "logps/chosen": -578.7180786132812,
      "logps/rejected": -500.9048156738281,
      "loss": 0.4876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5290363430976868,
      "rewards/margins": 0.596867561340332,
      "rewards/rejected": -1.125903844833374,
      "step": 8211
    },
    {
      "epoch": 1.0745890242905047,
      "grad_norm": 4.640523910522461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025312747806310654,
      "logits/rejected": -0.013582288287580013,
      "logps/chosen": -825.4899291992188,
      "logps/rejected": -565.6314697265625,
      "loss": 0.3025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.010625466704368591,
      "rewards/margins": 1.9024593830108643,
      "rewards/rejected": -1.913084864616394,
      "step": 8212
    },
    {
      "epoch": 1.0747198822278563,
      "grad_norm": 6.787992000579834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18939858675003052,
      "logits/rejected": -0.30931317806243896,
      "logps/chosen": -518.4299926757812,
      "logps/rejected": -567.7283325195312,
      "loss": 0.3358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11204512417316437,
      "rewards/margins": 1.6813313961029053,
      "rewards/rejected": -1.5692863464355469,
      "step": 8213
    },
    {
      "epoch": 1.074850740165208,
      "grad_norm": 7.688416004180908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11829303205013275,
      "logits/rejected": 0.14126840233802795,
      "logps/chosen": -604.5521240234375,
      "logps/rejected": -670.6478881835938,
      "loss": 0.5104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.25890201330184937,
      "rewards/margins": 1.1609596014022827,
      "rewards/rejected": -0.9020576477050781,
      "step": 8214
    },
    {
      "epoch": 1.0749815981025599,
      "grad_norm": 6.898360729217529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.001376984640955925,
      "logits/rejected": -0.1677931845188141,
      "logps/chosen": -383.50042724609375,
      "logps/rejected": -380.95172119140625,
      "loss": 0.4727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.010942280292510986,
      "rewards/margins": 0.9524698257446289,
      "rewards/rejected": -0.9634120464324951,
      "step": 8215
    },
    {
      "epoch": 1.0751124560399117,
      "grad_norm": 3.9241249561309814,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19719167053699493,
      "logits/rejected": -0.09494472295045853,
      "logps/chosen": -336.7659912109375,
      "logps/rejected": -371.36053466796875,
      "loss": 0.2847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6192821264266968,
      "rewards/margins": 1.740938425064087,
      "rewards/rejected": -2.3602206707000732,
      "step": 8216
    },
    {
      "epoch": 1.0752433139772635,
      "grad_norm": 10.716328620910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13894309103488922,
      "logits/rejected": -0.02610904909670353,
      "logps/chosen": -559.4190673828125,
      "logps/rejected": -588.0087280273438,
      "loss": 0.636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9071496725082397,
      "rewards/margins": 0.74983811378479,
      "rewards/rejected": -1.6569880247116089,
      "step": 8217
    },
    {
      "epoch": 1.0753741719146153,
      "grad_norm": 6.267411708831787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22288134694099426,
      "logits/rejected": -0.30893674492836,
      "logps/chosen": -362.687744140625,
      "logps/rejected": -407.8570556640625,
      "loss": 0.2275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9499257206916809,
      "rewards/margins": 2.517864465713501,
      "rewards/rejected": -3.467790126800537,
      "step": 8218
    },
    {
      "epoch": 1.075505029851967,
      "grad_norm": 8.307198524475098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18062442541122437,
      "logits/rejected": 0.03478964418172836,
      "logps/chosen": -449.62298583984375,
      "logps/rejected": -335.96038818359375,
      "loss": 0.5124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6814998984336853,
      "rewards/margins": 0.6730644106864929,
      "rewards/rejected": -1.3545643091201782,
      "step": 8219
    },
    {
      "epoch": 1.0756358877893186,
      "grad_norm": 3.487792730331421,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12869645655155182,
      "logits/rejected": -0.4186621904373169,
      "logps/chosen": -247.88824462890625,
      "logps/rejected": -308.9876708984375,
      "loss": 0.1995,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6799415349960327,
      "rewards/margins": 1.9744844436645508,
      "rewards/rejected": -2.654426097869873,
      "step": 8220
    },
    {
      "epoch": 1.0757667457266704,
      "grad_norm": 7.794610977172852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20935046672821045,
      "logits/rejected": -0.21487772464752197,
      "logps/chosen": -474.3970642089844,
      "logps/rejected": -413.79779052734375,
      "loss": 0.389,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4984525740146637,
      "rewards/margins": 1.0169289112091064,
      "rewards/rejected": -1.5153814554214478,
      "step": 8221
    },
    {
      "epoch": 1.0758976036640222,
      "grad_norm": 5.768772602081299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3087739944458008,
      "logits/rejected": -0.22825995087623596,
      "logps/chosen": -607.6676025390625,
      "logps/rejected": -457.79071044921875,
      "loss": 0.3213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8626911044120789,
      "rewards/margins": 1.5900192260742188,
      "rewards/rejected": -2.4527101516723633,
      "step": 8222
    },
    {
      "epoch": 1.076028461601374,
      "grad_norm": 5.668492317199707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11618362367153168,
      "logits/rejected": -0.29772713780403137,
      "logps/chosen": -424.02130126953125,
      "logps/rejected": -546.909423828125,
      "loss": 0.307,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27936458587646484,
      "rewards/margins": 1.9356755018234253,
      "rewards/rejected": -2.2150402069091797,
      "step": 8223
    },
    {
      "epoch": 1.0761593195387258,
      "grad_norm": 9.044032096862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026929548010230064,
      "logits/rejected": -0.1451430469751358,
      "logps/chosen": -499.8611755371094,
      "logps/rejected": -480.0242919921875,
      "loss": 0.6061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1017167568206787,
      "rewards/margins": 1.1617952585220337,
      "rewards/rejected": -2.263512134552002,
      "step": 8224
    },
    {
      "epoch": 1.0762901774760776,
      "grad_norm": 9.234000205993652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18464460968971252,
      "logits/rejected": -0.04318875074386597,
      "logps/chosen": -442.3182067871094,
      "logps/rejected": -728.0274658203125,
      "loss": 0.5029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3959648013114929,
      "rewards/margins": 1.21224844455719,
      "rewards/rejected": -1.6082133054733276,
      "step": 8225
    },
    {
      "epoch": 1.0764210354134294,
      "grad_norm": 5.617631912231445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09774500131607056,
      "logits/rejected": -0.032169535756111145,
      "logps/chosen": -429.3851013183594,
      "logps/rejected": -260.9549560546875,
      "loss": 0.4605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9859539866447449,
      "rewards/margins": 0.6429205536842346,
      "rewards/rejected": -1.628874659538269,
      "step": 8226
    },
    {
      "epoch": 1.0765518933507812,
      "grad_norm": 7.761423110961914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2101699709892273,
      "logits/rejected": 0.21026736497879028,
      "logps/chosen": -523.531982421875,
      "logps/rejected": -521.150390625,
      "loss": 0.4991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42018717527389526,
      "rewards/margins": 0.7325558066368103,
      "rewards/rejected": -1.1527429819107056,
      "step": 8227
    },
    {
      "epoch": 1.0766827512881327,
      "grad_norm": 4.360833168029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006105288863182068,
      "logits/rejected": -0.0645306408405304,
      "logps/chosen": -423.5596008300781,
      "logps/rejected": -656.5017700195312,
      "loss": 0.2262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5412501692771912,
      "rewards/margins": 1.7727105617523193,
      "rewards/rejected": -2.3139610290527344,
      "step": 8228
    },
    {
      "epoch": 1.0768136092254845,
      "grad_norm": 5.648009777069092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10323262959718704,
      "logits/rejected": 0.19743189215660095,
      "logps/chosen": -559.306396484375,
      "logps/rejected": -534.2272338867188,
      "loss": 0.2412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09870817512273788,
      "rewards/margins": 1.7802833318710327,
      "rewards/rejected": -1.6815749406814575,
      "step": 8229
    },
    {
      "epoch": 1.0769444671628363,
      "grad_norm": 14.427192687988281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25342079997062683,
      "logits/rejected": -0.3234555423259735,
      "logps/chosen": -667.8093872070312,
      "logps/rejected": -582.1995239257812,
      "loss": 1.0532,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09297393262386322,
      "rewards/margins": 0.3235257863998413,
      "rewards/rejected": -0.41649970412254333,
      "step": 8230
    },
    {
      "epoch": 1.077075325100188,
      "grad_norm": 9.063204765319824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009289141744375229,
      "logits/rejected": -0.13979288935661316,
      "logps/chosen": -434.51910400390625,
      "logps/rejected": -406.51116943359375,
      "loss": 0.6434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0914803743362427,
      "rewards/margins": 0.262020468711853,
      "rewards/rejected": -1.3535008430480957,
      "step": 8231
    },
    {
      "epoch": 1.07720618303754,
      "grad_norm": 8.409621238708496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21509535610675812,
      "logits/rejected": -0.22070403397083282,
      "logps/chosen": -557.10009765625,
      "logps/rejected": -581.8375854492188,
      "loss": 0.5316,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02844390645623207,
      "rewards/margins": 0.8409286141395569,
      "rewards/rejected": -0.869372546672821,
      "step": 8232
    },
    {
      "epoch": 1.0773370409748917,
      "grad_norm": 14.301642417907715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3677748441696167,
      "logits/rejected": -0.24349923431873322,
      "logps/chosen": -521.0423583984375,
      "logps/rejected": -377.9754333496094,
      "loss": 0.8063,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9027893543243408,
      "rewards/margins": 0.45103076100349426,
      "rewards/rejected": -1.3538200855255127,
      "step": 8233
    },
    {
      "epoch": 1.0774678989122435,
      "grad_norm": 4.758730888366699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1754070669412613,
      "logits/rejected": -0.24329589307308197,
      "logps/chosen": -404.4522705078125,
      "logps/rejected": -483.75628662109375,
      "loss": 0.3099,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.28225216269493103,
      "rewards/margins": 1.5375428199768066,
      "rewards/rejected": -1.2552908658981323,
      "step": 8234
    },
    {
      "epoch": 1.077598756849595,
      "grad_norm": 8.70329475402832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07386987656354904,
      "logits/rejected": -0.24364356696605682,
      "logps/chosen": -547.4390258789062,
      "logps/rejected": -356.33465576171875,
      "loss": 0.4074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.013643257319927216,
      "rewards/margins": 1.348174810409546,
      "rewards/rejected": -1.3618179559707642,
      "step": 8235
    },
    {
      "epoch": 1.0777296147869468,
      "grad_norm": 10.661683082580566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41949206590652466,
      "logits/rejected": -0.3658638000488281,
      "logps/chosen": -398.8140869140625,
      "logps/rejected": -359.3705139160156,
      "loss": 0.8729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2861210107803345,
      "rewards/margins": 0.1427110731601715,
      "rewards/rejected": -1.4288320541381836,
      "step": 8236
    },
    {
      "epoch": 1.0778604727242986,
      "grad_norm": 4.671780109405518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07722543925046921,
      "logits/rejected": -0.3236318826675415,
      "logps/chosen": -246.49026489257812,
      "logps/rejected": -409.4219055175781,
      "loss": 0.1872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.830461859703064,
      "rewards/margins": 1.9343217611312866,
      "rewards/rejected": -2.7647838592529297,
      "step": 8237
    },
    {
      "epoch": 1.0779913306616504,
      "grad_norm": 6.966940402984619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1323293149471283,
      "logits/rejected": 0.005153018981218338,
      "logps/chosen": -572.24462890625,
      "logps/rejected": -431.43011474609375,
      "loss": 0.3768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19894102215766907,
      "rewards/margins": 1.512082576751709,
      "rewards/rejected": -1.7110234498977661,
      "step": 8238
    },
    {
      "epoch": 1.0781221885990022,
      "grad_norm": 6.834848403930664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08494560420513153,
      "logits/rejected": -0.2534504532814026,
      "logps/chosen": -546.331787109375,
      "logps/rejected": -494.2633361816406,
      "loss": 0.4069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0013559460639953613,
      "rewards/margins": 1.2229578495025635,
      "rewards/rejected": -1.221601963043213,
      "step": 8239
    },
    {
      "epoch": 1.078253046536354,
      "grad_norm": 9.11977767944336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2095640003681183,
      "logits/rejected": -0.2459263801574707,
      "logps/chosen": -388.5821533203125,
      "logps/rejected": -319.837158203125,
      "loss": 0.6751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2454453706741333,
      "rewards/margins": 0.4770660400390625,
      "rewards/rejected": -1.7225114107131958,
      "step": 8240
    },
    {
      "epoch": 1.0783839044737058,
      "grad_norm": 5.277930736541748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030364524573087692,
      "logits/rejected": -0.21047382056713104,
      "logps/chosen": -497.0457763671875,
      "logps/rejected": -487.649169921875,
      "loss": 0.3868,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7091165781021118,
      "rewards/margins": 1.2753050327301025,
      "rewards/rejected": -1.9844214916229248,
      "step": 8241
    },
    {
      "epoch": 1.0785147624110576,
      "grad_norm": 7.758105754852295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07856187224388123,
      "logits/rejected": -0.22008848190307617,
      "logps/chosen": -565.5264282226562,
      "logps/rejected": -633.0781860351562,
      "loss": 0.4193,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5340505838394165,
      "rewards/margins": 1.2714543342590332,
      "rewards/rejected": -1.8055047988891602,
      "step": 8242
    },
    {
      "epoch": 1.0786456203484092,
      "grad_norm": 5.334377288818359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4205969572067261,
      "logits/rejected": -0.27403801679611206,
      "logps/chosen": -356.0434875488281,
      "logps/rejected": -335.7550048828125,
      "loss": 0.3246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5529743432998657,
      "rewards/margins": 1.503874659538269,
      "rewards/rejected": -2.0568490028381348,
      "step": 8243
    },
    {
      "epoch": 1.078776478285761,
      "grad_norm": 5.795261383056641,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1205272451043129,
      "logits/rejected": -0.32857653498649597,
      "logps/chosen": -698.6246337890625,
      "logps/rejected": -494.28375244140625,
      "loss": 0.4236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9114145040512085,
      "rewards/margins": 1.167860507965088,
      "rewards/rejected": -2.079274892807007,
      "step": 8244
    },
    {
      "epoch": 1.0789073362231127,
      "grad_norm": 5.775871753692627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23835349082946777,
      "logits/rejected": -0.35985487699508667,
      "logps/chosen": -542.7493896484375,
      "logps/rejected": -384.4561462402344,
      "loss": 0.3399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0669970512390137,
      "rewards/margins": 1.4748713970184326,
      "rewards/rejected": -2.541868209838867,
      "step": 8245
    },
    {
      "epoch": 1.0790381941604645,
      "grad_norm": 4.950323581695557,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010181000456213951,
      "logits/rejected": -0.3654184937477112,
      "logps/chosen": -514.3659057617188,
      "logps/rejected": -514.0901489257812,
      "loss": 0.3083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13003993034362793,
      "rewards/margins": 1.6587951183319092,
      "rewards/rejected": -1.5287551879882812,
      "step": 8246
    },
    {
      "epoch": 1.0791690520978163,
      "grad_norm": 5.908905029296875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1950007975101471,
      "logits/rejected": -0.04585973918437958,
      "logps/chosen": -386.9141845703125,
      "logps/rejected": -458.71234130859375,
      "loss": 0.2708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26736319065093994,
      "rewards/margins": 2.2254226207733154,
      "rewards/rejected": -1.958059549331665,
      "step": 8247
    },
    {
      "epoch": 1.0792999100351681,
      "grad_norm": 11.076339721679688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20166340470314026,
      "logits/rejected": -0.10788562148809433,
      "logps/chosen": -207.05625915527344,
      "logps/rejected": -367.5482482910156,
      "loss": 0.4656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6936323046684265,
      "rewards/margins": 1.2878823280334473,
      "rewards/rejected": -1.981514573097229,
      "step": 8248
    },
    {
      "epoch": 1.07943076797252,
      "grad_norm": 7.028276443481445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05727469548583031,
      "logits/rejected": -0.10151249170303345,
      "logps/chosen": -431.6739196777344,
      "logps/rejected": -424.3132019042969,
      "loss": 0.3865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.407659649848938,
      "rewards/margins": 1.3927558660507202,
      "rewards/rejected": -2.800415277481079,
      "step": 8249
    },
    {
      "epoch": 1.0795616259098715,
      "grad_norm": 11.099503517150879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0737849548459053,
      "logits/rejected": -0.05085340142250061,
      "logps/chosen": -350.12298583984375,
      "logps/rejected": -338.7935485839844,
      "loss": 0.3888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9096794724464417,
      "rewards/margins": 1.228299617767334,
      "rewards/rejected": -2.137979030609131,
      "step": 8250
    },
    {
      "epoch": 1.0796924838472233,
      "grad_norm": 9.945311546325684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04973744601011276,
      "logits/rejected": -0.21308888494968414,
      "logps/chosen": -476.4558410644531,
      "logps/rejected": -565.2738037109375,
      "loss": 0.5082,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3194049596786499,
      "rewards/margins": 1.0732905864715576,
      "rewards/rejected": -1.392695665359497,
      "step": 8251
    },
    {
      "epoch": 1.079823341784575,
      "grad_norm": 7.0717878341674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19644540548324585,
      "logits/rejected": -0.0947418287396431,
      "logps/chosen": -333.61627197265625,
      "logps/rejected": -348.929931640625,
      "loss": 0.4516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5296330451965332,
      "rewards/margins": 0.7608203887939453,
      "rewards/rejected": -2.2904534339904785,
      "step": 8252
    },
    {
      "epoch": 1.0799541997219269,
      "grad_norm": 4.22527551651001,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05033446103334427,
      "logits/rejected": -0.23853008449077606,
      "logps/chosen": -444.0881652832031,
      "logps/rejected": -425.4576110839844,
      "loss": 0.3453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11718502640724182,
      "rewards/margins": 1.4750714302062988,
      "rewards/rejected": -1.5922565460205078,
      "step": 8253
    },
    {
      "epoch": 1.0800850576592786,
      "grad_norm": 7.6222453117370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37321174144744873,
      "logits/rejected": -0.15220335125923157,
      "logps/chosen": -490.4726257324219,
      "logps/rejected": -507.48187255859375,
      "loss": 0.3641,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12277396768331528,
      "rewards/margins": 1.6639128923416138,
      "rewards/rejected": -1.5411388874053955,
      "step": 8254
    },
    {
      "epoch": 1.0802159155966304,
      "grad_norm": 5.504832744598389,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3292001187801361,
      "logits/rejected": 0.1952848732471466,
      "logps/chosen": -556.5936279296875,
      "logps/rejected": -498.1799011230469,
      "loss": 0.262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.235392764210701,
      "rewards/margins": 1.4995627403259277,
      "rewards/rejected": -1.7349555492401123,
      "step": 8255
    },
    {
      "epoch": 1.0803467735339822,
      "grad_norm": 6.852020740509033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07557656615972519,
      "logits/rejected": -0.1883327215909958,
      "logps/chosen": -268.6731872558594,
      "logps/rejected": -271.72369384765625,
      "loss": 0.3114,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6004413366317749,
      "rewards/margins": 2.358180522918701,
      "rewards/rejected": -2.9586219787597656,
      "step": 8256
    },
    {
      "epoch": 1.080477631471334,
      "grad_norm": 10.396089553833008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14186109602451324,
      "logits/rejected": -0.08185586333274841,
      "logps/chosen": -429.1955261230469,
      "logps/rejected": -364.63592529296875,
      "loss": 0.5979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4245535135269165,
      "rewards/margins": 0.6597154140472412,
      "rewards/rejected": -2.084268808364868,
      "step": 8257
    },
    {
      "epoch": 1.0806084894086858,
      "grad_norm": 7.286015033721924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07983807474374771,
      "logits/rejected": -0.0652630627155304,
      "logps/chosen": -436.91192626953125,
      "logps/rejected": -407.2922668457031,
      "loss": 0.4291,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2201259732246399,
      "rewards/margins": 1.2397160530090332,
      "rewards/rejected": -1.4598419666290283,
      "step": 8258
    },
    {
      "epoch": 1.0807393473460374,
      "grad_norm": 4.0970964431762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06149839237332344,
      "logits/rejected": -0.16733917593955994,
      "logps/chosen": -287.7993469238281,
      "logps/rejected": -284.3752136230469,
      "loss": 0.3854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9203006625175476,
      "rewards/margins": 1.0722709894180298,
      "rewards/rejected": -1.9925717115402222,
      "step": 8259
    },
    {
      "epoch": 1.0808702052833892,
      "grad_norm": 8.486095428466797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03508247807621956,
      "logits/rejected": -0.2178741693496704,
      "logps/chosen": -826.0985107421875,
      "logps/rejected": -521.2332153320312,
      "loss": 0.3992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5956791639328003,
      "rewards/margins": 1.809701919555664,
      "rewards/rejected": -2.405381202697754,
      "step": 8260
    },
    {
      "epoch": 1.081001063220741,
      "grad_norm": 3.0571086406707764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05761092156171799,
      "logits/rejected": -0.024176258593797684,
      "logps/chosen": -403.8082275390625,
      "logps/rejected": -374.88214111328125,
      "loss": 0.1628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.052016742527484894,
      "rewards/margins": 2.532609462738037,
      "rewards/rejected": -2.5846261978149414,
      "step": 8261
    },
    {
      "epoch": 1.0811319211580928,
      "grad_norm": 6.0297746658325195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07346482574939728,
      "logits/rejected": -0.12136707454919815,
      "logps/chosen": -469.0414733886719,
      "logps/rejected": -433.7650146484375,
      "loss": 0.3951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9438457489013672,
      "rewards/margins": 1.9861834049224854,
      "rewards/rejected": -2.9300293922424316,
      "step": 8262
    },
    {
      "epoch": 1.0812627790954445,
      "grad_norm": 3.582256555557251,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16207948327064514,
      "logits/rejected": -0.04943294823169708,
      "logps/chosen": -552.823486328125,
      "logps/rejected": -493.9573974609375,
      "loss": 0.1655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20788003504276276,
      "rewards/margins": 2.1700632572174072,
      "rewards/rejected": -2.3779430389404297,
      "step": 8263
    },
    {
      "epoch": 1.0813936370327963,
      "grad_norm": 8.19324016571045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17180630564689636,
      "logits/rejected": -0.22750478982925415,
      "logps/chosen": -500.82550048828125,
      "logps/rejected": -480.4822082519531,
      "loss": 0.5436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8307548761367798,
      "rewards/margins": 0.6526250839233398,
      "rewards/rejected": -2.483380079269409,
      "step": 8264
    },
    {
      "epoch": 1.0815244949701481,
      "grad_norm": 7.80501651763916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43562132120132446,
      "logits/rejected": -0.17065638303756714,
      "logps/chosen": -393.064208984375,
      "logps/rejected": -468.5342102050781,
      "loss": 0.5067,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5837425589561462,
      "rewards/margins": 1.0113732814788818,
      "rewards/rejected": -1.5951157808303833,
      "step": 8265
    },
    {
      "epoch": 1.0816553529074997,
      "grad_norm": 9.219134330749512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010668274015188217,
      "logits/rejected": -0.03751400485634804,
      "logps/chosen": -382.5554504394531,
      "logps/rejected": -397.998046875,
      "loss": 0.6249,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7329710721969604,
      "rewards/margins": 0.3009766638278961,
      "rewards/rejected": -1.0339477062225342,
      "step": 8266
    },
    {
      "epoch": 1.0817862108448515,
      "grad_norm": 6.579108238220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3760875463485718,
      "logits/rejected": -0.3408159017562866,
      "logps/chosen": -565.36474609375,
      "logps/rejected": -434.4752197265625,
      "loss": 0.4615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7161657810211182,
      "rewards/margins": 0.7221070528030396,
      "rewards/rejected": -1.4382729530334473,
      "step": 8267
    },
    {
      "epoch": 1.0819170687822033,
      "grad_norm": 5.480292797088623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45331525802612305,
      "logits/rejected": -0.49637871980667114,
      "logps/chosen": -522.691650390625,
      "logps/rejected": -577.4593505859375,
      "loss": 0.2802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07041418552398682,
      "rewards/margins": 1.5795799493789673,
      "rewards/rejected": -1.6499940156936646,
      "step": 8268
    },
    {
      "epoch": 1.082047926719555,
      "grad_norm": 9.734415054321289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3980187773704529,
      "logits/rejected": -0.4634348154067993,
      "logps/chosen": -310.9601135253906,
      "logps/rejected": -320.205322265625,
      "loss": 0.7547,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.336750030517578,
      "rewards/margins": 0.179880291223526,
      "rewards/rejected": -2.5166304111480713,
      "step": 8269
    },
    {
      "epoch": 1.0821787846569069,
      "grad_norm": 2.46036958694458,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11732710152864456,
      "logits/rejected": -0.09744995832443237,
      "logps/chosen": -327.2439880371094,
      "logps/rejected": -284.5867919921875,
      "loss": 0.1867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14498348534107208,
      "rewards/margins": 2.003528356552124,
      "rewards/rejected": -2.1485118865966797,
      "step": 8270
    },
    {
      "epoch": 1.0823096425942587,
      "grad_norm": 8.203393936157227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03127399832010269,
      "logits/rejected": -0.03481762856245041,
      "logps/chosen": -394.22479248046875,
      "logps/rejected": -367.0102844238281,
      "loss": 0.6461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5754427909851074,
      "rewards/margins": 0.4025697112083435,
      "rewards/rejected": -0.9780125021934509,
      "step": 8271
    },
    {
      "epoch": 1.0824405005316104,
      "grad_norm": 10.718193054199219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11852116882801056,
      "logits/rejected": 0.07902425527572632,
      "logps/chosen": -570.586181640625,
      "logps/rejected": -525.2410888671875,
      "loss": 0.6396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7987105846405029,
      "rewards/margins": 0.5354747772216797,
      "rewards/rejected": -1.3341854810714722,
      "step": 8272
    },
    {
      "epoch": 1.0825713584689622,
      "grad_norm": 6.709424018859863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09006748348474503,
      "logits/rejected": -0.1750386506319046,
      "logps/chosen": -525.2717895507812,
      "logps/rejected": -499.4188537597656,
      "loss": 0.2555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9720118045806885,
      "rewards/margins": 1.6764349937438965,
      "rewards/rejected": -2.648446559906006,
      "step": 8273
    },
    {
      "epoch": 1.0827022164063138,
      "grad_norm": 9.359103202819824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09000539779663086,
      "logits/rejected": 0.19323451817035675,
      "logps/chosen": -681.896484375,
      "logps/rejected": -421.5223388671875,
      "loss": 0.6052,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17630702257156372,
      "rewards/margins": 1.0561062097549438,
      "rewards/rejected": -1.2324130535125732,
      "step": 8274
    },
    {
      "epoch": 1.0828330743436656,
      "grad_norm": 6.8395819664001465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.105600506067276,
      "logits/rejected": -0.2670851945877075,
      "logps/chosen": -423.9536437988281,
      "logps/rejected": -357.7444152832031,
      "loss": 0.3508,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0294103492051363,
      "rewards/margins": 1.2863450050354004,
      "rewards/rejected": -1.2569347620010376,
      "step": 8275
    },
    {
      "epoch": 1.0829639322810174,
      "grad_norm": 8.18944263458252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06733818352222443,
      "logits/rejected": 0.09064160287380219,
      "logps/chosen": -502.18878173828125,
      "logps/rejected": -639.4896240234375,
      "loss": 0.4824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8253370523452759,
      "rewards/margins": 1.4412497282028198,
      "rewards/rejected": -2.2665867805480957,
      "step": 8276
    },
    {
      "epoch": 1.0830947902183692,
      "grad_norm": 9.876298904418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11362423002719879,
      "logits/rejected": -0.14768347144126892,
      "logps/chosen": -343.0401611328125,
      "logps/rejected": -658.5805053710938,
      "loss": 0.6646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4517920017242432,
      "rewards/margins": 1.4510449171066284,
      "rewards/rejected": -2.902837038040161,
      "step": 8277
    },
    {
      "epoch": 1.083225648155721,
      "grad_norm": 8.502825736999512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0978853851556778,
      "logits/rejected": -0.26096436381340027,
      "logps/chosen": -624.8449096679688,
      "logps/rejected": -452.49835205078125,
      "loss": 0.4999,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.064548134803772,
      "rewards/margins": 1.000169038772583,
      "rewards/rejected": -2.0647172927856445,
      "step": 8278
    },
    {
      "epoch": 1.0833565060930728,
      "grad_norm": 5.571845531463623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08936889469623566,
      "logits/rejected": -0.04299347847700119,
      "logps/chosen": -282.14556884765625,
      "logps/rejected": -454.1548767089844,
      "loss": 0.4182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19704385101795197,
      "rewards/margins": 1.632372260093689,
      "rewards/rejected": -1.8294161558151245,
      "step": 8279
    },
    {
      "epoch": 1.0834873640304246,
      "grad_norm": 7.59871244430542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09813079237937927,
      "logits/rejected": -0.11807449907064438,
      "logps/chosen": -410.7480773925781,
      "logps/rejected": -582.59716796875,
      "loss": 0.3684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5526089072227478,
      "rewards/margins": 2.3673412799835205,
      "rewards/rejected": -2.919950246810913,
      "step": 8280
    },
    {
      "epoch": 1.0836182219677761,
      "grad_norm": 9.377198219299316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2997507154941559,
      "logits/rejected": -0.33173826336860657,
      "logps/chosen": -355.7255859375,
      "logps/rejected": -505.72320556640625,
      "loss": 0.7463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3248862028121948,
      "rewards/margins": 0.7664893865585327,
      "rewards/rejected": -2.0913753509521484,
      "step": 8281
    },
    {
      "epoch": 1.083749079905128,
      "grad_norm": 10.074878692626953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05275910347700119,
      "logits/rejected": -0.22943291068077087,
      "logps/chosen": -507.28240966796875,
      "logps/rejected": -377.37261962890625,
      "loss": 0.475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8205106854438782,
      "rewards/margins": 1.417014718055725,
      "rewards/rejected": -2.237525463104248,
      "step": 8282
    },
    {
      "epoch": 1.0838799378424797,
      "grad_norm": 3.705899238586426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2753627300262451,
      "logits/rejected": -0.05718202888965607,
      "logps/chosen": -447.6668701171875,
      "logps/rejected": -378.93157958984375,
      "loss": 0.1869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6393305063247681,
      "rewards/margins": 2.222393751144409,
      "rewards/rejected": -2.8617238998413086,
      "step": 8283
    },
    {
      "epoch": 1.0840107957798315,
      "grad_norm": 3.2822580337524414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05458299070596695,
      "logits/rejected": -0.3599510192871094,
      "logps/chosen": -564.2174682617188,
      "logps/rejected": -431.0148620605469,
      "loss": 0.1339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0575087070465088,
      "rewards/margins": 2.334543466567993,
      "rewards/rejected": -3.392052173614502,
      "step": 8284
    },
    {
      "epoch": 1.0841416537171833,
      "grad_norm": 10.188590049743652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05796581506729126,
      "logits/rejected": -0.15928193926811218,
      "logps/chosen": -416.23468017578125,
      "logps/rejected": -374.22760009765625,
      "loss": 0.7097,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7708474397659302,
      "rewards/margins": 0.7041638493537903,
      "rewards/rejected": -1.4750112295150757,
      "step": 8285
    },
    {
      "epoch": 1.084272511654535,
      "grad_norm": 8.62891960144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1890532523393631,
      "logits/rejected": 0.1594340205192566,
      "logps/chosen": -335.0993957519531,
      "logps/rejected": -398.2171630859375,
      "loss": 0.5423,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0940654277801514,
      "rewards/margins": 0.8004902005195618,
      "rewards/rejected": -1.8945558071136475,
      "step": 8286
    },
    {
      "epoch": 1.0844033695918869,
      "grad_norm": 9.664738655090332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021132804453372955,
      "logits/rejected": 0.08449400961399078,
      "logps/chosen": -486.9043273925781,
      "logps/rejected": -420.572265625,
      "loss": 0.6894,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.690808117389679,
      "rewards/margins": 0.4112388789653778,
      "rewards/rejected": -1.1020469665527344,
      "step": 8287
    },
    {
      "epoch": 1.0845342275292387,
      "grad_norm": 4.677901268005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37689846754074097,
      "logits/rejected": -0.5274356007575989,
      "logps/chosen": -266.09197998046875,
      "logps/rejected": -506.9817810058594,
      "loss": 0.3505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8099081516265869,
      "rewards/margins": 2.289102554321289,
      "rewards/rejected": -3.099010705947876,
      "step": 8288
    },
    {
      "epoch": 1.0846650854665902,
      "grad_norm": 10.833378791809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26540040969848633,
      "logits/rejected": -0.07210052013397217,
      "logps/chosen": -441.279296875,
      "logps/rejected": -509.5295715332031,
      "loss": 0.702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3939847946166992,
      "rewards/margins": 0.8243411779403687,
      "rewards/rejected": -1.2183260917663574,
      "step": 8289
    },
    {
      "epoch": 1.084795943403942,
      "grad_norm": 10.758651733398438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1740567982196808,
      "logits/rejected": -0.35209399461746216,
      "logps/chosen": -405.333984375,
      "logps/rejected": -497.9538269042969,
      "loss": 0.696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8273898959159851,
      "rewards/margins": 0.9303420782089233,
      "rewards/rejected": -1.7577320337295532,
      "step": 8290
    },
    {
      "epoch": 1.0849268013412938,
      "grad_norm": 6.746289253234863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20335865020751953,
      "logits/rejected": -0.4221819043159485,
      "logps/chosen": -401.5592041015625,
      "logps/rejected": -384.77923583984375,
      "loss": 0.2535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.508224606513977,
      "rewards/margins": 2.0888655185699463,
      "rewards/rejected": -3.597090244293213,
      "step": 8291
    },
    {
      "epoch": 1.0850576592786456,
      "grad_norm": 8.345606803894043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15728476643562317,
      "logits/rejected": -0.18562304973602295,
      "logps/chosen": -377.8335266113281,
      "logps/rejected": -450.0659484863281,
      "loss": 0.5288,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9892666339874268,
      "rewards/margins": 0.9757257699966431,
      "rewards/rejected": -1.9649924039840698,
      "step": 8292
    },
    {
      "epoch": 1.0851885172159974,
      "grad_norm": 6.748347759246826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14840753376483917,
      "logits/rejected": -0.12646444141864777,
      "logps/chosen": -526.3845825195312,
      "logps/rejected": -573.1787109375,
      "loss": 0.2949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18724635243415833,
      "rewards/margins": 1.7110247611999512,
      "rewards/rejected": -1.8982713222503662,
      "step": 8293
    },
    {
      "epoch": 1.0853193751533492,
      "grad_norm": 11.823380470275879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33045029640197754,
      "logits/rejected": -0.4766813814640045,
      "logps/chosen": -553.7009887695312,
      "logps/rejected": -653.6056518554688,
      "loss": 0.8403,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1747020483016968,
      "rewards/margins": 0.06997144222259521,
      "rewards/rejected": -1.244673490524292,
      "step": 8294
    },
    {
      "epoch": 1.085450233090701,
      "grad_norm": 13.40700626373291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3968789279460907,
      "logits/rejected": -0.2958417236804962,
      "logps/chosen": -273.5377502441406,
      "logps/rejected": -346.22882080078125,
      "loss": 0.9245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5786914825439453,
      "rewards/margins": 0.042930781841278076,
      "rewards/rejected": -1.6216222047805786,
      "step": 8295
    },
    {
      "epoch": 1.0855810910280526,
      "grad_norm": 7.993640899658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.052100569009780884,
      "logits/rejected": -0.09700693935155869,
      "logps/chosen": -386.94244384765625,
      "logps/rejected": -400.790283203125,
      "loss": 0.408,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7515648603439331,
      "rewards/margins": 1.342427134513855,
      "rewards/rejected": -2.093991756439209,
      "step": 8296
    },
    {
      "epoch": 1.0857119489654043,
      "grad_norm": 9.088029861450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010278142988681793,
      "logits/rejected": -0.19117411971092224,
      "logps/chosen": -437.79852294921875,
      "logps/rejected": -272.3763427734375,
      "loss": 0.4432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3454973697662354,
      "rewards/margins": 1.0527291297912598,
      "rewards/rejected": -2.398226261138916,
      "step": 8297
    },
    {
      "epoch": 1.0858428069027561,
      "grad_norm": 4.5965046882629395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46453651785850525,
      "logits/rejected": -0.3952392339706421,
      "logps/chosen": -725.8976440429688,
      "logps/rejected": -472.8958435058594,
      "loss": 0.1766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1639603227376938,
      "rewards/margins": 2.5783207416534424,
      "rewards/rejected": -2.742280960083008,
      "step": 8298
    },
    {
      "epoch": 1.085973664840108,
      "grad_norm": 4.353679656982422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3104369640350342,
      "logits/rejected": -0.2352135181427002,
      "logps/chosen": -403.3934631347656,
      "logps/rejected": -385.28839111328125,
      "loss": 0.3023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6643085479736328,
      "rewards/margins": 1.8011271953582764,
      "rewards/rejected": -2.46543550491333,
      "step": 8299
    },
    {
      "epoch": 1.0861045227774597,
      "grad_norm": 9.380499839782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2445303499698639,
      "logits/rejected": 0.03067716397345066,
      "logps/chosen": -585.2322998046875,
      "logps/rejected": -427.13409423828125,
      "loss": 0.5368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12917043268680573,
      "rewards/margins": 1.2503982782363892,
      "rewards/rejected": -1.1212279796600342,
      "step": 8300
    },
    {
      "epoch": 1.0862353807148115,
      "grad_norm": 4.795499324798584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1756136268377304,
      "logits/rejected": 0.0021583829075098038,
      "logps/chosen": -366.8001708984375,
      "logps/rejected": -376.89666748046875,
      "loss": 0.3098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25278109312057495,
      "rewards/margins": 1.2257394790649414,
      "rewards/rejected": -1.4785206317901611,
      "step": 8301
    },
    {
      "epoch": 1.0863662386521633,
      "grad_norm": 5.288480758666992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08061128854751587,
      "logits/rejected": -0.2641124725341797,
      "logps/chosen": -539.6419677734375,
      "logps/rejected": -336.8966979980469,
      "loss": 0.2131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6857415437698364,
      "rewards/margins": 1.7631334066390991,
      "rewards/rejected": -2.4488751888275146,
      "step": 8302
    },
    {
      "epoch": 1.086497096589515,
      "grad_norm": 10.25464916229248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09149518609046936,
      "logits/rejected": -0.08803131431341171,
      "logps/chosen": -548.3536376953125,
      "logps/rejected": -446.69451904296875,
      "loss": 0.66,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0980229377746582,
      "rewards/margins": 0.7022404074668884,
      "rewards/rejected": -1.8002631664276123,
      "step": 8303
    },
    {
      "epoch": 1.0866279545268667,
      "grad_norm": 8.483597755432129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30090463161468506,
      "logits/rejected": -0.16957974433898926,
      "logps/chosen": -422.652587890625,
      "logps/rejected": -457.3897705078125,
      "loss": 0.3424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1236926317214966,
      "rewards/margins": 1.9347200393676758,
      "rewards/rejected": -3.0584123134613037,
      "step": 8304
    },
    {
      "epoch": 1.0867588124642185,
      "grad_norm": 8.746752738952637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3233982026576996,
      "logits/rejected": -0.6100428104400635,
      "logps/chosen": -450.5029296875,
      "logps/rejected": -562.1183471679688,
      "loss": 0.5884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9682365655899048,
      "rewards/margins": 1.831057071685791,
      "rewards/rejected": -3.7992937564849854,
      "step": 8305
    },
    {
      "epoch": 1.0868896704015703,
      "grad_norm": 3.0061588287353516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12039908766746521,
      "logits/rejected": -0.07823620736598969,
      "logps/chosen": -426.3047790527344,
      "logps/rejected": -421.1061096191406,
      "loss": 0.1513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02212047576904297,
      "rewards/margins": 2.9440431594848633,
      "rewards/rejected": -2.921922445297241,
      "step": 8306
    },
    {
      "epoch": 1.087020528338922,
      "grad_norm": 6.222754955291748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21825936436653137,
      "logits/rejected": -0.39163848757743835,
      "logps/chosen": -449.034423828125,
      "logps/rejected": -383.46405029296875,
      "loss": 0.3063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7556654810905457,
      "rewards/margins": 1.9657526016235352,
      "rewards/rejected": -2.7214179039001465,
      "step": 8307
    },
    {
      "epoch": 1.0871513862762738,
      "grad_norm": 5.1209869384765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19483716785907745,
      "logits/rejected": -0.2902948260307312,
      "logps/chosen": -330.9958801269531,
      "logps/rejected": -341.9559326171875,
      "loss": 0.3183,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08739682286977768,
      "rewards/margins": 1.9218485355377197,
      "rewards/rejected": -2.0092453956604004,
      "step": 8308
    },
    {
      "epoch": 1.0872822442136256,
      "grad_norm": 7.194474220275879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3670015335083008,
      "logits/rejected": -0.3428422212600708,
      "logps/chosen": -351.29461669921875,
      "logps/rejected": -451.1473388671875,
      "loss": 0.2872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.434178113937378,
      "rewards/margins": 1.3257050514221191,
      "rewards/rejected": -2.759882926940918,
      "step": 8309
    },
    {
      "epoch": 1.0874131021509774,
      "grad_norm": 8.014721870422363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34538379311561584,
      "logits/rejected": -0.2698511779308319,
      "logps/chosen": -398.7833251953125,
      "logps/rejected": -442.70343017578125,
      "loss": 0.6791,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5512864589691162,
      "rewards/margins": 0.5165302753448486,
      "rewards/rejected": -2.067816734313965,
      "step": 8310
    },
    {
      "epoch": 1.0875439600883292,
      "grad_norm": 7.065640449523926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04973403364419937,
      "logits/rejected": 0.023574622347950935,
      "logps/chosen": -615.055908203125,
      "logps/rejected": -412.22210693359375,
      "loss": 0.3596,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8570154905319214,
      "rewards/margins": 1.314769983291626,
      "rewards/rejected": -2.171785354614258,
      "step": 8311
    },
    {
      "epoch": 1.0876748180256808,
      "grad_norm": 7.3188910484313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27994680404663086,
      "logits/rejected": -0.17086803913116455,
      "logps/chosen": -486.5507507324219,
      "logps/rejected": -443.70416259765625,
      "loss": 0.3263,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0484014749526978,
      "rewards/margins": 1.930070161819458,
      "rewards/rejected": -2.9784717559814453,
      "step": 8312
    },
    {
      "epoch": 1.0878056759630326,
      "grad_norm": 7.896768569946289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23084011673927307,
      "logits/rejected": -0.40222322940826416,
      "logps/chosen": -439.1783142089844,
      "logps/rejected": -549.8865966796875,
      "loss": 0.4847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8547099828720093,
      "rewards/margins": 1.5589845180511475,
      "rewards/rejected": -2.413694381713867,
      "step": 8313
    },
    {
      "epoch": 1.0879365339003844,
      "grad_norm": 6.979340076446533,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2608124911785126,
      "logits/rejected": -0.25425711274147034,
      "logps/chosen": -616.4029541015625,
      "logps/rejected": -423.0226745605469,
      "loss": 0.3273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2723331451416016,
      "rewards/margins": 1.4501603841781616,
      "rewards/rejected": -2.7224936485290527,
      "step": 8314
    },
    {
      "epoch": 1.0880673918377362,
      "grad_norm": 6.3129496574401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.394756555557251,
      "logits/rejected": -0.26652151346206665,
      "logps/chosen": -459.3768310546875,
      "logps/rejected": -406.1277770996094,
      "loss": 0.1891,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.112703800201416,
      "rewards/margins": 2.4621334075927734,
      "rewards/rejected": -3.5748369693756104,
      "step": 8315
    },
    {
      "epoch": 1.088198249775088,
      "grad_norm": 9.207406997680664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17037001252174377,
      "logits/rejected": -0.26727840304374695,
      "logps/chosen": -511.4698791503906,
      "logps/rejected": -400.2857666015625,
      "loss": 0.3095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11656820774078369,
      "rewards/margins": 1.589812994003296,
      "rewards/rejected": -1.7063812017440796,
      "step": 8316
    },
    {
      "epoch": 1.0883291077124397,
      "grad_norm": 4.347567081451416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07901773601770401,
      "logits/rejected": -0.18957838416099548,
      "logps/chosen": -528.0543823242188,
      "logps/rejected": -344.2317199707031,
      "loss": 0.2237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18341276049613953,
      "rewards/margins": 2.124279022216797,
      "rewards/rejected": -2.307692050933838,
      "step": 8317
    },
    {
      "epoch": 1.0884599656497915,
      "grad_norm": 10.501816749572754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14826883375644684,
      "logits/rejected": -0.19015319645404816,
      "logps/chosen": -594.066162109375,
      "logps/rejected": -506.1744384765625,
      "loss": 0.5408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4252186119556427,
      "rewards/margins": 0.9675046801567078,
      "rewards/rejected": -1.3927233219146729,
      "step": 8318
    },
    {
      "epoch": 1.0885908235871433,
      "grad_norm": 9.857429504394531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037538111209869385,
      "logits/rejected": -0.14006701111793518,
      "logps/chosen": -570.5272216796875,
      "logps/rejected": -572.5702514648438,
      "loss": 0.6338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4097527265548706,
      "rewards/margins": 0.8556362986564636,
      "rewards/rejected": -2.2653889656066895,
      "step": 8319
    },
    {
      "epoch": 1.0887216815244949,
      "grad_norm": 9.723419189453125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20115327835083008,
      "logits/rejected": 0.13845795392990112,
      "logps/chosen": -466.2000427246094,
      "logps/rejected": -473.1613464355469,
      "loss": 0.5948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6857470273971558,
      "rewards/margins": 0.8429232835769653,
      "rewards/rejected": -2.528670310974121,
      "step": 8320
    },
    {
      "epoch": 1.0888525394618467,
      "grad_norm": 5.476944923400879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1902199536561966,
      "logits/rejected": -0.26537230610847473,
      "logps/chosen": -510.8647766113281,
      "logps/rejected": -455.1694030761719,
      "loss": 0.2015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3103702664375305,
      "rewards/margins": 2.2024247646331787,
      "rewards/rejected": -2.5127949714660645,
      "step": 8321
    },
    {
      "epoch": 1.0889833973991985,
      "grad_norm": 10.15676498413086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20180734992027283,
      "logits/rejected": -0.29787880182266235,
      "logps/chosen": -314.0334777832031,
      "logps/rejected": -418.00469970703125,
      "loss": 0.7122,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1428972482681274,
      "rewards/margins": 1.154394268989563,
      "rewards/rejected": -2.2972917556762695,
      "step": 8322
    },
    {
      "epoch": 1.0891142553365503,
      "grad_norm": 7.3612213134765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19955095648765564,
      "logits/rejected": -0.1958727389574051,
      "logps/chosen": -419.27764892578125,
      "logps/rejected": -364.86773681640625,
      "loss": 0.4778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2370058298110962,
      "rewards/margins": 1.0434191226959229,
      "rewards/rejected": -2.2804248332977295,
      "step": 8323
    },
    {
      "epoch": 1.089245113273902,
      "grad_norm": 11.907510757446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0001519478391855955,
      "logits/rejected": 0.16566552221775055,
      "logps/chosen": -498.167236328125,
      "logps/rejected": -638.0501708984375,
      "loss": 0.9251,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7061001062393188,
      "rewards/margins": -0.057954609394073486,
      "rewards/rejected": -0.6481454968452454,
      "step": 8324
    },
    {
      "epoch": 1.0893759712112538,
      "grad_norm": 11.895209312438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14845742285251617,
      "logits/rejected": -0.3004681468009949,
      "logps/chosen": -524.11279296875,
      "logps/rejected": -443.3337097167969,
      "loss": 0.4321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47429096698760986,
      "rewards/margins": 1.6370575428009033,
      "rewards/rejected": -2.1113486289978027,
      "step": 8325
    },
    {
      "epoch": 1.0895068291486056,
      "grad_norm": 6.280266284942627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0692092627286911,
      "logits/rejected": -0.13501545786857605,
      "logps/chosen": -459.8067321777344,
      "logps/rejected": -425.92559814453125,
      "loss": 0.3478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48481327295303345,
      "rewards/margins": 1.063692569732666,
      "rewards/rejected": -1.5485057830810547,
      "step": 8326
    },
    {
      "epoch": 1.0896376870859572,
      "grad_norm": 5.459132671356201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20452378690242767,
      "logits/rejected": -0.3262578248977661,
      "logps/chosen": -454.3132019042969,
      "logps/rejected": -521.76220703125,
      "loss": 0.3309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4901539087295532,
      "rewards/margins": 1.8158718347549438,
      "rewards/rejected": -3.306025743484497,
      "step": 8327
    },
    {
      "epoch": 1.089768545023309,
      "grad_norm": 8.622223854064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1278350055217743,
      "logits/rejected": -0.31677278876304626,
      "logps/chosen": -563.7736206054688,
      "logps/rejected": -386.6843566894531,
      "loss": 0.5084,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6635486483573914,
      "rewards/margins": 1.5136744976043701,
      "rewards/rejected": -2.1772232055664062,
      "step": 8328
    },
    {
      "epoch": 1.0898994029606608,
      "grad_norm": 10.665297508239746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06512178480625153,
      "logits/rejected": -0.3927639126777649,
      "logps/chosen": -386.62469482421875,
      "logps/rejected": -396.4736022949219,
      "loss": 0.8155,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.646502137184143,
      "rewards/margins": 0.37325233221054077,
      "rewards/rejected": -2.019754409790039,
      "step": 8329
    },
    {
      "epoch": 1.0900302608980126,
      "grad_norm": 5.686038494110107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23310337960720062,
      "logits/rejected": -0.3152729868888855,
      "logps/chosen": -423.64251708984375,
      "logps/rejected": -547.4781494140625,
      "loss": 0.3578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7521790266036987,
      "rewards/margins": 1.402331829071045,
      "rewards/rejected": -2.154510974884033,
      "step": 8330
    },
    {
      "epoch": 1.0901611188353644,
      "grad_norm": 4.850903034210205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5062496066093445,
      "logits/rejected": -0.332122802734375,
      "logps/chosen": -726.91259765625,
      "logps/rejected": -548.2947998046875,
      "loss": 0.2014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03322068601846695,
      "rewards/margins": 2.168778657913208,
      "rewards/rejected": -2.1355578899383545,
      "step": 8331
    },
    {
      "epoch": 1.0902919767727162,
      "grad_norm": 7.039819240570068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12496724724769592,
      "logits/rejected": -0.0764661431312561,
      "logps/chosen": -384.74432373046875,
      "logps/rejected": -321.2850646972656,
      "loss": 0.4685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5465189218521118,
      "rewards/margins": 1.0341739654541016,
      "rewards/rejected": -1.5806928873062134,
      "step": 8332
    },
    {
      "epoch": 1.090422834710068,
      "grad_norm": 9.272848129272461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.130834698677063,
      "logits/rejected": -0.13944849371910095,
      "logps/chosen": -704.7019653320312,
      "logps/rejected": -481.4852294921875,
      "loss": 0.505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4919476509094238,
      "rewards/margins": 1.1780624389648438,
      "rewards/rejected": -2.6700098514556885,
      "step": 8333
    },
    {
      "epoch": 1.0905536926474197,
      "grad_norm": 11.207695960998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3472227454185486,
      "logits/rejected": -0.5155054926872253,
      "logps/chosen": -435.0172119140625,
      "logps/rejected": -659.3665161132812,
      "loss": 0.351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.471349835395813,
      "rewards/margins": 1.3800959587097168,
      "rewards/rejected": -2.851445436477661,
      "step": 8334
    },
    {
      "epoch": 1.0906845505847713,
      "grad_norm": 7.169813632965088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24589262902736664,
      "logits/rejected": -0.2539787292480469,
      "logps/chosen": -772.9757080078125,
      "logps/rejected": -585.2470703125,
      "loss": 0.3234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9793784022331238,
      "rewards/margins": 1.2673826217651367,
      "rewards/rejected": -2.2467613220214844,
      "step": 8335
    },
    {
      "epoch": 1.090815408522123,
      "grad_norm": 6.367734432220459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16063159704208374,
      "logits/rejected": -0.4917854368686676,
      "logps/chosen": -291.4085388183594,
      "logps/rejected": -369.90350341796875,
      "loss": 0.4129,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0677160024642944,
      "rewards/margins": 1.516625165939331,
      "rewards/rejected": -2.584341287612915,
      "step": 8336
    },
    {
      "epoch": 1.090946266459475,
      "grad_norm": 5.228379726409912,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21135374903678894,
      "logits/rejected": -0.06564705818891525,
      "logps/chosen": -539.2255249023438,
      "logps/rejected": -502.4093017578125,
      "loss": 0.3017,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49256592988967896,
      "rewards/margins": 1.8484143018722534,
      "rewards/rejected": -2.340980291366577,
      "step": 8337
    },
    {
      "epoch": 1.0910771243968267,
      "grad_norm": 8.015172958374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32887136936187744,
      "logits/rejected": -0.26792120933532715,
      "logps/chosen": -553.1642456054688,
      "logps/rejected": -502.7463073730469,
      "loss": 0.4516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03717154264450073,
      "rewards/margins": 1.5994977951049805,
      "rewards/rejected": -1.6366692781448364,
      "step": 8338
    },
    {
      "epoch": 1.0912079823341785,
      "grad_norm": 7.661944389343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14160096645355225,
      "logits/rejected": -0.3902086615562439,
      "logps/chosen": -472.2463073730469,
      "logps/rejected": -350.4106750488281,
      "loss": 0.5302,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9671198129653931,
      "rewards/margins": 0.6270357370376587,
      "rewards/rejected": -1.5941554307937622,
      "step": 8339
    },
    {
      "epoch": 1.0913388402715303,
      "grad_norm": 6.192210674285889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20772872865200043,
      "logits/rejected": -0.35913753509521484,
      "logps/chosen": -395.3271789550781,
      "logps/rejected": -423.9235534667969,
      "loss": 0.297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1553866863250732,
      "rewards/margins": 1.9620306491851807,
      "rewards/rejected": -3.117417335510254,
      "step": 8340
    },
    {
      "epoch": 1.091469698208882,
      "grad_norm": 8.761845588684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2449023276567459,
      "logits/rejected": -0.09576264023780823,
      "logps/chosen": -463.3724365234375,
      "logps/rejected": -403.8976745605469,
      "loss": 0.3884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2664778232574463,
      "rewards/margins": 1.3655774593353271,
      "rewards/rejected": -2.6320550441741943,
      "step": 8341
    },
    {
      "epoch": 1.0916005561462336,
      "grad_norm": 14.744760513305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16873915493488312,
      "logits/rejected": -0.12725874781608582,
      "logps/chosen": -731.8555908203125,
      "logps/rejected": -415.3582763671875,
      "loss": 0.7825,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2262307405471802,
      "rewards/margins": 0.8069244623184204,
      "rewards/rejected": -2.0331549644470215,
      "step": 8342
    },
    {
      "epoch": 1.0917314140835854,
      "grad_norm": 8.525497436523438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42552605271339417,
      "logits/rejected": -0.3072689473628998,
      "logps/chosen": -411.1646728515625,
      "logps/rejected": -396.0351257324219,
      "loss": 0.543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8289037942886353,
      "rewards/margins": 1.2391178607940674,
      "rewards/rejected": -2.068021535873413,
      "step": 8343
    },
    {
      "epoch": 1.0918622720209372,
      "grad_norm": 8.130250930786133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23941512405872345,
      "logits/rejected": -0.10883830487728119,
      "logps/chosen": -579.0874633789062,
      "logps/rejected": -588.1087036132812,
      "loss": 0.6582,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13444212079048157,
      "rewards/margins": 1.0872786045074463,
      "rewards/rejected": -0.9528366923332214,
      "step": 8344
    },
    {
      "epoch": 1.091993129958289,
      "grad_norm": 6.924389839172363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020041659474372864,
      "logits/rejected": -0.13954703509807587,
      "logps/chosen": -644.1471557617188,
      "logps/rejected": -477.76470947265625,
      "loss": 0.208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46679502725601196,
      "rewards/margins": 2.2813024520874023,
      "rewards/rejected": -2.7480971813201904,
      "step": 8345
    },
    {
      "epoch": 1.0921239878956408,
      "grad_norm": 6.236490249633789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035988274961709976,
      "logits/rejected": -0.18108879029750824,
      "logps/chosen": -555.7565307617188,
      "logps/rejected": -517.1039428710938,
      "loss": 0.242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.521148681640625,
      "rewards/margins": 2.0203192234039307,
      "rewards/rejected": -2.5414676666259766,
      "step": 8346
    },
    {
      "epoch": 1.0922548458329926,
      "grad_norm": 8.807947158813477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18188774585723877,
      "logits/rejected": -0.15861614048480988,
      "logps/chosen": -478.71514892578125,
      "logps/rejected": -402.13372802734375,
      "loss": 0.5547,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2467622756958008,
      "rewards/margins": 0.6413041353225708,
      "rewards/rejected": -1.8880665302276611,
      "step": 8347
    },
    {
      "epoch": 1.0923857037703444,
      "grad_norm": 5.2073140144348145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0019401945173740387,
      "logits/rejected": 0.012798361480236053,
      "logps/chosen": -405.1584777832031,
      "logps/rejected": -366.021484375,
      "loss": 0.3665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0935128927230835,
      "rewards/margins": 1.124341368675232,
      "rewards/rejected": -2.2178542613983154,
      "step": 8348
    },
    {
      "epoch": 1.0925165617076962,
      "grad_norm": 5.924165725708008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13052216172218323,
      "logits/rejected": 0.02234628237783909,
      "logps/chosen": -436.1731872558594,
      "logps/rejected": -445.59759521484375,
      "loss": 0.1785,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.123305082321167,
      "rewards/margins": 2.216522455215454,
      "rewards/rejected": -3.339827537536621,
      "step": 8349
    },
    {
      "epoch": 1.0926474196450477,
      "grad_norm": 8.045061111450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005994419567286968,
      "logits/rejected": 0.027373619377613068,
      "logps/chosen": -413.6114501953125,
      "logps/rejected": -330.350830078125,
      "loss": 0.8563,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6874380111694336,
      "rewards/margins": 0.41850221157073975,
      "rewards/rejected": -1.1059402227401733,
      "step": 8350
    },
    {
      "epoch": 1.0927782775823995,
      "grad_norm": 5.5122389793396,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013328876346349716,
      "logits/rejected": -0.07703179121017456,
      "logps/chosen": -446.21087646484375,
      "logps/rejected": -485.99749755859375,
      "loss": 0.3997,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4010519981384277,
      "rewards/margins": 1.0825440883636475,
      "rewards/rejected": -2.483596086502075,
      "step": 8351
    },
    {
      "epoch": 1.0929091355197513,
      "grad_norm": 9.530871391296387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3545677661895752,
      "logits/rejected": -0.39721497893333435,
      "logps/chosen": -619.686279296875,
      "logps/rejected": -465.124267578125,
      "loss": 0.5952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.321001797914505,
      "rewards/margins": 0.4652157127857208,
      "rewards/rejected": -0.786217451095581,
      "step": 8352
    },
    {
      "epoch": 1.0930399934571031,
      "grad_norm": 10.932955741882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021121246740221977,
      "logits/rejected": -0.04989717900753021,
      "logps/chosen": -533.3228759765625,
      "logps/rejected": -632.9348754882812,
      "loss": 0.6767,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9528293013572693,
      "rewards/margins": 0.6658168435096741,
      "rewards/rejected": -1.618646264076233,
      "step": 8353
    },
    {
      "epoch": 1.093170851394455,
      "grad_norm": 5.873958587646484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042301103472709656,
      "logits/rejected": -0.11698371917009354,
      "logps/chosen": -405.4839172363281,
      "logps/rejected": -342.431640625,
      "loss": 0.3601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8577708601951599,
      "rewards/margins": 1.1471362113952637,
      "rewards/rejected": -2.0049071311950684,
      "step": 8354
    },
    {
      "epoch": 1.0933017093318067,
      "grad_norm": 13.397357940673828,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05232393369078636,
      "logits/rejected": -0.4195352792739868,
      "logps/chosen": -432.3752136230469,
      "logps/rejected": -425.3500061035156,
      "loss": 0.646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9336458444595337,
      "rewards/margins": 0.5947624444961548,
      "rewards/rejected": -1.528408169746399,
      "step": 8355
    },
    {
      "epoch": 1.0934325672691585,
      "grad_norm": 17.25537109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.179066002368927,
      "logits/rejected": -0.4142823815345764,
      "logps/chosen": -500.16070556640625,
      "logps/rejected": -369.3727722167969,
      "loss": 0.8191,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4344301223754883,
      "rewards/margins": 0.5524961948394775,
      "rewards/rejected": -1.9869264364242554,
      "step": 8356
    },
    {
      "epoch": 1.09356342520651,
      "grad_norm": 9.352320671081543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25698649883270264,
      "logits/rejected": -0.29539570212364197,
      "logps/chosen": -421.2426452636719,
      "logps/rejected": -378.2587585449219,
      "loss": 0.4372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9313684701919556,
      "rewards/margins": 1.2502110004425049,
      "rewards/rejected": -2.181579351425171,
      "step": 8357
    },
    {
      "epoch": 1.0936942831438619,
      "grad_norm": 10.164528846740723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1360691487789154,
      "logits/rejected": 0.03585198521614075,
      "logps/chosen": -560.1908569335938,
      "logps/rejected": -403.0611267089844,
      "loss": 0.5582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4892559051513672,
      "rewards/margins": 1.7432960271835327,
      "rewards/rejected": -2.2325520515441895,
      "step": 8358
    },
    {
      "epoch": 1.0938251410812136,
      "grad_norm": 6.414652347564697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04211506247520447,
      "logits/rejected": 0.10904975235462189,
      "logps/chosen": -357.8280029296875,
      "logps/rejected": -409.72259521484375,
      "loss": 0.3145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2968740165233612,
      "rewards/margins": 2.3563811779022217,
      "rewards/rejected": -2.6532552242279053,
      "step": 8359
    },
    {
      "epoch": 1.0939559990185654,
      "grad_norm": 6.853203773498535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0846988633275032,
      "logits/rejected": -0.3463413715362549,
      "logps/chosen": -481.46295166015625,
      "logps/rejected": -563.0560913085938,
      "loss": 0.3069,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4361206889152527,
      "rewards/margins": 1.446753978729248,
      "rewards/rejected": -1.0106333494186401,
      "step": 8360
    },
    {
      "epoch": 1.0940868569559172,
      "grad_norm": 9.84603500366211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.155967578291893,
      "logits/rejected": -0.3882867097854614,
      "logps/chosen": -540.6248168945312,
      "logps/rejected": -500.1348876953125,
      "loss": 0.5489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0657869577407837,
      "rewards/margins": 0.8525792360305786,
      "rewards/rejected": -1.9183661937713623,
      "step": 8361
    },
    {
      "epoch": 1.094217714893269,
      "grad_norm": 6.295225143432617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.000714421272277832,
      "logits/rejected": -0.24787631630897522,
      "logps/chosen": -640.1785888671875,
      "logps/rejected": -532.5580444335938,
      "loss": 0.347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29950523376464844,
      "rewards/margins": 1.4829676151275635,
      "rewards/rejected": -1.7824726104736328,
      "step": 8362
    },
    {
      "epoch": 1.0943485728306208,
      "grad_norm": 4.1823272705078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2930034101009369,
      "logits/rejected": -0.1468542218208313,
      "logps/chosen": -484.3807678222656,
      "logps/rejected": -492.046630859375,
      "loss": 0.2109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15463927388191223,
      "rewards/margins": 2.0449471473693848,
      "rewards/rejected": -2.1995863914489746,
      "step": 8363
    },
    {
      "epoch": 1.0944794307679726,
      "grad_norm": 7.729499816894531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17176643013954163,
      "logits/rejected": -0.06339114159345627,
      "logps/chosen": -420.14617919921875,
      "logps/rejected": -425.0242004394531,
      "loss": 0.5864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7531561255455017,
      "rewards/margins": 0.4010888934135437,
      "rewards/rejected": -1.1542450189590454,
      "step": 8364
    },
    {
      "epoch": 1.0946102887053244,
      "grad_norm": 4.706833362579346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23619206249713898,
      "logits/rejected": -0.301551878452301,
      "logps/chosen": -640.7210083007812,
      "logps/rejected": -774.7395629882812,
      "loss": 0.2229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.034099966287612915,
      "rewards/margins": 1.4675639867782593,
      "rewards/rejected": -1.5016639232635498,
      "step": 8365
    },
    {
      "epoch": 1.094741146642676,
      "grad_norm": 6.651069641113281,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30345094203948975,
      "logits/rejected": 0.13131988048553467,
      "logps/chosen": -596.5523071289062,
      "logps/rejected": -658.2294311523438,
      "loss": 0.2867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5575242042541504,
      "rewards/margins": 2.381549596786499,
      "rewards/rejected": -2.9390738010406494,
      "step": 8366
    },
    {
      "epoch": 1.0948720045800278,
      "grad_norm": 6.695866107940674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17734931409358978,
      "logits/rejected": -0.20106306672096252,
      "logps/chosen": -530.9742431640625,
      "logps/rejected": -573.38916015625,
      "loss": 0.3902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.35921138525009155,
      "rewards/margins": 1.0938996076583862,
      "rewards/rejected": -0.7346882224082947,
      "step": 8367
    },
    {
      "epoch": 1.0950028625173795,
      "grad_norm": 4.114150524139404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08901379257440567,
      "logits/rejected": -0.024948369711637497,
      "logps/chosen": -427.06243896484375,
      "logps/rejected": -405.3865966796875,
      "loss": 0.2469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15565931797027588,
      "rewards/margins": 1.9509556293487549,
      "rewards/rejected": -2.106614828109741,
      "step": 8368
    },
    {
      "epoch": 1.0951337204547313,
      "grad_norm": 7.718247890472412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3428151309490204,
      "logits/rejected": -0.3672678768634796,
      "logps/chosen": -418.5186462402344,
      "logps/rejected": -339.6557922363281,
      "loss": 0.486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7126095294952393,
      "rewards/margins": 1.4621455669403076,
      "rewards/rejected": -2.174755096435547,
      "step": 8369
    },
    {
      "epoch": 1.0952645783920831,
      "grad_norm": 5.601639270782471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.324555903673172,
      "logits/rejected": -0.502548098564148,
      "logps/chosen": -463.0464782714844,
      "logps/rejected": -348.7881774902344,
      "loss": 0.388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019669055938720703,
      "rewards/margins": 1.4604718685150146,
      "rewards/rejected": -1.4408029317855835,
      "step": 8370
    },
    {
      "epoch": 1.095395436329435,
      "grad_norm": 5.723066329956055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41030481457710266,
      "logits/rejected": -0.610193133354187,
      "logps/chosen": -444.8336181640625,
      "logps/rejected": -352.658203125,
      "loss": 0.3618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2678329646587372,
      "rewards/margins": 1.4633897542953491,
      "rewards/rejected": -1.7312226295471191,
      "step": 8371
    },
    {
      "epoch": 1.0955262942667867,
      "grad_norm": 6.89874267578125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0657232478260994,
      "logits/rejected": 0.05638216435909271,
      "logps/chosen": -352.779541015625,
      "logps/rejected": -358.80975341796875,
      "loss": 0.3512,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.040597379207611084,
      "rewards/margins": 1.853210687637329,
      "rewards/rejected": -1.893808126449585,
      "step": 8372
    },
    {
      "epoch": 1.0956571522041383,
      "grad_norm": 8.199677467346191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24805553257465363,
      "logits/rejected": 0.13976426422595978,
      "logps/chosen": -474.3997802734375,
      "logps/rejected": -500.6144104003906,
      "loss": 0.413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4374426007270813,
      "rewards/margins": 1.128911018371582,
      "rewards/rejected": -1.566353678703308,
      "step": 8373
    },
    {
      "epoch": 1.09578801014149,
      "grad_norm": 10.136716842651367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35097700357437134,
      "logits/rejected": -0.43564173579216003,
      "logps/chosen": -398.6607360839844,
      "logps/rejected": -370.6939392089844,
      "loss": 0.6755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7285820245742798,
      "rewards/margins": 0.7219012975692749,
      "rewards/rejected": -1.4504833221435547,
      "step": 8374
    },
    {
      "epoch": 1.0959188680788419,
      "grad_norm": 7.5995917320251465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12900900840759277,
      "logits/rejected": -0.1578933596611023,
      "logps/chosen": -423.968017578125,
      "logps/rejected": -422.2065124511719,
      "loss": 0.5982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9980415105819702,
      "rewards/margins": 0.5950714945793152,
      "rewards/rejected": -1.5931129455566406,
      "step": 8375
    },
    {
      "epoch": 1.0960497260161937,
      "grad_norm": 13.363483428955078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21133732795715332,
      "logits/rejected": -0.16435743868350983,
      "logps/chosen": -556.8362426757812,
      "logps/rejected": -620.5564575195312,
      "loss": 0.6689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18636341392993927,
      "rewards/margins": 0.6305555701255798,
      "rewards/rejected": -0.444192111492157,
      "step": 8376
    },
    {
      "epoch": 1.0961805839535455,
      "grad_norm": 9.677311897277832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1742398738861084,
      "logits/rejected": 0.0916110947728157,
      "logps/chosen": -534.471435546875,
      "logps/rejected": -550.299072265625,
      "loss": 0.6436,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7990684509277344,
      "rewards/margins": 0.412689208984375,
      "rewards/rejected": -1.2117576599121094,
      "step": 8377
    },
    {
      "epoch": 1.0963114418908972,
      "grad_norm": 8.175707817077637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08955521881580353,
      "logits/rejected": -0.1704373061656952,
      "logps/chosen": -459.1994934082031,
      "logps/rejected": -496.9740295410156,
      "loss": 0.5743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6154782772064209,
      "rewards/margins": 0.9430824518203735,
      "rewards/rejected": -1.558560848236084,
      "step": 8378
    },
    {
      "epoch": 1.096442299828249,
      "grad_norm": 9.197779655456543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28465211391448975,
      "logits/rejected": -0.317064493894577,
      "logps/chosen": -371.1977233886719,
      "logps/rejected": -537.373046875,
      "loss": 0.642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0162594318389893,
      "rewards/margins": 0.43080615997314453,
      "rewards/rejected": -1.4470655918121338,
      "step": 8379
    },
    {
      "epoch": 1.0965731577656008,
      "grad_norm": 5.4614949226379395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09342770278453827,
      "logits/rejected": -0.19217002391815186,
      "logps/chosen": -513.3845825195312,
      "logps/rejected": -421.7982177734375,
      "loss": 0.3454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6808164715766907,
      "rewards/margins": 0.9960987567901611,
      "rewards/rejected": -0.31528228521347046,
      "step": 8380
    },
    {
      "epoch": 1.0967040157029524,
      "grad_norm": 8.439404487609863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2732710540294647,
      "logits/rejected": 0.25354433059692383,
      "logps/chosen": -426.4486083984375,
      "logps/rejected": -429.17694091796875,
      "loss": 0.5568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5223320126533508,
      "rewards/margins": 0.73788982629776,
      "rewards/rejected": -1.2602217197418213,
      "step": 8381
    },
    {
      "epoch": 1.0968348736403042,
      "grad_norm": 6.232144832611084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.5625478625297546,
      "logits/rejected": 0.02942918986082077,
      "logps/chosen": -405.7935791015625,
      "logps/rejected": -343.8616027832031,
      "loss": 0.2672,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3233732283115387,
      "rewards/margins": 1.81630539894104,
      "rewards/rejected": -1.4929324388504028,
      "step": 8382
    },
    {
      "epoch": 1.096965731577656,
      "grad_norm": 15.90096664428711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21346233785152435,
      "logits/rejected": -0.09147103875875473,
      "logps/chosen": -556.0477905273438,
      "logps/rejected": -720.3382568359375,
      "loss": 0.8929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6650761365890503,
      "rewards/margins": 0.18512004613876343,
      "rewards/rejected": -0.8501961827278137,
      "step": 8383
    },
    {
      "epoch": 1.0970965895150078,
      "grad_norm": 7.266566753387451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13078761100769043,
      "logits/rejected": -0.10654513537883759,
      "logps/chosen": -455.0638427734375,
      "logps/rejected": -400.5501403808594,
      "loss": 0.5038,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.005406565964221954,
      "rewards/margins": 1.5111138820648193,
      "rewards/rejected": -1.5165205001831055,
      "step": 8384
    },
    {
      "epoch": 1.0972274474523596,
      "grad_norm": 8.13133430480957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07865916192531586,
      "logits/rejected": 0.2730082869529724,
      "logps/chosen": -508.77362060546875,
      "logps/rejected": -582.5228271484375,
      "loss": 0.499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06297758966684341,
      "rewards/margins": 0.6759382486343384,
      "rewards/rejected": -0.612960696220398,
      "step": 8385
    },
    {
      "epoch": 1.0973583053897114,
      "grad_norm": 5.880058765411377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14515702426433563,
      "logits/rejected": 0.017700236290693283,
      "logps/chosen": -422.71728515625,
      "logps/rejected": -513.2363891601562,
      "loss": 0.248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4418686032295227,
      "rewards/margins": 1.744513750076294,
      "rewards/rejected": -2.186382293701172,
      "step": 8386
    },
    {
      "epoch": 1.0974891633270631,
      "grad_norm": 10.789767265319824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09130090475082397,
      "logits/rejected": -0.2532240152359009,
      "logps/chosen": -467.7532043457031,
      "logps/rejected": -515.36669921875,
      "loss": 0.6166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6721204519271851,
      "rewards/margins": 0.8708325624465942,
      "rewards/rejected": -1.5429528951644897,
      "step": 8387
    },
    {
      "epoch": 1.0976200212644147,
      "grad_norm": 10.713300704956055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.036289773881435394,
      "logits/rejected": -0.034220222383737564,
      "logps/chosen": -502.7847900390625,
      "logps/rejected": -478.884033203125,
      "loss": 0.5444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.522530198097229,
      "rewards/margins": 0.6555683612823486,
      "rewards/rejected": -1.1780985593795776,
      "step": 8388
    },
    {
      "epoch": 1.0977508792017665,
      "grad_norm": 5.639734745025635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10379450023174286,
      "logits/rejected": -0.11854317784309387,
      "logps/chosen": -412.58282470703125,
      "logps/rejected": -413.7012634277344,
      "loss": 0.3878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5420166850090027,
      "rewards/margins": 1.5579560995101929,
      "rewards/rejected": -2.099972724914551,
      "step": 8389
    },
    {
      "epoch": 1.0978817371391183,
      "grad_norm": 9.557685852050781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022974250838160515,
      "logits/rejected": 0.0027196742594242096,
      "logps/chosen": -506.72955322265625,
      "logps/rejected": -503.4589538574219,
      "loss": 0.5273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2928731441497803,
      "rewards/margins": 0.9094193577766418,
      "rewards/rejected": -2.2022926807403564,
      "step": 8390
    },
    {
      "epoch": 1.09801259507647,
      "grad_norm": 4.1160807609558105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08182045072317123,
      "logits/rejected": 0.02552364021539688,
      "logps/chosen": -335.6939697265625,
      "logps/rejected": -447.0836181640625,
      "loss": 0.2369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.169420063495636,
      "rewards/margins": 2.130363702774048,
      "rewards/rejected": -2.299783706665039,
      "step": 8391
    },
    {
      "epoch": 1.0981434530138219,
      "grad_norm": 7.386931419372559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4757276177406311,
      "logits/rejected": -0.2763296365737915,
      "logps/chosen": -627.9609375,
      "logps/rejected": -655.1275634765625,
      "loss": 0.3464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23556940257549286,
      "rewards/margins": 1.8821150064468384,
      "rewards/rejected": -1.646545648574829,
      "step": 8392
    },
    {
      "epoch": 1.0982743109511737,
      "grad_norm": 7.529964923858643,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09510848671197891,
      "logits/rejected": 0.11658736318349838,
      "logps/chosen": -499.9471435546875,
      "logps/rejected": -404.58721923828125,
      "loss": 0.4866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10053541511297226,
      "rewards/margins": 0.82398921251297,
      "rewards/rejected": -0.9245246648788452,
      "step": 8393
    },
    {
      "epoch": 1.0984051688885255,
      "grad_norm": 6.648439884185791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1731196939945221,
      "logits/rejected": -0.004677727818489075,
      "logps/chosen": -434.8953857421875,
      "logps/rejected": -414.31976318359375,
      "loss": 0.4026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12130222469568253,
      "rewards/margins": 1.2647881507873535,
      "rewards/rejected": -1.3860903978347778,
      "step": 8394
    },
    {
      "epoch": 1.0985360268258773,
      "grad_norm": 3.2498464584350586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23600634932518005,
      "logits/rejected": -0.009084828197956085,
      "logps/chosen": -524.3329467773438,
      "logps/rejected": -399.53143310546875,
      "loss": 0.1898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.672863781452179,
      "rewards/margins": 1.9231593608856201,
      "rewards/rejected": -2.5960230827331543,
      "step": 8395
    },
    {
      "epoch": 1.0986668847632288,
      "grad_norm": 6.277329444885254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12866146862506866,
      "logits/rejected": -0.27904218435287476,
      "logps/chosen": -502.6898498535156,
      "logps/rejected": -422.95196533203125,
      "loss": 0.2627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29299110174179077,
      "rewards/margins": 1.846564531326294,
      "rewards/rejected": -2.1395554542541504,
      "step": 8396
    },
    {
      "epoch": 1.0987977427005806,
      "grad_norm": 9.164114952087402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08726881444454193,
      "logits/rejected": -0.24550902843475342,
      "logps/chosen": -448.0671691894531,
      "logps/rejected": -488.9285583496094,
      "loss": 0.5203,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10084020346403122,
      "rewards/margins": 0.8880962133407593,
      "rewards/rejected": -0.7872560620307922,
      "step": 8397
    },
    {
      "epoch": 1.0989286006379324,
      "grad_norm": 9.0650634765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1156802847981453,
      "logits/rejected": -0.3404527008533478,
      "logps/chosen": -400.46563720703125,
      "logps/rejected": -371.2679443359375,
      "loss": 0.6478,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0554077625274658,
      "rewards/margins": 0.364802747964859,
      "rewards/rejected": -1.4202104806900024,
      "step": 8398
    },
    {
      "epoch": 1.0990594585752842,
      "grad_norm": 8.118779182434082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043832335621118546,
      "logits/rejected": -0.1394270360469818,
      "logps/chosen": -466.2712097167969,
      "logps/rejected": -648.6409912109375,
      "loss": 0.5523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.992405891418457,
      "rewards/margins": 1.7411653995513916,
      "rewards/rejected": -2.7335712909698486,
      "step": 8399
    },
    {
      "epoch": 1.099190316512636,
      "grad_norm": 4.950045585632324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028849104419350624,
      "logits/rejected": -0.11341144144535065,
      "logps/chosen": -286.1401672363281,
      "logps/rejected": -369.72503662109375,
      "loss": 0.2313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6323606967926025,
      "rewards/margins": 2.2128186225891113,
      "rewards/rejected": -2.8451790809631348,
      "step": 8400
    },
    {
      "epoch": 1.0993211744499878,
      "grad_norm": 9.523983001708984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14288106560707092,
      "logits/rejected": -0.268586665391922,
      "logps/chosen": -594.4672241210938,
      "logps/rejected": -448.63275146484375,
      "loss": 0.4145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7466888427734375,
      "rewards/margins": 2.5312485694885254,
      "rewards/rejected": -3.277937412261963,
      "step": 8401
    },
    {
      "epoch": 1.0994520323873396,
      "grad_norm": 8.589988708496094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09638738632202148,
      "logits/rejected": 0.0735572800040245,
      "logps/chosen": -504.6614990234375,
      "logps/rejected": -315.2618103027344,
      "loss": 0.3388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5327603220939636,
      "rewards/margins": 1.9320502281188965,
      "rewards/rejected": -2.464810609817505,
      "step": 8402
    },
    {
      "epoch": 1.0995828903246911,
      "grad_norm": 6.547765731811523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14658017456531525,
      "logits/rejected": -0.2742373049259186,
      "logps/chosen": -440.7787170410156,
      "logps/rejected": -450.1215515136719,
      "loss": 0.4078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1808265894651413,
      "rewards/margins": 1.3564934730529785,
      "rewards/rejected": -1.1756668090820312,
      "step": 8403
    },
    {
      "epoch": 1.099713748262043,
      "grad_norm": 8.2346773147583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02853453904390335,
      "logits/rejected": -0.06901314109563828,
      "logps/chosen": -417.2455139160156,
      "logps/rejected": -584.15478515625,
      "loss": 0.5723,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7028059363365173,
      "rewards/margins": 0.6084190607070923,
      "rewards/rejected": -1.3112249374389648,
      "step": 8404
    },
    {
      "epoch": 1.0998446061993947,
      "grad_norm": 8.662870407104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019905388355255127,
      "logits/rejected": -0.0869804322719574,
      "logps/chosen": -601.3145751953125,
      "logps/rejected": -645.908203125,
      "loss": 0.4632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12065869569778442,
      "rewards/margins": 1.6228173971176147,
      "rewards/rejected": -1.743476152420044,
      "step": 8405
    },
    {
      "epoch": 1.0999754641367465,
      "grad_norm": 8.387184143066406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.318812757730484,
      "logits/rejected": -0.16639620065689087,
      "logps/chosen": -309.40899658203125,
      "logps/rejected": -381.76434326171875,
      "loss": 0.6739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6932629346847534,
      "rewards/margins": 1.412986397743225,
      "rewards/rejected": -2.1062493324279785,
      "step": 8406
    },
    {
      "epoch": 1.1001063220740983,
      "grad_norm": 4.9792656898498535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011910729110240936,
      "logits/rejected": -0.05425264686346054,
      "logps/chosen": -479.30706787109375,
      "logps/rejected": -515.8282470703125,
      "loss": 0.2696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.087706208229065,
      "rewards/margins": 1.3857150077819824,
      "rewards/rejected": -2.473421096801758,
      "step": 8407
    },
    {
      "epoch": 1.10023718001145,
      "grad_norm": 5.173338890075684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0013357102870941162,
      "logits/rejected": 0.03340553492307663,
      "logps/chosen": -490.830078125,
      "logps/rejected": -563.4400634765625,
      "loss": 0.3132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8054237365722656,
      "rewards/margins": 1.7058483362197876,
      "rewards/rejected": -2.5112721920013428,
      "step": 8408
    },
    {
      "epoch": 1.100368037948802,
      "grad_norm": 9.440553665161133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12757427990436554,
      "logits/rejected": -0.14217033982276917,
      "logps/chosen": -535.7181396484375,
      "logps/rejected": -404.7933654785156,
      "loss": 0.5045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23344501852989197,
      "rewards/margins": 1.2163244485855103,
      "rewards/rejected": -1.449769377708435,
      "step": 8409
    },
    {
      "epoch": 1.1004988958861537,
      "grad_norm": 9.319534301757812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2278004288673401,
      "logits/rejected": -0.29410499334335327,
      "logps/chosen": -497.4486999511719,
      "logps/rejected": -425.5276184082031,
      "loss": 0.6955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2933962047100067,
      "rewards/margins": 0.7790890336036682,
      "rewards/rejected": -1.0724852085113525,
      "step": 8410
    },
    {
      "epoch": 1.1006297538235055,
      "grad_norm": 6.187548637390137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0495978482067585,
      "logits/rejected": -0.002162378281354904,
      "logps/chosen": -335.1278381347656,
      "logps/rejected": -303.7328186035156,
      "loss": 0.3919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08041457086801529,
      "rewards/margins": 1.0709831714630127,
      "rewards/rejected": -0.9905686378479004,
      "step": 8411
    },
    {
      "epoch": 1.100760611760857,
      "grad_norm": 7.049903869628906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06712216138839722,
      "logits/rejected": -0.16514478623867035,
      "logps/chosen": -585.0391845703125,
      "logps/rejected": -475.64337158203125,
      "loss": 0.3982,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2415388524532318,
      "rewards/margins": 1.4973750114440918,
      "rewards/rejected": -1.2558363676071167,
      "step": 8412
    },
    {
      "epoch": 1.1008914696982088,
      "grad_norm": 6.270900249481201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1338878571987152,
      "logits/rejected": -0.4248404800891876,
      "logps/chosen": -493.9134826660156,
      "logps/rejected": -414.0547790527344,
      "loss": 0.3443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3778226673603058,
      "rewards/margins": 2.2061355113983154,
      "rewards/rejected": -2.583958387374878,
      "step": 8413
    },
    {
      "epoch": 1.1010223276355606,
      "grad_norm": 4.9510674476623535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4481961131095886,
      "logits/rejected": -0.29565832018852234,
      "logps/chosen": -350.3056640625,
      "logps/rejected": -331.5412902832031,
      "loss": 0.3828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1538868248462677,
      "rewards/margins": 1.1625492572784424,
      "rewards/rejected": -1.0086625814437866,
      "step": 8414
    },
    {
      "epoch": 1.1011531855729124,
      "grad_norm": 5.675812721252441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.170728862285614,
      "logits/rejected": -0.19370490312576294,
      "logps/chosen": -384.8824157714844,
      "logps/rejected": -397.1915588378906,
      "loss": 0.3219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10804043710231781,
      "rewards/margins": 1.7829302549362183,
      "rewards/rejected": -1.8909707069396973,
      "step": 8415
    },
    {
      "epoch": 1.1012840435102642,
      "grad_norm": 7.783021926879883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025188030675053596,
      "logits/rejected": 0.03443531692028046,
      "logps/chosen": -421.9162292480469,
      "logps/rejected": -475.8612365722656,
      "loss": 0.4247,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35538941621780396,
      "rewards/margins": 1.114243507385254,
      "rewards/rejected": -1.469632863998413,
      "step": 8416
    },
    {
      "epoch": 1.101414901447616,
      "grad_norm": 9.28632926940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14041070640087128,
      "logits/rejected": -0.2603561282157898,
      "logps/chosen": -285.3647766113281,
      "logps/rejected": -412.50396728515625,
      "loss": 0.605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5621800422668457,
      "rewards/margins": 1.4557342529296875,
      "rewards/rejected": -2.017914295196533,
      "step": 8417
    },
    {
      "epoch": 1.1015457593849678,
      "grad_norm": 6.042771339416504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38180291652679443,
      "logits/rejected": -0.2790687084197998,
      "logps/chosen": -348.3705139160156,
      "logps/rejected": -373.7732238769531,
      "loss": 0.3821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30321332812309265,
      "rewards/margins": 2.6901767253875732,
      "rewards/rejected": -2.9933900833129883,
      "step": 8418
    },
    {
      "epoch": 1.1016766173223194,
      "grad_norm": 7.967164039611816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31904253363609314,
      "logits/rejected": -0.5036435723304749,
      "logps/chosen": -510.3992004394531,
      "logps/rejected": -405.8554992675781,
      "loss": 0.4337,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9319427013397217,
      "rewards/margins": 1.3052623271942139,
      "rewards/rejected": -2.2372047901153564,
      "step": 8419
    },
    {
      "epoch": 1.1018074752596712,
      "grad_norm": 11.738471984863281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18487648665905,
      "logits/rejected": -0.3920365273952484,
      "logps/chosen": -465.992431640625,
      "logps/rejected": -304.2417907714844,
      "loss": 0.7316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3153539896011353,
      "rewards/margins": 0.7851554155349731,
      "rewards/rejected": -2.1005096435546875,
      "step": 8420
    },
    {
      "epoch": 1.101938333197023,
      "grad_norm": 5.323921203613281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06295456737279892,
      "logits/rejected": -0.024891257286071777,
      "logps/chosen": -354.5615234375,
      "logps/rejected": -381.0958251953125,
      "loss": 0.3268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8837728500366211,
      "rewards/margins": 1.4528449773788452,
      "rewards/rejected": -2.3366177082061768,
      "step": 8421
    },
    {
      "epoch": 1.1020691911343747,
      "grad_norm": 7.180863857269287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08679106831550598,
      "logits/rejected": 0.02347181737422943,
      "logps/chosen": -480.1479187011719,
      "logps/rejected": -438.8014831542969,
      "loss": 0.3733,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5422981977462769,
      "rewards/margins": 1.5922878980636597,
      "rewards/rejected": -2.1345860958099365,
      "step": 8422
    },
    {
      "epoch": 1.1022000490717265,
      "grad_norm": 5.3737053871154785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07063540071249008,
      "logits/rejected": -0.2713125944137573,
      "logps/chosen": -458.8480224609375,
      "logps/rejected": -540.1033935546875,
      "loss": 0.321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20896360278129578,
      "rewards/margins": 2.353818893432617,
      "rewards/rejected": -2.5627827644348145,
      "step": 8423
    },
    {
      "epoch": 1.1023309070090783,
      "grad_norm": 8.29342269897461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1958695650100708,
      "logits/rejected": 0.1564675271511078,
      "logps/chosen": -467.3597412109375,
      "logps/rejected": -389.6789245605469,
      "loss": 0.452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6051878929138184,
      "rewards/margins": 1.1794633865356445,
      "rewards/rejected": -0.5742756128311157,
      "step": 8424
    },
    {
      "epoch": 1.1024617649464301,
      "grad_norm": 7.593052387237549,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09502080827951431,
      "logits/rejected": 0.32216766476631165,
      "logps/chosen": -586.0692138671875,
      "logps/rejected": -663.753173828125,
      "loss": 0.3738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2074189931154251,
      "rewards/margins": 1.3478097915649414,
      "rewards/rejected": -1.1403908729553223,
      "step": 8425
    },
    {
      "epoch": 1.102592622883782,
      "grad_norm": 11.015706062316895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12015517055988312,
      "logits/rejected": 0.016895819455385208,
      "logps/chosen": -472.695068359375,
      "logps/rejected": -439.72186279296875,
      "loss": 0.7163,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2415395975112915,
      "rewards/margins": 0.32194504141807556,
      "rewards/rejected": -1.5634845495224,
      "step": 8426
    },
    {
      "epoch": 1.1027234808211335,
      "grad_norm": 6.152841091156006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10192139446735382,
      "logits/rejected": -0.11872288584709167,
      "logps/chosen": -390.7216796875,
      "logps/rejected": -527.5394897460938,
      "loss": 0.3851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7021436095237732,
      "rewards/margins": 1.0314817428588867,
      "rewards/rejected": -1.7336255311965942,
      "step": 8427
    },
    {
      "epoch": 1.1028543387584853,
      "grad_norm": 4.970137596130371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16041719913482666,
      "logits/rejected": 0.3211960196495056,
      "logps/chosen": -492.361572265625,
      "logps/rejected": -505.486328125,
      "loss": 0.2232,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5714293122291565,
      "rewards/margins": 1.677495002746582,
      "rewards/rejected": -2.2489242553710938,
      "step": 8428
    },
    {
      "epoch": 1.102985196695837,
      "grad_norm": 4.855069160461426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02056184411048889,
      "logits/rejected": 0.28881311416625977,
      "logps/chosen": -334.49688720703125,
      "logps/rejected": -515.5767822265625,
      "loss": 0.286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2072322368621826,
      "rewards/margins": 1.6013084650039673,
      "rewards/rejected": -2.8085408210754395,
      "step": 8429
    },
    {
      "epoch": 1.1031160546331888,
      "grad_norm": 5.56134557723999,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1469608098268509,
      "logits/rejected": 0.05330853909254074,
      "logps/chosen": -466.899169921875,
      "logps/rejected": -341.4554138183594,
      "loss": 0.4391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5448015332221985,
      "rewards/margins": 1.6191082000732422,
      "rewards/rejected": -2.163909912109375,
      "step": 8430
    },
    {
      "epoch": 1.1032469125705406,
      "grad_norm": 8.554414749145508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13490168750286102,
      "logits/rejected": -0.06291612237691879,
      "logps/chosen": -342.487548828125,
      "logps/rejected": -519.5625,
      "loss": 0.4869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7399922013282776,
      "rewards/margins": 1.3218250274658203,
      "rewards/rejected": -2.061817169189453,
      "step": 8431
    },
    {
      "epoch": 1.1033777705078924,
      "grad_norm": 12.339951515197754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.000833313912153244,
      "logits/rejected": 0.09379993379116058,
      "logps/chosen": -620.3693237304688,
      "logps/rejected": -358.34661865234375,
      "loss": 0.4306,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7990795373916626,
      "rewards/margins": 1.1439015865325928,
      "rewards/rejected": -1.942981243133545,
      "step": 8432
    },
    {
      "epoch": 1.1035086284452442,
      "grad_norm": 4.789804935455322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18007241189479828,
      "logits/rejected": -0.059713833034038544,
      "logps/chosen": -561.9642944335938,
      "logps/rejected": -542.838623046875,
      "loss": 0.1892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37264901399612427,
      "rewards/margins": 2.192857265472412,
      "rewards/rejected": -1.8202084302902222,
      "step": 8433
    },
    {
      "epoch": 1.1036394863825958,
      "grad_norm": 6.411264419555664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01906641200184822,
      "logits/rejected": -0.029493091627955437,
      "logps/chosen": -592.9193115234375,
      "logps/rejected": -607.3098754882812,
      "loss": 0.3182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33516043424606323,
      "rewards/margins": 1.4662728309631348,
      "rewards/rejected": -1.8014332056045532,
      "step": 8434
    },
    {
      "epoch": 1.1037703443199476,
      "grad_norm": 9.178361892700195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021303659304976463,
      "logits/rejected": -0.13831639289855957,
      "logps/chosen": -444.62750244140625,
      "logps/rejected": -473.32635498046875,
      "loss": 0.5674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6179420351982117,
      "rewards/margins": 1.3246420621871948,
      "rewards/rejected": -1.9425841569900513,
      "step": 8435
    },
    {
      "epoch": 1.1039012022572994,
      "grad_norm": 7.9698286056518555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11331058293581009,
      "logits/rejected": -0.3466518223285675,
      "logps/chosen": -375.50628662109375,
      "logps/rejected": -406.17767333984375,
      "loss": 0.3774,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4104405641555786,
      "rewards/margins": 1.1968209743499756,
      "rewards/rejected": -2.6072614192962646,
      "step": 8436
    },
    {
      "epoch": 1.1040320601946512,
      "grad_norm": 3.8151676654815674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05067606270313263,
      "logits/rejected": -0.26749637722969055,
      "logps/chosen": -366.9386901855469,
      "logps/rejected": -369.35009765625,
      "loss": 0.175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20797614753246307,
      "rewards/margins": 2.3250768184661865,
      "rewards/rejected": -2.533052921295166,
      "step": 8437
    },
    {
      "epoch": 1.104162918132003,
      "grad_norm": 7.838129997253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0016288645565509796,
      "logits/rejected": 0.1729249507188797,
      "logps/chosen": -422.93292236328125,
      "logps/rejected": -408.80572509765625,
      "loss": 0.4432,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6330841183662415,
      "rewards/margins": 1.1939977407455444,
      "rewards/rejected": -1.8270819187164307,
      "step": 8438
    },
    {
      "epoch": 1.1042937760693547,
      "grad_norm": 4.800224781036377,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019744936376810074,
      "logits/rejected": -0.07861700654029846,
      "logps/chosen": -449.67333984375,
      "logps/rejected": -454.65948486328125,
      "loss": 0.3205,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5956723690032959,
      "rewards/margins": 1.3891420364379883,
      "rewards/rejected": -0.7934696078300476,
      "step": 8439
    },
    {
      "epoch": 1.1044246340067065,
      "grad_norm": 9.873794555664062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15734587609767914,
      "logits/rejected": -0.11876991391181946,
      "logps/chosen": -457.73114013671875,
      "logps/rejected": -383.89111328125,
      "loss": 0.6932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7962653636932373,
      "rewards/margins": 0.9494810700416565,
      "rewards/rejected": -1.7457464933395386,
      "step": 8440
    },
    {
      "epoch": 1.1045554919440583,
      "grad_norm": 16.333097457885742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18222658336162567,
      "logits/rejected": 0.008995224721729755,
      "logps/chosen": -523.274658203125,
      "logps/rejected": -468.24755859375,
      "loss": 1.2191,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.9831879138946533,
      "rewards/margins": 0.10706856846809387,
      "rewards/rejected": -2.090256452560425,
      "step": 8441
    },
    {
      "epoch": 1.10468634988141,
      "grad_norm": 7.764911651611328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10598829388618469,
      "logits/rejected": -0.3427986800670624,
      "logps/chosen": -660.5439453125,
      "logps/rejected": -435.5760498046875,
      "loss": 0.4344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44183748960494995,
      "rewards/margins": 1.7611072063446045,
      "rewards/rejected": -1.3192695379257202,
      "step": 8442
    },
    {
      "epoch": 1.1048172078187617,
      "grad_norm": 10.462347030639648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13719330728054047,
      "logits/rejected": 0.1439763903617859,
      "logps/chosen": -420.1441345214844,
      "logps/rejected": -323.82000732421875,
      "loss": 0.5499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2437915802001953,
      "rewards/margins": 0.6305882334709167,
      "rewards/rejected": -1.8743797540664673,
      "step": 8443
    },
    {
      "epoch": 1.1049480657561135,
      "grad_norm": 5.432566165924072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40614810585975647,
      "logits/rejected": -0.28891193866729736,
      "logps/chosen": -541.1220092773438,
      "logps/rejected": -617.3405151367188,
      "loss": 0.2482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4805055856704712,
      "rewards/margins": 2.108205795288086,
      "rewards/rejected": -2.5887115001678467,
      "step": 8444
    },
    {
      "epoch": 1.1050789236934653,
      "grad_norm": 10.008963584899902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46858280897140503,
      "logits/rejected": -0.48866569995880127,
      "logps/chosen": -689.0883178710938,
      "logps/rejected": -465.7998046875,
      "loss": 0.4671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8598487377166748,
      "rewards/margins": 1.1171232461929321,
      "rewards/rejected": -1.9769721031188965,
      "step": 8445
    },
    {
      "epoch": 1.105209781630817,
      "grad_norm": 2.7177884578704834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07969687879085541,
      "logits/rejected": -0.2653680741786957,
      "logps/chosen": -362.180419921875,
      "logps/rejected": -498.0791931152344,
      "loss": 0.1075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5901010632514954,
      "rewards/margins": 2.4381673336029053,
      "rewards/rejected": -3.0282680988311768,
      "step": 8446
    },
    {
      "epoch": 1.1053406395681689,
      "grad_norm": 6.267631530761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38838493824005127,
      "logits/rejected": -0.3274030089378357,
      "logps/chosen": -416.2265319824219,
      "logps/rejected": -471.62628173828125,
      "loss": 0.4237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5260573625564575,
      "rewards/margins": 0.9901119470596313,
      "rewards/rejected": -1.5161693096160889,
      "step": 8447
    },
    {
      "epoch": 1.1054714975055207,
      "grad_norm": 7.691106796264648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29584360122680664,
      "logits/rejected": -0.09661370515823364,
      "logps/chosen": -581.0935668945312,
      "logps/rejected": -591.981689453125,
      "loss": 0.3542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0562469959259033,
      "rewards/margins": 1.0523297786712646,
      "rewards/rejected": -2.108576774597168,
      "step": 8448
    },
    {
      "epoch": 1.1056023554428722,
      "grad_norm": 4.229101181030273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2134079784154892,
      "logits/rejected": -0.079300656914711,
      "logps/chosen": -517.483642578125,
      "logps/rejected": -482.7991638183594,
      "loss": 0.2323,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27222979068756104,
      "rewards/margins": 1.9397964477539062,
      "rewards/rejected": -2.2120261192321777,
      "step": 8449
    },
    {
      "epoch": 1.105733213380224,
      "grad_norm": 5.5771870613098145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0693420022726059,
      "logits/rejected": -0.04477094113826752,
      "logps/chosen": -602.525146484375,
      "logps/rejected": -668.79931640625,
      "loss": 0.2405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5657658576965332,
      "rewards/margins": 2.264685869216919,
      "rewards/rejected": -1.6989198923110962,
      "step": 8450
    },
    {
      "epoch": 1.1058640713175758,
      "grad_norm": 6.026632308959961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1049969345331192,
      "logits/rejected": 0.18005284667015076,
      "logps/chosen": -333.4762878417969,
      "logps/rejected": -324.8053283691406,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34275683760643005,
      "rewards/margins": 1.0295439958572388,
      "rewards/rejected": -1.3723007440567017,
      "step": 8451
    },
    {
      "epoch": 1.1059949292549276,
      "grad_norm": 7.599752902984619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1925724297761917,
      "logits/rejected": -0.23602968454360962,
      "logps/chosen": -533.27490234375,
      "logps/rejected": -481.32086181640625,
      "loss": 0.5629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30136945843696594,
      "rewards/margins": 1.4768025875091553,
      "rewards/rejected": -1.7781718969345093,
      "step": 8452
    },
    {
      "epoch": 1.1061257871922794,
      "grad_norm": 5.896973609924316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13328897953033447,
      "logits/rejected": 0.1040675938129425,
      "logps/chosen": -396.92230224609375,
      "logps/rejected": -461.75714111328125,
      "loss": 0.2925,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3883209526538849,
      "rewards/margins": 1.8551745414733887,
      "rewards/rejected": -2.243495464324951,
      "step": 8453
    },
    {
      "epoch": 1.1062566451296312,
      "grad_norm": 12.23729419708252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5092069506645203,
      "logits/rejected": -0.5477288961410522,
      "logps/chosen": -331.939697265625,
      "logps/rejected": -465.78851318359375,
      "loss": 0.7711,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8680663108825684,
      "rewards/margins": -0.06907978653907776,
      "rewards/rejected": -1.7989864349365234,
      "step": 8454
    },
    {
      "epoch": 1.106387503066983,
      "grad_norm": 9.827001571655273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17758022248744965,
      "logits/rejected": 0.1495816856622696,
      "logps/chosen": -428.0478820800781,
      "logps/rejected": -449.14471435546875,
      "loss": 0.675,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9998393654823303,
      "rewards/margins": 0.3182833194732666,
      "rewards/rejected": -1.3181227445602417,
      "step": 8455
    },
    {
      "epoch": 1.1065183610043348,
      "grad_norm": 6.313000202178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006681017577648163,
      "logits/rejected": -0.14905433356761932,
      "logps/chosen": -693.473876953125,
      "logps/rejected": -526.0288696289062,
      "loss": 0.3363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6245394945144653,
      "rewards/margins": 2.454519748687744,
      "rewards/rejected": -1.8299802541732788,
      "step": 8456
    },
    {
      "epoch": 1.1066492189416863,
      "grad_norm": 9.004490852355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07430963963270187,
      "logits/rejected": -0.038792684674263,
      "logps/chosen": -509.9601135253906,
      "logps/rejected": -647.8372192382812,
      "loss": 0.6688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2045601606369019,
      "rewards/margins": 0.7709774971008301,
      "rewards/rejected": -1.975537896156311,
      "step": 8457
    },
    {
      "epoch": 1.1067800768790381,
      "grad_norm": 4.992410659790039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.051963530480861664,
      "logits/rejected": 0.07493074983358383,
      "logps/chosen": -302.4360656738281,
      "logps/rejected": -344.3453674316406,
      "loss": 0.2746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5653707981109619,
      "rewards/margins": 1.801876187324524,
      "rewards/rejected": -2.3672468662261963,
      "step": 8458
    },
    {
      "epoch": 1.10691093481639,
      "grad_norm": 6.672609329223633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019188471138477325,
      "logits/rejected": -0.20050078630447388,
      "logps/chosen": -394.21234130859375,
      "logps/rejected": -311.4134216308594,
      "loss": 0.5979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0114752054214478,
      "rewards/margins": 0.48627471923828125,
      "rewards/rejected": -1.497749924659729,
      "step": 8459
    },
    {
      "epoch": 1.1070417927537417,
      "grad_norm": 7.068110942840576,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20287390053272247,
      "logits/rejected": 0.02968643605709076,
      "logps/chosen": -428.2239990234375,
      "logps/rejected": -322.1560974121094,
      "loss": 0.4861,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6375483274459839,
      "rewards/margins": 0.847450852394104,
      "rewards/rejected": -1.484999179840088,
      "step": 8460
    },
    {
      "epoch": 1.1071726506910935,
      "grad_norm": 7.294525623321533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01350882276892662,
      "logits/rejected": 0.004063507542014122,
      "logps/chosen": -488.46282958984375,
      "logps/rejected": -379.9727478027344,
      "loss": 0.3209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9404188394546509,
      "rewards/margins": 1.6166298389434814,
      "rewards/rejected": -2.557048797607422,
      "step": 8461
    },
    {
      "epoch": 1.1073035086284453,
      "grad_norm": 6.100061893463135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.197515070438385,
      "logits/rejected": 0.21993711590766907,
      "logps/chosen": -320.6867980957031,
      "logps/rejected": -354.11395263671875,
      "loss": 0.3985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30423736572265625,
      "rewards/margins": 0.9798373579978943,
      "rewards/rejected": -1.2840749025344849,
      "step": 8462
    },
    {
      "epoch": 1.107434366565797,
      "grad_norm": 5.932379722595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032013896852731705,
      "logits/rejected": 0.05219919979572296,
      "logps/chosen": -351.790771484375,
      "logps/rejected": -444.5181579589844,
      "loss": 0.3751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6239035725593567,
      "rewards/margins": 1.3152610063552856,
      "rewards/rejected": -1.9391647577285767,
      "step": 8463
    },
    {
      "epoch": 1.1075652245031489,
      "grad_norm": 16.401235580444336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16156034171581268,
      "logits/rejected": -0.06787320971488953,
      "logps/chosen": -515.0025024414062,
      "logps/rejected": -525.07177734375,
      "loss": 1.1163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2975233793258667,
      "rewards/margins": 0.8451911211013794,
      "rewards/rejected": -2.142714500427246,
      "step": 8464
    },
    {
      "epoch": 1.1076960824405004,
      "grad_norm": 7.781128406524658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1634741872549057,
      "logits/rejected": -0.21364155411720276,
      "logps/chosen": -364.4265441894531,
      "logps/rejected": -565.8011474609375,
      "loss": 0.4582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3883819580078125,
      "rewards/margins": 1.878639578819275,
      "rewards/rejected": -3.267021417617798,
      "step": 8465
    },
    {
      "epoch": 1.1078269403778522,
      "grad_norm": 3.8712902069091797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030030183494091034,
      "logits/rejected": 0.1345694214105606,
      "logps/chosen": -490.609375,
      "logps/rejected": -516.1314086914062,
      "loss": 0.1893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26735439896583557,
      "rewards/margins": 2.0558130741119385,
      "rewards/rejected": -1.7884585857391357,
      "step": 8466
    },
    {
      "epoch": 1.107957798315204,
      "grad_norm": 7.2967305183410645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0679725706577301,
      "logits/rejected": -0.14768415689468384,
      "logps/chosen": -449.2498779296875,
      "logps/rejected": -464.39312744140625,
      "loss": 0.4616,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3340076804161072,
      "rewards/margins": 2.1580729484558105,
      "rewards/rejected": -2.4920809268951416,
      "step": 8467
    },
    {
      "epoch": 1.1080886562525558,
      "grad_norm": 7.033667087554932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1672418713569641,
      "logits/rejected": -0.11537767946720123,
      "logps/chosen": -429.0249328613281,
      "logps/rejected": -531.6380615234375,
      "loss": 0.4567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3399829864501953,
      "rewards/margins": 1.5765042304992676,
      "rewards/rejected": -1.9164870977401733,
      "step": 8468
    },
    {
      "epoch": 1.1082195141899076,
      "grad_norm": 13.72607707977295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1150471568107605,
      "logits/rejected": -0.37195295095443726,
      "logps/chosen": -571.3375244140625,
      "logps/rejected": -525.3550415039062,
      "loss": 0.4249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2009443044662476,
      "rewards/margins": 1.7369885444641113,
      "rewards/rejected": -2.9379327297210693,
      "step": 8469
    },
    {
      "epoch": 1.1083503721272594,
      "grad_norm": 7.300783157348633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15658697485923767,
      "logits/rejected": -0.26359668374061584,
      "logps/chosen": -572.5399169921875,
      "logps/rejected": -471.2516784667969,
      "loss": 0.5036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8322458267211914,
      "rewards/margins": 0.9744086265563965,
      "rewards/rejected": -1.806654453277588,
      "step": 8470
    },
    {
      "epoch": 1.1084812300646112,
      "grad_norm": 9.549769401550293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0035958215594291687,
      "logits/rejected": 0.022446483373641968,
      "logps/chosen": -453.1485900878906,
      "logps/rejected": -410.70489501953125,
      "loss": 0.6843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1252845674753189,
      "rewards/margins": 0.4009372889995575,
      "rewards/rejected": -0.5262218713760376,
      "step": 8471
    },
    {
      "epoch": 1.108612088001963,
      "grad_norm": 5.011193752288818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2526992857456207,
      "logits/rejected": 0.0032299328595399857,
      "logps/chosen": -472.25482177734375,
      "logps/rejected": -404.7036437988281,
      "loss": 0.4086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13550475239753723,
      "rewards/margins": 1.5192697048187256,
      "rewards/rejected": -1.65477454662323,
      "step": 8472
    },
    {
      "epoch": 1.1087429459393146,
      "grad_norm": 4.618459701538086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23195704817771912,
      "logits/rejected": -0.03764509782195091,
      "logps/chosen": -270.0213623046875,
      "logps/rejected": -332.2127380371094,
      "loss": 0.3571,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7132809162139893,
      "rewards/margins": 1.2431282997131348,
      "rewards/rejected": -1.956409215927124,
      "step": 8473
    },
    {
      "epoch": 1.1088738038766663,
      "grad_norm": 4.18994665145874,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013371869921684265,
      "logits/rejected": 0.05545927956700325,
      "logps/chosen": -433.8599853515625,
      "logps/rejected": -380.5638122558594,
      "loss": 0.2678,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08689593523740768,
      "rewards/margins": 1.9855670928955078,
      "rewards/rejected": -2.072462797164917,
      "step": 8474
    },
    {
      "epoch": 1.1090046618140181,
      "grad_norm": 8.885516166687012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042874716222286224,
      "logits/rejected": -0.1904873549938202,
      "logps/chosen": -443.6466979980469,
      "logps/rejected": -405.41204833984375,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11026591062545776,
      "rewards/margins": 0.5474616289138794,
      "rewards/rejected": -0.6577276587486267,
      "step": 8475
    },
    {
      "epoch": 1.10913551975137,
      "grad_norm": 9.904019355773926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.196956604719162,
      "logits/rejected": -0.12994955480098724,
      "logps/chosen": -619.18701171875,
      "logps/rejected": -652.2918090820312,
      "loss": 0.609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17310699820518494,
      "rewards/margins": 1.032209873199463,
      "rewards/rejected": -1.2053167819976807,
      "step": 8476
    },
    {
      "epoch": 1.1092663776887217,
      "grad_norm": 8.075000762939453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08380109071731567,
      "logits/rejected": -0.10038704425096512,
      "logps/chosen": -291.5910339355469,
      "logps/rejected": -331.0748596191406,
      "loss": 0.6097,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8275648355484009,
      "rewards/margins": 0.5798353552818298,
      "rewards/rejected": -1.4074002504348755,
      "step": 8477
    },
    {
      "epoch": 1.1093972356260735,
      "grad_norm": 5.728774547576904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0025065515656024218,
      "logits/rejected": 0.020580381155014038,
      "logps/chosen": -367.1994934082031,
      "logps/rejected": -373.90985107421875,
      "loss": 0.376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5605504512786865,
      "rewards/margins": 1.5279306173324585,
      "rewards/rejected": -2.0884809494018555,
      "step": 8478
    },
    {
      "epoch": 1.1095280935634253,
      "grad_norm": 6.278707981109619,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015988217666745186,
      "logits/rejected": -0.009938033297657967,
      "logps/chosen": -389.50604248046875,
      "logps/rejected": -405.70867919921875,
      "loss": 0.3989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6836786270141602,
      "rewards/margins": 0.8595656752586365,
      "rewards/rejected": -1.5432443618774414,
      "step": 8479
    },
    {
      "epoch": 1.1096589515007769,
      "grad_norm": 5.720361709594727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22563952207565308,
      "logits/rejected": 0.19246602058410645,
      "logps/chosen": -557.401611328125,
      "logps/rejected": -398.1233215332031,
      "loss": 0.2891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7871049642562866,
      "rewards/margins": 1.2702059745788574,
      "rewards/rejected": -2.0573110580444336,
      "step": 8480
    },
    {
      "epoch": 1.1097898094381287,
      "grad_norm": 10.904031753540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016295067965984344,
      "logits/rejected": -0.1366642266511917,
      "logps/chosen": -456.40533447265625,
      "logps/rejected": -378.3410949707031,
      "loss": 0.9245,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5055030584335327,
      "rewards/margins": 0.23125800490379333,
      "rewards/rejected": -1.7367610931396484,
      "step": 8481
    },
    {
      "epoch": 1.1099206673754805,
      "grad_norm": 11.006057739257812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11620641499757767,
      "logits/rejected": 0.060123153030872345,
      "logps/chosen": -431.85052490234375,
      "logps/rejected": -353.1279602050781,
      "loss": 0.7651,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.744330883026123,
      "rewards/margins": 0.4066723883152008,
      "rewards/rejected": -2.151003360748291,
      "step": 8482
    },
    {
      "epoch": 1.1100515253128322,
      "grad_norm": 6.010775566101074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14494073390960693,
      "logits/rejected": -0.0010928288102149963,
      "logps/chosen": -339.11846923828125,
      "logps/rejected": -281.27191162109375,
      "loss": 0.3699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13791875541210175,
      "rewards/margins": 0.996509850025177,
      "rewards/rejected": -0.8585911393165588,
      "step": 8483
    },
    {
      "epoch": 1.110182383250184,
      "grad_norm": 4.668667793273926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17936742305755615,
      "logits/rejected": -0.12119224667549133,
      "logps/chosen": -511.88726806640625,
      "logps/rejected": -419.0841369628906,
      "loss": 0.2938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7961516976356506,
      "rewards/margins": 1.604691505432129,
      "rewards/rejected": -2.400843381881714,
      "step": 8484
    },
    {
      "epoch": 1.1103132411875358,
      "grad_norm": 10.146418571472168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10777457803487778,
      "logits/rejected": -0.25629615783691406,
      "logps/chosen": -483.0248718261719,
      "logps/rejected": -395.36907958984375,
      "loss": 0.6181,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2713063955307007,
      "rewards/margins": 0.778017520904541,
      "rewards/rejected": -2.049323797225952,
      "step": 8485
    },
    {
      "epoch": 1.1104440991248876,
      "grad_norm": 4.58678674697876,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04771095886826515,
      "logits/rejected": -0.09321264177560806,
      "logps/chosen": -655.062255859375,
      "logps/rejected": -660.0338745117188,
      "loss": 0.2648,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08799822628498077,
      "rewards/margins": 2.191044330596924,
      "rewards/rejected": -2.10304594039917,
      "step": 8486
    },
    {
      "epoch": 1.1105749570622394,
      "grad_norm": 12.020270347595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11742208153009415,
      "logits/rejected": -0.041284289211034775,
      "logps/chosen": -618.7276611328125,
      "logps/rejected": -550.2017822265625,
      "loss": 0.6607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7949134707450867,
      "rewards/margins": 0.26449742913246155,
      "rewards/rejected": -1.059410810470581,
      "step": 8487
    },
    {
      "epoch": 1.110705814999591,
      "grad_norm": 7.175088882446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12134844809770584,
      "logits/rejected": -0.4688749611377716,
      "logps/chosen": -356.48382568359375,
      "logps/rejected": -461.9900817871094,
      "loss": 0.2889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9013859033584595,
      "rewards/margins": 2.112414836883545,
      "rewards/rejected": -3.013801097869873,
      "step": 8488
    },
    {
      "epoch": 1.1108366729369428,
      "grad_norm": 7.188058376312256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29892581701278687,
      "logits/rejected": -0.3533123731613159,
      "logps/chosen": -479.24322509765625,
      "logps/rejected": -417.5716247558594,
      "loss": 0.4564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6980545520782471,
      "rewards/margins": 1.0403410196304321,
      "rewards/rejected": -1.7383954524993896,
      "step": 8489
    },
    {
      "epoch": 1.1109675308742946,
      "grad_norm": 7.692070007324219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10648965835571289,
      "logits/rejected": -0.17129305005073547,
      "logps/chosen": -432.3581237792969,
      "logps/rejected": -362.12188720703125,
      "loss": 0.3746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5138339996337891,
      "rewards/margins": 1.1653990745544434,
      "rewards/rejected": -1.6792330741882324,
      "step": 8490
    },
    {
      "epoch": 1.1110983888116464,
      "grad_norm": 10.227920532226562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03842955827713013,
      "logits/rejected": -0.10470188409090042,
      "logps/chosen": -468.191650390625,
      "logps/rejected": -575.8446044921875,
      "loss": 0.5763,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35757899284362793,
      "rewards/margins": 1.663824439048767,
      "rewards/rejected": -2.0214035511016846,
      "step": 8491
    },
    {
      "epoch": 1.1112292467489981,
      "grad_norm": 7.209628105163574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009623246267437935,
      "logits/rejected": -0.13492079079151154,
      "logps/chosen": -459.3327331542969,
      "logps/rejected": -716.6945190429688,
      "loss": 0.434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5141023993492126,
      "rewards/margins": 1.825963020324707,
      "rewards/rejected": -2.3400654792785645,
      "step": 8492
    },
    {
      "epoch": 1.11136010468635,
      "grad_norm": 4.716139793395996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15335439145565033,
      "logits/rejected": -0.30922386050224304,
      "logps/chosen": -767.0986938476562,
      "logps/rejected": -501.13787841796875,
      "loss": 0.2428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2395666241645813,
      "rewards/margins": 2.041440963745117,
      "rewards/rejected": -1.8018741607666016,
      "step": 8493
    },
    {
      "epoch": 1.1114909626237017,
      "grad_norm": 7.3743791580200195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4270361661911011,
      "logits/rejected": -0.47648781538009644,
      "logps/chosen": -491.8854675292969,
      "logps/rejected": -538.80126953125,
      "loss": 0.3993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35186082124710083,
      "rewards/margins": 2.518979072570801,
      "rewards/rejected": -2.870839834213257,
      "step": 8494
    },
    {
      "epoch": 1.1116218205610533,
      "grad_norm": 11.448356628417969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1618010252714157,
      "logits/rejected": -0.18422865867614746,
      "logps/chosen": -613.7503662109375,
      "logps/rejected": -341.8767395019531,
      "loss": 0.653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3737138509750366,
      "rewards/margins": 0.597419023513794,
      "rewards/rejected": -1.9711329936981201,
      "step": 8495
    },
    {
      "epoch": 1.111752678498405,
      "grad_norm": 4.912971019744873,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026278231292963028,
      "logits/rejected": -0.24715538322925568,
      "logps/chosen": -663.3451538085938,
      "logps/rejected": -482.89288330078125,
      "loss": 0.2437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5375149250030518,
      "rewards/margins": 2.0058255195617676,
      "rewards/rejected": -2.5433402061462402,
      "step": 8496
    },
    {
      "epoch": 1.1118835364357569,
      "grad_norm": 11.6221342086792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0570499449968338,
      "logits/rejected": -0.0884799063205719,
      "logps/chosen": -373.9681091308594,
      "logps/rejected": -430.7813720703125,
      "loss": 0.6217,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49096760153770447,
      "rewards/margins": 1.4890165328979492,
      "rewards/rejected": -1.979983925819397,
      "step": 8497
    },
    {
      "epoch": 1.1120143943731087,
      "grad_norm": 11.866124153137207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055713362991809845,
      "logits/rejected": -0.0018774019554257393,
      "logps/chosen": -444.650146484375,
      "logps/rejected": -438.509765625,
      "loss": 0.8624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4761207699775696,
      "rewards/margins": 0.5946319103240967,
      "rewards/rejected": -1.0707526206970215,
      "step": 8498
    },
    {
      "epoch": 1.1121452523104605,
      "grad_norm": 9.920528411865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.272407591342926,
      "logits/rejected": -0.056863781064748764,
      "logps/chosen": -365.1797790527344,
      "logps/rejected": -438.01025390625,
      "loss": 0.4211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2560064196586609,
      "rewards/margins": 1.5566537380218506,
      "rewards/rejected": -1.8126603364944458,
      "step": 8499
    },
    {
      "epoch": 1.1122761102478123,
      "grad_norm": 5.085936069488525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041845254600048065,
      "logits/rejected": -0.09463389962911606,
      "logps/chosen": -358.0672302246094,
      "logps/rejected": -366.66375732421875,
      "loss": 0.3769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6942508816719055,
      "rewards/margins": 2.1347203254699707,
      "rewards/rejected": -2.8289711475372314,
      "step": 8500
    },
    {
      "epoch": 1.112406968185164,
      "grad_norm": 8.844292640686035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16948120296001434,
      "logits/rejected": -0.09210442006587982,
      "logps/chosen": -721.998291015625,
      "logps/rejected": -493.0779113769531,
      "loss": 0.4345,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4861776530742645,
      "rewards/margins": 1.6892285346984863,
      "rewards/rejected": -2.1754062175750732,
      "step": 8501
    },
    {
      "epoch": 1.1125378261225158,
      "grad_norm": 4.37510871887207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14143133163452148,
      "logits/rejected": 0.01368031743913889,
      "logps/chosen": -403.7288818359375,
      "logps/rejected": -475.5980529785156,
      "loss": 0.3131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4203964173793793,
      "rewards/margins": 1.684719204902649,
      "rewards/rejected": -2.1051156520843506,
      "step": 8502
    },
    {
      "epoch": 1.1126686840598674,
      "grad_norm": 4.464987277984619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18502271175384521,
      "logits/rejected": -0.29027682542800903,
      "logps/chosen": -435.2702941894531,
      "logps/rejected": -351.1804504394531,
      "loss": 0.2813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.032518770545721054,
      "rewards/margins": 1.5287407636642456,
      "rewards/rejected": -1.5612596273422241,
      "step": 8503
    },
    {
      "epoch": 1.1127995419972192,
      "grad_norm": 6.996791839599609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1279972791671753,
      "logits/rejected": -0.06569722294807434,
      "logps/chosen": -457.7364501953125,
      "logps/rejected": -497.560791015625,
      "loss": 0.3122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6835806369781494,
      "rewards/margins": 1.81591796875,
      "rewards/rejected": -2.4994983673095703,
      "step": 8504
    },
    {
      "epoch": 1.112930399934571,
      "grad_norm": 8.577402114868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.130362868309021,
      "logits/rejected": -0.2065601944923401,
      "logps/chosen": -554.763916015625,
      "logps/rejected": -543.6859741210938,
      "loss": 0.4519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.551072359085083,
      "rewards/margins": 0.9162869453430176,
      "rewards/rejected": -2.4673590660095215,
      "step": 8505
    },
    {
      "epoch": 1.1130612578719228,
      "grad_norm": 6.015432357788086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07393105328083038,
      "logits/rejected": 0.16810452938079834,
      "logps/chosen": -535.7059936523438,
      "logps/rejected": -613.8331298828125,
      "loss": 0.3278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6548996567726135,
      "rewards/margins": 1.2843159437179565,
      "rewards/rejected": -0.629416286945343,
      "step": 8506
    },
    {
      "epoch": 1.1131921158092746,
      "grad_norm": 6.9356160163879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17704349756240845,
      "logits/rejected": 0.14645150303840637,
      "logps/chosen": -473.4967346191406,
      "logps/rejected": -417.6862487792969,
      "loss": 0.4127,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6391675472259521,
      "rewards/margins": 1.0245550870895386,
      "rewards/rejected": -1.6637227535247803,
      "step": 8507
    },
    {
      "epoch": 1.1133229737466264,
      "grad_norm": 4.322902679443359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4050728678703308,
      "logits/rejected": -0.5599918365478516,
      "logps/chosen": -471.2742919921875,
      "logps/rejected": -548.9837036132812,
      "loss": 0.1879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3612304627895355,
      "rewards/margins": 2.4179792404174805,
      "rewards/rejected": -2.779209613800049,
      "step": 8508
    },
    {
      "epoch": 1.1134538316839782,
      "grad_norm": 6.928945064544678,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06669756770133972,
      "logits/rejected": 0.012107837945222855,
      "logps/chosen": -470.54022216796875,
      "logps/rejected": -437.9939880371094,
      "loss": 0.4484,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47283652424812317,
      "rewards/margins": 1.3195492029190063,
      "rewards/rejected": -1.7923855781555176,
      "step": 8509
    },
    {
      "epoch": 1.1135846896213297,
      "grad_norm": 6.4306960105896,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12292879819869995,
      "logits/rejected": -0.06926071643829346,
      "logps/chosen": -502.35797119140625,
      "logps/rejected": -396.60516357421875,
      "loss": 0.4144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2089206874370575,
      "rewards/margins": 1.0930757522583008,
      "rewards/rejected": -1.3019965887069702,
      "step": 8510
    },
    {
      "epoch": 1.1137155475586815,
      "grad_norm": 11.516524314880371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16996747255325317,
      "logits/rejected": -0.2292424887418747,
      "logps/chosen": -449.5834045410156,
      "logps/rejected": -374.2565002441406,
      "loss": 0.9426,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6444846987724304,
      "rewards/margins": -0.2874128222465515,
      "rewards/rejected": -0.3570718765258789,
      "step": 8511
    },
    {
      "epoch": 1.1138464054960333,
      "grad_norm": 9.133455276489258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2049325704574585,
      "logits/rejected": 0.22259600460529327,
      "logps/chosen": -545.2661743164062,
      "logps/rejected": -534.0902099609375,
      "loss": 0.5332,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4054153561592102,
      "rewards/margins": 0.6763252019882202,
      "rewards/rejected": -1.0817406177520752,
      "step": 8512
    },
    {
      "epoch": 1.113977263433385,
      "grad_norm": 8.040541648864746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026120102033019066,
      "logits/rejected": -0.16023589670658112,
      "logps/chosen": -570.5230712890625,
      "logps/rejected": -515.4852294921875,
      "loss": 0.4527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8317109942436218,
      "rewards/margins": 1.871649146080017,
      "rewards/rejected": -2.703360080718994,
      "step": 8513
    },
    {
      "epoch": 1.114108121370737,
      "grad_norm": 8.087608337402344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15440061688423157,
      "logits/rejected": -0.15335328876972198,
      "logps/chosen": -506.167724609375,
      "logps/rejected": -372.9412841796875,
      "loss": 0.3276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9188394546508789,
      "rewards/margins": 1.3685051202774048,
      "rewards/rejected": -2.287344455718994,
      "step": 8514
    },
    {
      "epoch": 1.1142389793080887,
      "grad_norm": 10.539715766906738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23418393731117249,
      "logits/rejected": -0.5254936814308167,
      "logps/chosen": -556.0624389648438,
      "logps/rejected": -534.7637939453125,
      "loss": 0.5132,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3042287826538086,
      "rewards/margins": 1.6349787712097168,
      "rewards/rejected": -1.9392074346542358,
      "step": 8515
    },
    {
      "epoch": 1.1143698372454405,
      "grad_norm": 4.902479648590088,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22040490806102753,
      "logits/rejected": 0.25292733311653137,
      "logps/chosen": -389.0547790527344,
      "logps/rejected": -371.68194580078125,
      "loss": 0.223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44003045558929443,
      "rewards/margins": 1.7318494319915771,
      "rewards/rejected": -1.2918189764022827,
      "step": 8516
    },
    {
      "epoch": 1.1145006951827923,
      "grad_norm": 5.685092926025391,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3153156042098999,
      "logits/rejected": -0.44053733348846436,
      "logps/chosen": -330.3446350097656,
      "logps/rejected": -370.9701232910156,
      "loss": 0.3321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8123655915260315,
      "rewards/margins": 1.750999927520752,
      "rewards/rejected": -2.5633654594421387,
      "step": 8517
    },
    {
      "epoch": 1.114631553120144,
      "grad_norm": 4.507125377655029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005913164466619492,
      "logits/rejected": -0.13152272999286652,
      "logps/chosen": -305.9146728515625,
      "logps/rejected": -277.0080261230469,
      "loss": 0.3246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7213279604911804,
      "rewards/margins": 1.113713026046753,
      "rewards/rejected": -1.8350411653518677,
      "step": 8518
    },
    {
      "epoch": 1.1147624110574956,
      "grad_norm": 7.7363810539245605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1258849799633026,
      "logits/rejected": 0.226544588804245,
      "logps/chosen": -728.5184936523438,
      "logps/rejected": -554.9963989257812,
      "loss": 0.4108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5795555114746094,
      "rewards/margins": 1.233842372894287,
      "rewards/rejected": -1.813398003578186,
      "step": 8519
    },
    {
      "epoch": 1.1148932689948474,
      "grad_norm": 9.63819694519043,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1518731713294983,
      "logits/rejected": 0.15795932710170746,
      "logps/chosen": -495.2734375,
      "logps/rejected": -591.095947265625,
      "loss": 0.6382,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09287717938423157,
      "rewards/margins": 1.0274038314819336,
      "rewards/rejected": -1.1202809810638428,
      "step": 8520
    },
    {
      "epoch": 1.1150241269321992,
      "grad_norm": 11.834305763244629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01237025111913681,
      "logits/rejected": -0.42718106508255005,
      "logps/chosen": -651.7242431640625,
      "logps/rejected": -563.3768310546875,
      "loss": 0.7751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4342013597488403,
      "rewards/margins": 0.453189492225647,
      "rewards/rejected": -1.8873909711837769,
      "step": 8521
    },
    {
      "epoch": 1.115154984869551,
      "grad_norm": 4.6979899406433105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06359134614467621,
      "logits/rejected": 0.003758637700229883,
      "logps/chosen": -321.50872802734375,
      "logps/rejected": -322.9030456542969,
      "loss": 0.2984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.885315477848053,
      "rewards/margins": 1.4839013814926147,
      "rewards/rejected": -2.3692169189453125,
      "step": 8522
    },
    {
      "epoch": 1.1152858428069028,
      "grad_norm": 8.388227462768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016559869050979614,
      "logits/rejected": -0.10673859715461731,
      "logps/chosen": -536.0662841796875,
      "logps/rejected": -408.40240478515625,
      "loss": 0.553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20042745769023895,
      "rewards/margins": 0.5831876993179321,
      "rewards/rejected": -0.3827601969242096,
      "step": 8523
    },
    {
      "epoch": 1.1154167007442546,
      "grad_norm": 18.58466339111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032709166407585144,
      "logits/rejected": 0.04431808739900589,
      "logps/chosen": -811.3154907226562,
      "logps/rejected": -604.2191162109375,
      "loss": 1.2919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.245823860168457,
      "rewards/margins": -0.02510926127433777,
      "rewards/rejected": -2.220714807510376,
      "step": 8524
    },
    {
      "epoch": 1.1155475586816064,
      "grad_norm": 5.189408779144287,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13090626895427704,
      "logits/rejected": -0.05547436699271202,
      "logps/chosen": -470.16888427734375,
      "logps/rejected": -295.2483825683594,
      "loss": 0.3221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03304654359817505,
      "rewards/margins": 1.4156633615493774,
      "rewards/rejected": -1.3826169967651367,
      "step": 8525
    },
    {
      "epoch": 1.115678416618958,
      "grad_norm": 7.281444549560547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1331220269203186,
      "logits/rejected": 0.022745339199900627,
      "logps/chosen": -563.1549682617188,
      "logps/rejected": -506.7353515625,
      "loss": 0.4146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8424595594406128,
      "rewards/margins": 1.6432180404663086,
      "rewards/rejected": -2.485677719116211,
      "step": 8526
    },
    {
      "epoch": 1.1158092745563097,
      "grad_norm": 8.182840347290039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08397769927978516,
      "logits/rejected": 0.03250138834118843,
      "logps/chosen": -393.3841857910156,
      "logps/rejected": -348.7941589355469,
      "loss": 0.5383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39417970180511475,
      "rewards/margins": 0.9306865930557251,
      "rewards/rejected": -1.3248662948608398,
      "step": 8527
    },
    {
      "epoch": 1.1159401324936615,
      "grad_norm": 8.956804275512695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.058397725224494934,
      "logits/rejected": 0.053184278309345245,
      "logps/chosen": -398.23944091796875,
      "logps/rejected": -514.6771850585938,
      "loss": 0.5631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3926209211349487,
      "rewards/margins": 0.6545396447181702,
      "rewards/rejected": -2.0471606254577637,
      "step": 8528
    },
    {
      "epoch": 1.1160709904310133,
      "grad_norm": 7.335103511810303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03977271914482117,
      "logits/rejected": -0.15017171204090118,
      "logps/chosen": -453.9305725097656,
      "logps/rejected": -435.28021240234375,
      "loss": 0.5514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7619163990020752,
      "rewards/margins": 1.0849469900131226,
      "rewards/rejected": -1.8468633890151978,
      "step": 8529
    },
    {
      "epoch": 1.1162018483683651,
      "grad_norm": 7.394342422485352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1088627353310585,
      "logits/rejected": 0.05642060190439224,
      "logps/chosen": -385.32879638671875,
      "logps/rejected": -407.83746337890625,
      "loss": 0.3571,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.507972002029419,
      "rewards/margins": 1.1573344469070435,
      "rewards/rejected": -1.665306568145752,
      "step": 8530
    },
    {
      "epoch": 1.116332706305717,
      "grad_norm": 5.158576488494873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04259674623608589,
      "logits/rejected": 0.09658092260360718,
      "logps/chosen": -636.2215576171875,
      "logps/rejected": -478.2665710449219,
      "loss": 0.2396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2427186667919159,
      "rewards/margins": 2.1607577800750732,
      "rewards/rejected": -1.9180388450622559,
      "step": 8531
    },
    {
      "epoch": 1.1164635642430687,
      "grad_norm": 4.2099432945251465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024343475699424744,
      "logits/rejected": -0.03417161852121353,
      "logps/chosen": -526.7754516601562,
      "logps/rejected": -493.4984130859375,
      "loss": 0.1949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15414905548095703,
      "rewards/margins": 1.6801201105117798,
      "rewards/rejected": -1.5259710550308228,
      "step": 8532
    },
    {
      "epoch": 1.1165944221804205,
      "grad_norm": 12.810059547424316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12316005676984787,
      "logits/rejected": -0.22249014675617218,
      "logps/chosen": -501.41583251953125,
      "logps/rejected": -364.52020263671875,
      "loss": 0.7966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2698357105255127,
      "rewards/margins": 0.1164160817861557,
      "rewards/rejected": -1.3862519264221191,
      "step": 8533
    },
    {
      "epoch": 1.116725280117772,
      "grad_norm": 11.31277847290039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17923961579799652,
      "logits/rejected": -0.12079785764217377,
      "logps/chosen": -410.82965087890625,
      "logps/rejected": -432.81109619140625,
      "loss": 0.6376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5882968902587891,
      "rewards/margins": 0.37403833866119385,
      "rewards/rejected": -0.9623351693153381,
      "step": 8534
    },
    {
      "epoch": 1.1168561380551238,
      "grad_norm": 7.609182834625244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17683830857276917,
      "logits/rejected": 0.10294046998023987,
      "logps/chosen": -467.1785888671875,
      "logps/rejected": -486.0679626464844,
      "loss": 0.5471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4046459496021271,
      "rewards/margins": 0.45166245102882385,
      "rewards/rejected": -0.8563084006309509,
      "step": 8535
    },
    {
      "epoch": 1.1169869959924756,
      "grad_norm": 5.522397518157959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14458498358726501,
      "logits/rejected": -0.3095966875553131,
      "logps/chosen": -665.5606079101562,
      "logps/rejected": -538.4273071289062,
      "loss": 0.4112,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9996431469917297,
      "rewards/margins": 1.3580979108810425,
      "rewards/rejected": -2.357741117477417,
      "step": 8536
    },
    {
      "epoch": 1.1171178539298274,
      "grad_norm": 5.527436256408691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23551897704601288,
      "logits/rejected": 0.2145031988620758,
      "logps/chosen": -538.5013427734375,
      "logps/rejected": -507.4530334472656,
      "loss": 0.3269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4224560260772705,
      "rewards/margins": 1.8153562545776367,
      "rewards/rejected": -1.392899990081787,
      "step": 8537
    },
    {
      "epoch": 1.1172487118671792,
      "grad_norm": 9.026121139526367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2689816355705261,
      "logits/rejected": -0.15935775637626648,
      "logps/chosen": -349.193115234375,
      "logps/rejected": -435.8085632324219,
      "loss": 0.4579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6735668182373047,
      "rewards/margins": 1.0718374252319336,
      "rewards/rejected": -1.7454042434692383,
      "step": 8538
    },
    {
      "epoch": 1.117379569804531,
      "grad_norm": 7.650084972381592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22538012266159058,
      "logits/rejected": -0.21120448410511017,
      "logps/chosen": -536.0827026367188,
      "logps/rejected": -533.1054077148438,
      "loss": 0.3941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05081787705421448,
      "rewards/margins": 1.115748405456543,
      "rewards/rejected": -1.0649305582046509,
      "step": 8539
    },
    {
      "epoch": 1.1175104277418828,
      "grad_norm": 6.238037109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07109510898590088,
      "logits/rejected": 0.012431643903255463,
      "logps/chosen": -530.2923583984375,
      "logps/rejected": -434.77569580078125,
      "loss": 0.3769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.053474415093660355,
      "rewards/margins": 1.735896110534668,
      "rewards/rejected": -1.6824216842651367,
      "step": 8540
    },
    {
      "epoch": 1.1176412856792344,
      "grad_norm": 7.7308173179626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08013638108968735,
      "logits/rejected": 0.007367093116044998,
      "logps/chosen": -432.41815185546875,
      "logps/rejected": -390.75689697265625,
      "loss": 0.5049,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3313447833061218,
      "rewards/margins": 1.1310958862304688,
      "rewards/rejected": -1.4624407291412354,
      "step": 8541
    },
    {
      "epoch": 1.1177721436165862,
      "grad_norm": 8.035426139831543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26634010672569275,
      "logits/rejected": -0.15437713265419006,
      "logps/chosen": -387.433837890625,
      "logps/rejected": -363.13616943359375,
      "loss": 0.4906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4973461329936981,
      "rewards/margins": 1.2743785381317139,
      "rewards/rejected": -1.7717247009277344,
      "step": 8542
    },
    {
      "epoch": 1.117903001553938,
      "grad_norm": 7.743760108947754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03135666996240616,
      "logits/rejected": -0.0057273488491773605,
      "logps/chosen": -504.925537109375,
      "logps/rejected": -440.48614501953125,
      "loss": 0.5449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5489783883094788,
      "rewards/margins": 0.911792516708374,
      "rewards/rejected": -1.460770845413208,
      "step": 8543
    },
    {
      "epoch": 1.1180338594912897,
      "grad_norm": 6.6409711837768555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1226745992898941,
      "logits/rejected": -0.07583007216453552,
      "logps/chosen": -508.8397521972656,
      "logps/rejected": -436.86785888671875,
      "loss": 0.4768,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4316379129886627,
      "rewards/margins": 1.2949278354644775,
      "rewards/rejected": -0.863290011882782,
      "step": 8544
    },
    {
      "epoch": 1.1181647174286415,
      "grad_norm": 5.5559844970703125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15621191263198853,
      "logits/rejected": -0.1088571548461914,
      "logps/chosen": -471.06365966796875,
      "logps/rejected": -515.7645263671875,
      "loss": 0.2462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33852824568748474,
      "rewards/margins": 2.4157838821411133,
      "rewards/rejected": -2.754312515258789,
      "step": 8545
    },
    {
      "epoch": 1.1182955753659933,
      "grad_norm": 6.6303324699401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0796317458152771,
      "logits/rejected": -0.1343819797039032,
      "logps/chosen": -475.3072204589844,
      "logps/rejected": -414.39324951171875,
      "loss": 0.6133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23302462697029114,
      "rewards/margins": 1.4137179851531982,
      "rewards/rejected": -1.6467427015304565,
      "step": 8546
    },
    {
      "epoch": 1.1184264333033451,
      "grad_norm": 7.135304927825928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05857659876346588,
      "logits/rejected": -0.05930793657898903,
      "logps/chosen": -398.0238952636719,
      "logps/rejected": -411.955322265625,
      "loss": 0.5323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6331164836883545,
      "rewards/margins": 1.4367676973342896,
      "rewards/rejected": -2.0698843002319336,
      "step": 8547
    },
    {
      "epoch": 1.118557291240697,
      "grad_norm": 6.305968284606934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05165163055062294,
      "logits/rejected": 0.040888696908950806,
      "logps/chosen": -394.36322021484375,
      "logps/rejected": -363.8428649902344,
      "loss": 0.3267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3035426735877991,
      "rewards/margins": 1.503464698791504,
      "rewards/rejected": -1.8070074319839478,
      "step": 8548
    },
    {
      "epoch": 1.1186881491780485,
      "grad_norm": 6.616814613342285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30269497632980347,
      "logits/rejected": -0.23558413982391357,
      "logps/chosen": -478.6900329589844,
      "logps/rejected": -427.8517150878906,
      "loss": 0.2661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4094611406326294,
      "rewards/margins": 1.5554295778274536,
      "rewards/rejected": -1.9648905992507935,
      "step": 8549
    },
    {
      "epoch": 1.1188190071154003,
      "grad_norm": 7.783444404602051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0849950909614563,
      "logits/rejected": 0.04855211824178696,
      "logps/chosen": -505.5209655761719,
      "logps/rejected": -688.2561645507812,
      "loss": 0.3239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11833494156599045,
      "rewards/margins": 1.754047155380249,
      "rewards/rejected": -1.8723821640014648,
      "step": 8550
    },
    {
      "epoch": 1.118949865052752,
      "grad_norm": 4.545641899108887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2031388282775879,
      "logits/rejected": -0.22678807377815247,
      "logps/chosen": -555.698974609375,
      "logps/rejected": -489.0443420410156,
      "loss": 0.3174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5216688513755798,
      "rewards/margins": 1.2771714925765991,
      "rewards/rejected": -1.7988404035568237,
      "step": 8551
    },
    {
      "epoch": 1.1190807229901039,
      "grad_norm": 7.652423858642578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23450346291065216,
      "logits/rejected": 0.13689790666103363,
      "logps/chosen": -499.74774169921875,
      "logps/rejected": -589.5570068359375,
      "loss": 0.316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.006618857383728,
      "rewards/margins": 2.211329460144043,
      "rewards/rejected": -3.2179484367370605,
      "step": 8552
    },
    {
      "epoch": 1.1192115809274557,
      "grad_norm": 6.77015495300293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12939676642417908,
      "logits/rejected": 0.2523985803127289,
      "logps/chosen": -401.8409729003906,
      "logps/rejected": -408.84259033203125,
      "loss": 0.458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4489784240722656,
      "rewards/margins": 0.8864423632621765,
      "rewards/rejected": -1.335420846939087,
      "step": 8553
    },
    {
      "epoch": 1.1193424388648074,
      "grad_norm": 5.682043552398682,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08436832576990128,
      "logits/rejected": -0.1667904257774353,
      "logps/chosen": -566.9654541015625,
      "logps/rejected": -496.22882080078125,
      "loss": 0.3395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2678816318511963,
      "rewards/margins": 1.2261017560958862,
      "rewards/rejected": -0.9582201242446899,
      "step": 8554
    },
    {
      "epoch": 1.1194732968021592,
      "grad_norm": 11.309574127197266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11779467761516571,
      "logits/rejected": -0.12766298651695251,
      "logps/chosen": -622.9595947265625,
      "logps/rejected": -502.02899169921875,
      "loss": 0.7835,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27523019909858704,
      "rewards/margins": 0.5980848073959351,
      "rewards/rejected": -0.8733150959014893,
      "step": 8555
    },
    {
      "epoch": 1.1196041547395108,
      "grad_norm": 6.956723690032959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08739150315523148,
      "logits/rejected": 0.09241516888141632,
      "logps/chosen": -400.2882080078125,
      "logps/rejected": -416.1597900390625,
      "loss": 0.333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1989816278219223,
      "rewards/margins": 1.479879379272461,
      "rewards/rejected": -1.2808977365493774,
      "step": 8556
    },
    {
      "epoch": 1.1197350126768626,
      "grad_norm": 7.76517915725708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06800577044487,
      "logits/rejected": -0.09509417414665222,
      "logps/chosen": -618.8455810546875,
      "logps/rejected": -446.7433776855469,
      "loss": 0.4084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2933429777622223,
      "rewards/margins": 1.2358824014663696,
      "rewards/rejected": -1.5292253494262695,
      "step": 8557
    },
    {
      "epoch": 1.1198658706142144,
      "grad_norm": 7.184216022491455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09599162638187408,
      "logits/rejected": -0.12675166130065918,
      "logps/chosen": -436.5083923339844,
      "logps/rejected": -456.7169189453125,
      "loss": 0.4656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1064953804016113,
      "rewards/margins": 1.4028840065002441,
      "rewards/rejected": -2.5093793869018555,
      "step": 8558
    },
    {
      "epoch": 1.1199967285515662,
      "grad_norm": 5.301325798034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05047512799501419,
      "logits/rejected": 0.04905601218342781,
      "logps/chosen": -650.3318481445312,
      "logps/rejected": -692.7774047851562,
      "loss": 0.242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12126541137695312,
      "rewards/margins": 1.995362639427185,
      "rewards/rejected": -2.1166281700134277,
      "step": 8559
    },
    {
      "epoch": 1.120127586488918,
      "grad_norm": 3.862962484359741,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09285356104373932,
      "logits/rejected": -0.12944164872169495,
      "logps/chosen": -533.7703247070312,
      "logps/rejected": -442.8744812011719,
      "loss": 0.2818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6922624111175537,
      "rewards/margins": 1.8336844444274902,
      "rewards/rejected": -2.525946617126465,
      "step": 8560
    },
    {
      "epoch": 1.1202584444262698,
      "grad_norm": 5.638490200042725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13150638341903687,
      "logits/rejected": -0.23513412475585938,
      "logps/chosen": -472.0716247558594,
      "logps/rejected": -507.9556884765625,
      "loss": 0.3346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7649624943733215,
      "rewards/margins": 1.5906093120574951,
      "rewards/rejected": -2.355571985244751,
      "step": 8561
    },
    {
      "epoch": 1.1203893023636216,
      "grad_norm": 5.419081211090088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16004399955272675,
      "logits/rejected": -0.03813972324132919,
      "logps/chosen": -394.07330322265625,
      "logps/rejected": -441.2997131347656,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48960554599761963,
      "rewards/margins": 1.822331428527832,
      "rewards/rejected": -2.311936855316162,
      "step": 8562
    },
    {
      "epoch": 1.1205201603009733,
      "grad_norm": 4.320306777954102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1578506976366043,
      "logits/rejected": -0.05962274223566055,
      "logps/chosen": -515.963623046875,
      "logps/rejected": -561.5309448242188,
      "loss": 0.24,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4106788635253906,
      "rewards/margins": 2.4501237869262695,
      "rewards/rejected": -2.86080265045166,
      "step": 8563
    },
    {
      "epoch": 1.1206510182383251,
      "grad_norm": 4.442824840545654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17075426876544952,
      "logits/rejected": -0.102512888610363,
      "logps/chosen": -458.61065673828125,
      "logps/rejected": -471.23809814453125,
      "loss": 0.2082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.531346321105957,
      "rewards/margins": 1.8709216117858887,
      "rewards/rejected": -2.4022679328918457,
      "step": 8564
    },
    {
      "epoch": 1.1207818761756767,
      "grad_norm": 7.620718479156494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1530674248933792,
      "logits/rejected": -0.06860406696796417,
      "logps/chosen": -563.4564819335938,
      "logps/rejected": -609.2240600585938,
      "loss": 0.303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5099758505821228,
      "rewards/margins": 2.226172924041748,
      "rewards/rejected": -2.736149311065674,
      "step": 8565
    },
    {
      "epoch": 1.1209127341130285,
      "grad_norm": 6.7951226234436035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07193787395954132,
      "logits/rejected": -0.04214960336685181,
      "logps/chosen": -294.4377746582031,
      "logps/rejected": -239.76333618164062,
      "loss": 0.448,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5162903070449829,
      "rewards/margins": 0.9662433862686157,
      "rewards/rejected": -1.482533574104309,
      "step": 8566
    },
    {
      "epoch": 1.1210435920503803,
      "grad_norm": 9.04212474822998,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03518451750278473,
      "logits/rejected": 0.23348909616470337,
      "logps/chosen": -567.8402709960938,
      "logps/rejected": -543.3536376953125,
      "loss": 0.4362,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3867206573486328,
      "rewards/margins": 1.292457938194275,
      "rewards/rejected": -1.6791784763336182,
      "step": 8567
    },
    {
      "epoch": 1.121174449987732,
      "grad_norm": 10.387788772583008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06790396571159363,
      "logits/rejected": -0.15268336236476898,
      "logps/chosen": -387.67718505859375,
      "logps/rejected": -452.5257873535156,
      "loss": 0.5168,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7723580598831177,
      "rewards/margins": 1.145678997039795,
      "rewards/rejected": -1.9180370569229126,
      "step": 8568
    },
    {
      "epoch": 1.1213053079250839,
      "grad_norm": 10.889378547668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07040317356586456,
      "logits/rejected": 0.03462681174278259,
      "logps/chosen": -461.8839111328125,
      "logps/rejected": -449.722412109375,
      "loss": 0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3593566417694092,
      "rewards/margins": 0.7220900058746338,
      "rewards/rejected": -2.081446647644043,
      "step": 8569
    },
    {
      "epoch": 1.1214361658624357,
      "grad_norm": 6.395432472229004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29070839285850525,
      "logits/rejected": -0.12786918878555298,
      "logps/chosen": -329.3293151855469,
      "logps/rejected": -416.6815490722656,
      "loss": 0.3699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7986114025115967,
      "rewards/margins": 1.5640722513198853,
      "rewards/rejected": -2.3626837730407715,
      "step": 8570
    },
    {
      "epoch": 1.1215670237997875,
      "grad_norm": 6.083571434020996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2608042061328888,
      "logits/rejected": -0.3494899272918701,
      "logps/chosen": -427.62939453125,
      "logps/rejected": -332.4398193359375,
      "loss": 0.398,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8816918134689331,
      "rewards/margins": 1.8960583209991455,
      "rewards/rejected": -2.7777504920959473,
      "step": 8571
    },
    {
      "epoch": 1.121697881737139,
      "grad_norm": 6.144331455230713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34523463249206543,
      "logits/rejected": -0.3089117705821991,
      "logps/chosen": -565.318359375,
      "logps/rejected": -596.7271118164062,
      "loss": 0.2817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1751592606306076,
      "rewards/margins": 2.008488655090332,
      "rewards/rejected": -1.8333293199539185,
      "step": 8572
    },
    {
      "epoch": 1.1218287396744908,
      "grad_norm": 7.19893217086792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3279602527618408,
      "logits/rejected": -0.40237411856651306,
      "logps/chosen": -460.48980712890625,
      "logps/rejected": -471.28118896484375,
      "loss": 0.2682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0095672607421875,
      "rewards/margins": 1.9450619220733643,
      "rewards/rejected": -1.9354946613311768,
      "step": 8573
    },
    {
      "epoch": 1.1219595976118426,
      "grad_norm": 10.063461303710938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0986044853925705,
      "logits/rejected": -0.03418441116809845,
      "logps/chosen": -356.2864074707031,
      "logps/rejected": -424.2097473144531,
      "loss": 0.8765,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0657463073730469,
      "rewards/margins": 0.38827234506607056,
      "rewards/rejected": -1.4540187120437622,
      "step": 8574
    },
    {
      "epoch": 1.1220904555491944,
      "grad_norm": 7.77247953414917,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07824073731899261,
      "logits/rejected": 0.00584927573800087,
      "logps/chosen": -578.4777221679688,
      "logps/rejected": -622.8579711914062,
      "loss": 0.4728,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13982562720775604,
      "rewards/margins": 0.5988708734512329,
      "rewards/rejected": -0.7386965155601501,
      "step": 8575
    },
    {
      "epoch": 1.1222213134865462,
      "grad_norm": 7.000572204589844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16461646556854248,
      "logits/rejected": -0.3892325758934021,
      "logps/chosen": -457.0760192871094,
      "logps/rejected": -442.1346740722656,
      "loss": 0.3401,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8909037113189697,
      "rewards/margins": 1.7432156801223755,
      "rewards/rejected": -2.6341192722320557,
      "step": 8576
    },
    {
      "epoch": 1.122352171423898,
      "grad_norm": 6.9058990478515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3392302095890045,
      "logits/rejected": 0.028200872242450714,
      "logps/chosen": -525.041748046875,
      "logps/rejected": -470.04364013671875,
      "loss": 0.4071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5320044159889221,
      "rewards/margins": 0.9149076342582703,
      "rewards/rejected": -1.4469119310379028,
      "step": 8577
    },
    {
      "epoch": 1.1224830293612498,
      "grad_norm": 11.251069068908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36601558327674866,
      "logits/rejected": -0.23185290396213531,
      "logps/chosen": -678.6828002929688,
      "logps/rejected": -369.7453308105469,
      "loss": 0.6098,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0723376274108887,
      "rewards/margins": 0.37667909264564514,
      "rewards/rejected": -2.449016809463501,
      "step": 8578
    },
    {
      "epoch": 1.1226138872986016,
      "grad_norm": 5.960334300994873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3250771760940552,
      "logits/rejected": -0.040646836161613464,
      "logps/chosen": -295.7592468261719,
      "logps/rejected": -380.684814453125,
      "loss": 0.4192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.055423378944397,
      "rewards/margins": 1.508084774017334,
      "rewards/rejected": -2.5635082721710205,
      "step": 8579
    },
    {
      "epoch": 1.1227447452359531,
      "grad_norm": 3.8937582969665527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29462915658950806,
      "logits/rejected": 0.11037395149469376,
      "logps/chosen": -432.8055725097656,
      "logps/rejected": -405.2246398925781,
      "loss": 0.1932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9050437211990356,
      "rewards/margins": 1.8829598426818848,
      "rewards/rejected": -2.78800368309021,
      "step": 8580
    },
    {
      "epoch": 1.122875603173305,
      "grad_norm": 6.202267169952393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01910937763750553,
      "logits/rejected": -0.12799875438213348,
      "logps/chosen": -539.5385131835938,
      "logps/rejected": -439.84942626953125,
      "loss": 0.3697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05693456530570984,
      "rewards/margins": 1.2687814235687256,
      "rewards/rejected": -1.2118467092514038,
      "step": 8581
    },
    {
      "epoch": 1.1230064611106567,
      "grad_norm": 7.696865558624268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.443268358707428,
      "logits/rejected": -0.45200130343437195,
      "logps/chosen": -450.738525390625,
      "logps/rejected": -550.218994140625,
      "loss": 0.4446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9033190608024597,
      "rewards/margins": 1.442478060722351,
      "rewards/rejected": -2.345796823501587,
      "step": 8582
    },
    {
      "epoch": 1.1231373190480085,
      "grad_norm": 7.448300838470459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3222408890724182,
      "logits/rejected": -0.34428584575653076,
      "logps/chosen": -463.0198974609375,
      "logps/rejected": -432.6819763183594,
      "loss": 0.4829,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7695119976997375,
      "rewards/margins": 1.1610718965530396,
      "rewards/rejected": -1.9305837154388428,
      "step": 8583
    },
    {
      "epoch": 1.1232681769853603,
      "grad_norm": 6.332037925720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02524578757584095,
      "logits/rejected": -0.16371339559555054,
      "logps/chosen": -680.470458984375,
      "logps/rejected": -550.986572265625,
      "loss": 0.3444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0344631671905518,
      "rewards/margins": 1.20613694190979,
      "rewards/rejected": -2.240600109100342,
      "step": 8584
    },
    {
      "epoch": 1.123399034922712,
      "grad_norm": 4.978139400482178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05935526266694069,
      "logits/rejected": 0.005907735787332058,
      "logps/chosen": -469.388916015625,
      "logps/rejected": -421.44415283203125,
      "loss": 0.307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14979439973831177,
      "rewards/margins": 1.6644881963729858,
      "rewards/rejected": -1.8142825365066528,
      "step": 8585
    },
    {
      "epoch": 1.1235298928600639,
      "grad_norm": 6.87546968460083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5357356071472168,
      "logits/rejected": -0.32371675968170166,
      "logps/chosen": -385.4627380371094,
      "logps/rejected": -390.1839294433594,
      "loss": 0.2939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9114089012145996,
      "rewards/margins": 1.6811141967773438,
      "rewards/rejected": -3.5925228595733643,
      "step": 8586
    },
    {
      "epoch": 1.1236607507974155,
      "grad_norm": 5.282736778259277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10974529385566711,
      "logits/rejected": -0.02739328145980835,
      "logps/chosen": -557.8884887695312,
      "logps/rejected": -511.6436767578125,
      "loss": 0.2257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7944750785827637,
      "rewards/margins": 2.4832239151000977,
      "rewards/rejected": -3.2776989936828613,
      "step": 8587
    },
    {
      "epoch": 1.1237916087347672,
      "grad_norm": 2.844897508621216,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24607250094413757,
      "logits/rejected": -0.08486641943454742,
      "logps/chosen": -438.7353515625,
      "logps/rejected": -502.69110107421875,
      "loss": 0.1105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7095955610275269,
      "rewards/margins": 3.292264699935913,
      "rewards/rejected": -4.001859664916992,
      "step": 8588
    },
    {
      "epoch": 1.123922466672119,
      "grad_norm": 4.446109294891357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14870145916938782,
      "logits/rejected": -0.14125259220600128,
      "logps/chosen": -378.2415771484375,
      "logps/rejected": -439.7643737792969,
      "loss": 0.3851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7076627612113953,
      "rewards/margins": 1.0345569849014282,
      "rewards/rejected": -1.7422198057174683,
      "step": 8589
    },
    {
      "epoch": 1.1240533246094708,
      "grad_norm": 6.96258544921875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2618310749530792,
      "logits/rejected": -0.37070491909980774,
      "logps/chosen": -484.7799072265625,
      "logps/rejected": -417.81915283203125,
      "loss": 0.4905,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0303658246994019,
      "rewards/margins": 1.1441031694412231,
      "rewards/rejected": -2.174468994140625,
      "step": 8590
    },
    {
      "epoch": 1.1241841825468226,
      "grad_norm": 6.392162799835205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08045092225074768,
      "logits/rejected": -0.31486907601356506,
      "logps/chosen": -550.513427734375,
      "logps/rejected": -400.0032653808594,
      "loss": 0.3127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8397863507270813,
      "rewards/margins": 1.5718927383422852,
      "rewards/rejected": -2.4116790294647217,
      "step": 8591
    },
    {
      "epoch": 1.1243150404841744,
      "grad_norm": 9.521749496459961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12176729738712311,
      "logits/rejected": -0.1756974160671234,
      "logps/chosen": -584.0966186523438,
      "logps/rejected": -568.701904296875,
      "loss": 0.3992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.579528272151947,
      "rewards/margins": 2.2536308765411377,
      "rewards/rejected": -1.674102544784546,
      "step": 8592
    },
    {
      "epoch": 1.1244458984215262,
      "grad_norm": 8.568727493286133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3239627182483673,
      "logits/rejected": 0.012599382549524307,
      "logps/chosen": -541.999267578125,
      "logps/rejected": -452.3394775390625,
      "loss": 0.555,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2915233373641968,
      "rewards/margins": 1.6006067991256714,
      "rewards/rejected": -1.8921302556991577,
      "step": 8593
    },
    {
      "epoch": 1.124576756358878,
      "grad_norm": 1.5574216842651367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2340867519378662,
      "logits/rejected": -0.15797096490859985,
      "logps/chosen": -563.0878295898438,
      "logps/rejected": -424.953125,
      "loss": 0.2294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18621960282325745,
      "rewards/margins": 2.474194049835205,
      "rewards/rejected": -2.2879745960235596,
      "step": 8594
    },
    {
      "epoch": 1.1247076142962296,
      "grad_norm": 7.9527082443237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043382495641708374,
      "logits/rejected": -0.0729442834854126,
      "logps/chosen": -546.8869018554688,
      "logps/rejected": -458.63720703125,
      "loss": 0.3423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30221861600875854,
      "rewards/margins": 1.6220159530639648,
      "rewards/rejected": -1.9242345094680786,
      "step": 8595
    },
    {
      "epoch": 1.1248384722335814,
      "grad_norm": 5.810466289520264,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27608537673950195,
      "logits/rejected": 0.1990954577922821,
      "logps/chosen": -425.474853515625,
      "logps/rejected": -375.72454833984375,
      "loss": 0.3727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9543359279632568,
      "rewards/margins": 1.933498740196228,
      "rewards/rejected": -2.8878347873687744,
      "step": 8596
    },
    {
      "epoch": 1.1249693301709331,
      "grad_norm": 7.780068874359131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006547026336193085,
      "logits/rejected": -0.19610652327537537,
      "logps/chosen": -275.572265625,
      "logps/rejected": -242.72109985351562,
      "loss": 0.6085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2201898097991943,
      "rewards/margins": 0.4549614191055298,
      "rewards/rejected": -1.6751513481140137,
      "step": 8597
    },
    {
      "epoch": 1.125100188108285,
      "grad_norm": 6.696033000946045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0355374850332737,
      "logits/rejected": -0.04423142969608307,
      "logps/chosen": -582.0888061523438,
      "logps/rejected": -402.5828857421875,
      "loss": 0.3855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7654086351394653,
      "rewards/margins": 1.5861256122589111,
      "rewards/rejected": -2.351534605026245,
      "step": 8598
    },
    {
      "epoch": 1.1252310460456367,
      "grad_norm": 9.431421279907227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19993409514427185,
      "logits/rejected": -0.021172888576984406,
      "logps/chosen": -359.5448303222656,
      "logps/rejected": -426.1643371582031,
      "loss": 0.7692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7789461016654968,
      "rewards/margins": 0.523684561252594,
      "rewards/rejected": -1.3026306629180908,
      "step": 8599
    },
    {
      "epoch": 1.1253619039829885,
      "grad_norm": 6.244888782501221,
      "learning_rate": 5e-06,
      "logits/chosen": 0.033236369490623474,
      "logits/rejected": -0.14888042211532593,
      "logps/chosen": -495.48980712890625,
      "logps/rejected": -534.5822143554688,
      "loss": 0.3463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6104167699813843,
      "rewards/margins": 1.3827502727508545,
      "rewards/rejected": -1.9931671619415283,
      "step": 8600
    },
    {
      "epoch": 1.1254927619203403,
      "grad_norm": 5.329633712768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04140787571668625,
      "logits/rejected": 0.08139846473932266,
      "logps/chosen": -384.4540100097656,
      "logps/rejected": -429.0203552246094,
      "loss": 0.4377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0689045190811157,
      "rewards/margins": 1.3969353437423706,
      "rewards/rejected": -2.4658398628234863,
      "step": 8601
    },
    {
      "epoch": 1.1256236198576919,
      "grad_norm": 13.81450366973877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31021201610565186,
      "logits/rejected": -0.3327156901359558,
      "logps/chosen": -561.6421508789062,
      "logps/rejected": -497.5986633300781,
      "loss": 0.9239,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0214911699295044,
      "rewards/margins": 0.06641121208667755,
      "rewards/rejected": -1.087902545928955,
      "step": 8602
    },
    {
      "epoch": 1.1257544777950437,
      "grad_norm": 8.232474327087402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14595085382461548,
      "logits/rejected": 0.1359180361032486,
      "logps/chosen": -667.9989624023438,
      "logps/rejected": -579.7667236328125,
      "loss": 0.3877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9613741040229797,
      "rewards/margins": 1.5680922269821167,
      "rewards/rejected": -2.529466152191162,
      "step": 8603
    },
    {
      "epoch": 1.1258853357323955,
      "grad_norm": 6.958706378936768,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17159658670425415,
      "logits/rejected": -0.07705092430114746,
      "logps/chosen": -304.8951416015625,
      "logps/rejected": -342.6822204589844,
      "loss": 0.5165,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37771493196487427,
      "rewards/margins": 0.8288918733596802,
      "rewards/rejected": -1.2066068649291992,
      "step": 8604
    },
    {
      "epoch": 1.1260161936697473,
      "grad_norm": 5.512491703033447,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15506576001644135,
      "logits/rejected": -0.07479864358901978,
      "logps/chosen": -415.5760192871094,
      "logps/rejected": -501.121826171875,
      "loss": 0.2503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2630385458469391,
      "rewards/margins": 2.190890073776245,
      "rewards/rejected": -1.927851676940918,
      "step": 8605
    },
    {
      "epoch": 1.126147051607099,
      "grad_norm": 5.751401901245117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09942430257797241,
      "logits/rejected": -0.21552273631095886,
      "logps/chosen": -492.5821228027344,
      "logps/rejected": -375.31890869140625,
      "loss": 0.327,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4855901896953583,
      "rewards/margins": 2.1914453506469727,
      "rewards/rejected": -2.6770358085632324,
      "step": 8606
    },
    {
      "epoch": 1.1262779095444508,
      "grad_norm": 10.705673217773438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3260536789894104,
      "logits/rejected": -0.0027636662125587463,
      "logps/chosen": -470.1811218261719,
      "logps/rejected": -385.7612609863281,
      "loss": 0.505,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32988643646240234,
      "rewards/margins": 1.046912431716919,
      "rewards/rejected": -1.3767988681793213,
      "step": 8607
    },
    {
      "epoch": 1.1264087674818026,
      "grad_norm": 5.8138813972473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025828685611486435,
      "logits/rejected": -0.027308590710163116,
      "logps/chosen": -287.9915466308594,
      "logps/rejected": -331.38232421875,
      "loss": 0.3545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24676264822483063,
      "rewards/margins": 1.6594899892807007,
      "rewards/rejected": -1.9062527418136597,
      "step": 8608
    },
    {
      "epoch": 1.1265396254191544,
      "grad_norm": 10.556800842285156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027541140094399452,
      "logits/rejected": 0.11441648751497269,
      "logps/chosen": -577.763427734375,
      "logps/rejected": -616.733642578125,
      "loss": 0.6333,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.569981336593628,
      "rewards/margins": 0.44303497672080994,
      "rewards/rejected": -2.01301646232605,
      "step": 8609
    },
    {
      "epoch": 1.1266704833565062,
      "grad_norm": 7.804429054260254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07910597324371338,
      "logits/rejected": -0.087750643491745,
      "logps/chosen": -470.7510070800781,
      "logps/rejected": -554.2108764648438,
      "loss": 0.5712,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7328618764877319,
      "rewards/margins": 0.6648756265640259,
      "rewards/rejected": -1.3977375030517578,
      "step": 8610
    },
    {
      "epoch": 1.1268013412938578,
      "grad_norm": 7.456358432769775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4683742821216583,
      "logits/rejected": -0.3690108358860016,
      "logps/chosen": -349.3232421875,
      "logps/rejected": -547.0377197265625,
      "loss": 0.3131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2459615468978882,
      "rewards/margins": 1.6961560249328613,
      "rewards/rejected": -2.94211745262146,
      "step": 8611
    },
    {
      "epoch": 1.1269321992312096,
      "grad_norm": 7.639649391174316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.175340473651886,
      "logits/rejected": 0.11391468346118927,
      "logps/chosen": -479.2343444824219,
      "logps/rejected": -532.6551513671875,
      "loss": 0.4323,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5074321627616882,
      "rewards/margins": 1.5214229822158813,
      "rewards/rejected": -1.0139904022216797,
      "step": 8612
    },
    {
      "epoch": 1.1270630571685614,
      "grad_norm": 11.343599319458008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.211710587143898,
      "logits/rejected": 0.09808602929115295,
      "logps/chosen": -597.349365234375,
      "logps/rejected": -502.9068603515625,
      "loss": 0.8384,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.920941174030304,
      "rewards/margins": 0.33982351422309875,
      "rewards/rejected": -1.2607645988464355,
      "step": 8613
    },
    {
      "epoch": 1.1271939151059132,
      "grad_norm": 7.848577499389648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014003602787852287,
      "logits/rejected": -0.26231351494789124,
      "logps/chosen": -699.4850463867188,
      "logps/rejected": -412.3587341308594,
      "loss": 0.2914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0759519636631012,
      "rewards/margins": 1.8703994750976562,
      "rewards/rejected": -1.9463515281677246,
      "step": 8614
    },
    {
      "epoch": 1.127324773043265,
      "grad_norm": 7.473202228546143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011255085468292236,
      "logits/rejected": -0.02259562909603119,
      "logps/chosen": -325.7994079589844,
      "logps/rejected": -421.1866149902344,
      "loss": 0.6217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23820264637470245,
      "rewards/margins": 0.8245294690132141,
      "rewards/rejected": -1.0627321004867554,
      "step": 8615
    },
    {
      "epoch": 1.1274556309806167,
      "grad_norm": 8.247536659240723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06917694956064224,
      "logits/rejected": 0.03503735363483429,
      "logps/chosen": -369.23980712890625,
      "logps/rejected": -502.9114990234375,
      "loss": 0.6402,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.600368857383728,
      "rewards/margins": 0.5943393707275391,
      "rewards/rejected": -1.194708228111267,
      "step": 8616
    },
    {
      "epoch": 1.1275864889179683,
      "grad_norm": 10.267242431640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08339948207139969,
      "logits/rejected": -0.04516031593084335,
      "logps/chosen": -438.5913391113281,
      "logps/rejected": -462.21588134765625,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5597795248031616,
      "rewards/margins": 0.9047083258628845,
      "rewards/rejected": -1.4644877910614014,
      "step": 8617
    },
    {
      "epoch": 1.12771734685532,
      "grad_norm": 9.504193305969238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06864649057388306,
      "logits/rejected": -0.26406508684158325,
      "logps/chosen": -493.50604248046875,
      "logps/rejected": -448.39019775390625,
      "loss": 0.4759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1237131357192993,
      "rewards/margins": 0.8270483016967773,
      "rewards/rejected": -1.9507614374160767,
      "step": 8618
    },
    {
      "epoch": 1.127848204792672,
      "grad_norm": 7.252764701843262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14524029195308685,
      "logits/rejected": 0.051881272345781326,
      "logps/chosen": -469.16546630859375,
      "logps/rejected": -344.60015869140625,
      "loss": 0.4712,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.443040668964386,
      "rewards/margins": 1.0260064601898193,
      "rewards/rejected": -1.46904718875885,
      "step": 8619
    },
    {
      "epoch": 1.1279790627300237,
      "grad_norm": 7.394859790802002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24004970490932465,
      "logits/rejected": -0.3914673924446106,
      "logps/chosen": -468.9646911621094,
      "logps/rejected": -326.8578186035156,
      "loss": 0.5799,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22671452164649963,
      "rewards/margins": 0.5394630432128906,
      "rewards/rejected": -0.7661775350570679,
      "step": 8620
    },
    {
      "epoch": 1.1281099206673755,
      "grad_norm": 4.72526216506958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2480999380350113,
      "logits/rejected": -0.23677313327789307,
      "logps/chosen": -532.4566040039062,
      "logps/rejected": -414.1652526855469,
      "loss": 0.2475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4645843505859375,
      "rewards/margins": 1.8963055610656738,
      "rewards/rejected": -2.3608899116516113,
      "step": 8621
    },
    {
      "epoch": 1.1282407786047273,
      "grad_norm": 11.543533325195312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30270272493362427,
      "logits/rejected": -0.4697323739528656,
      "logps/chosen": -601.9930419921875,
      "logps/rejected": -598.1888427734375,
      "loss": 0.673,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.930928111076355,
      "rewards/margins": 0.8237617015838623,
      "rewards/rejected": -1.7546896934509277,
      "step": 8622
    },
    {
      "epoch": 1.128371636542079,
      "grad_norm": 4.890889644622803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29545390605926514,
      "logits/rejected": -0.4264790117740631,
      "logps/chosen": -304.28387451171875,
      "logps/rejected": -456.8934631347656,
      "loss": 0.2414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35455742478370667,
      "rewards/margins": 2.5924630165100098,
      "rewards/rejected": -2.9470205307006836,
      "step": 8623
    },
    {
      "epoch": 1.1285024944794309,
      "grad_norm": 6.305055141448975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15069477260112762,
      "logits/rejected": -0.003913372755050659,
      "logps/chosen": -455.35748291015625,
      "logps/rejected": -323.8936767578125,
      "loss": 0.443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3104303479194641,
      "rewards/margins": 1.3903000354766846,
      "rewards/rejected": -1.7007304430007935,
      "step": 8624
    },
    {
      "epoch": 1.1286333524167826,
      "grad_norm": 10.888917922973633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04930464178323746,
      "logits/rejected": 0.04403065890073776,
      "logps/chosen": -521.8756103515625,
      "logps/rejected": -409.04345703125,
      "loss": 0.7169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04219568520784378,
      "rewards/margins": 0.39560410380363464,
      "rewards/rejected": -0.35340845584869385,
      "step": 8625
    },
    {
      "epoch": 1.1287642103541342,
      "grad_norm": 5.9017333984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1226329654455185,
      "logits/rejected": -0.1687013804912567,
      "logps/chosen": -545.0814208984375,
      "logps/rejected": -399.29010009765625,
      "loss": 0.2532,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.004991525784134865,
      "rewards/margins": 1.6099002361297607,
      "rewards/rejected": -1.6049085855484009,
      "step": 8626
    },
    {
      "epoch": 1.128895068291486,
      "grad_norm": 8.551995277404785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2733605206012726,
      "logits/rejected": -0.15138331055641174,
      "logps/chosen": -356.9750061035156,
      "logps/rejected": -404.6549377441406,
      "loss": 0.4278,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9196826815605164,
      "rewards/margins": 1.5670915842056274,
      "rewards/rejected": -2.486774444580078,
      "step": 8627
    },
    {
      "epoch": 1.1290259262288378,
      "grad_norm": 3.359160900115967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11733647435903549,
      "logits/rejected": -0.21114327013492584,
      "logps/chosen": -358.04248046875,
      "logps/rejected": -446.9594421386719,
      "loss": 0.175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25613078474998474,
      "rewards/margins": 2.2101871967315674,
      "rewards/rejected": -1.9540565013885498,
      "step": 8628
    },
    {
      "epoch": 1.1291567841661896,
      "grad_norm": 11.308433532714844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1380126029253006,
      "logits/rejected": -0.10572553426027298,
      "logps/chosen": -520.7920532226562,
      "logps/rejected": -486.5530090332031,
      "loss": 0.819,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6190977096557617,
      "rewards/margins": 0.06057186797261238,
      "rewards/rejected": -0.6796696186065674,
      "step": 8629
    },
    {
      "epoch": 1.1292876421035414,
      "grad_norm": 6.923962593078613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12292982637882233,
      "logits/rejected": -0.1561794877052307,
      "logps/chosen": -503.60614013671875,
      "logps/rejected": -509.9742736816406,
      "loss": 0.2894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10780567675828934,
      "rewards/margins": 1.3462908267974854,
      "rewards/rejected": -1.238485336303711,
      "step": 8630
    },
    {
      "epoch": 1.1294185000408932,
      "grad_norm": 6.741246223449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19628573954105377,
      "logits/rejected": -0.17913560569286346,
      "logps/chosen": -493.63006591796875,
      "logps/rejected": -363.195556640625,
      "loss": 0.4076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7709048986434937,
      "rewards/margins": 0.9242754578590393,
      "rewards/rejected": -1.6951805353164673,
      "step": 8631
    },
    {
      "epoch": 1.129549357978245,
      "grad_norm": 6.365873336791992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31791502237319946,
      "logits/rejected": -0.38611650466918945,
      "logps/chosen": -608.8921508789062,
      "logps/rejected": -512.2264404296875,
      "loss": 0.4732,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002359062433242798,
      "rewards/margins": 1.70139741897583,
      "rewards/rejected": -1.6990382671356201,
      "step": 8632
    },
    {
      "epoch": 1.1296802159155965,
      "grad_norm": 5.181854724884033,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1609431803226471,
      "logits/rejected": -0.025102611631155014,
      "logps/chosen": -334.29046630859375,
      "logps/rejected": -358.183349609375,
      "loss": 0.2845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31607839465141296,
      "rewards/margins": 2.1245036125183105,
      "rewards/rejected": -2.440582036972046,
      "step": 8633
    },
    {
      "epoch": 1.1298110738529483,
      "grad_norm": 7.36511754989624,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0093812495470047,
      "logits/rejected": -0.09428323060274124,
      "logps/chosen": -433.484375,
      "logps/rejected": -401.86590576171875,
      "loss": 0.3693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4912199378013611,
      "rewards/margins": 1.4438228607177734,
      "rewards/rejected": -1.9350428581237793,
      "step": 8634
    },
    {
      "epoch": 1.1299419317903001,
      "grad_norm": 5.785655975341797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041537247598171234,
      "logits/rejected": -0.2319607436656952,
      "logps/chosen": -444.26678466796875,
      "logps/rejected": -476.9472961425781,
      "loss": 0.3797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4065234959125519,
      "rewards/margins": 1.4901936054229736,
      "rewards/rejected": -1.8967170715332031,
      "step": 8635
    },
    {
      "epoch": 1.130072789727652,
      "grad_norm": 10.32390308380127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19411399960517883,
      "logits/rejected": -0.33726200461387634,
      "logps/chosen": -473.0733947753906,
      "logps/rejected": -387.137451171875,
      "loss": 0.6023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1817042827606201,
      "rewards/margins": 0.7475314736366272,
      "rewards/rejected": -1.9292359352111816,
      "step": 8636
    },
    {
      "epoch": 1.1302036476650037,
      "grad_norm": 11.394613265991211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2338022142648697,
      "logits/rejected": 0.15765926241874695,
      "logps/chosen": -555.4954223632812,
      "logps/rejected": -334.3858337402344,
      "loss": 0.5285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6006185412406921,
      "rewards/margins": 0.7759327292442322,
      "rewards/rejected": -1.3765512704849243,
      "step": 8637
    },
    {
      "epoch": 1.1303345056023555,
      "grad_norm": 5.584282398223877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10228089988231659,
      "logits/rejected": -0.16326384246349335,
      "logps/chosen": -350.579833984375,
      "logps/rejected": -273.8037109375,
      "loss": 0.3997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6083654165267944,
      "rewards/margins": 1.336485743522644,
      "rewards/rejected": -1.944851279258728,
      "step": 8638
    },
    {
      "epoch": 1.1304653635397073,
      "grad_norm": 8.241634368896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0707118958234787,
      "logits/rejected": -0.4403505325317383,
      "logps/chosen": -376.4017639160156,
      "logps/rejected": -342.1878967285156,
      "loss": 0.5074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10276928544044495,
      "rewards/margins": 0.7537298202514648,
      "rewards/rejected": -0.8564991354942322,
      "step": 8639
    },
    {
      "epoch": 1.130596221477059,
      "grad_norm": 5.652411937713623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17738689482212067,
      "logits/rejected": -0.3175630569458008,
      "logps/chosen": -496.893798828125,
      "logps/rejected": -393.0625305175781,
      "loss": 0.3057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.129964679479599,
      "rewards/margins": 2.0104100704193115,
      "rewards/rejected": -1.8804453611373901,
      "step": 8640
    },
    {
      "epoch": 1.1307270794144106,
      "grad_norm": 9.420351028442383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4988994598388672,
      "logits/rejected": 0.31356877088546753,
      "logps/chosen": -457.87322998046875,
      "logps/rejected": -387.765869140625,
      "loss": 0.9004,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3882409930229187,
      "rewards/margins": 0.5881380438804626,
      "rewards/rejected": -0.9763790369033813,
      "step": 8641
    },
    {
      "epoch": 1.1308579373517624,
      "grad_norm": 9.112151145935059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20467983186244965,
      "logits/rejected": -0.09285829216241837,
      "logps/chosen": -421.71923828125,
      "logps/rejected": -436.6324462890625,
      "loss": 0.4923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6181280016899109,
      "rewards/margins": 1.4112842082977295,
      "rewards/rejected": -2.029412269592285,
      "step": 8642
    },
    {
      "epoch": 1.1309887952891142,
      "grad_norm": 7.8928303718566895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11452323198318481,
      "logits/rejected": -0.2948075830936432,
      "logps/chosen": -421.3061218261719,
      "logps/rejected": -413.5640563964844,
      "loss": 0.5769,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7889037132263184,
      "rewards/margins": 0.5615605115890503,
      "rewards/rejected": -1.3504643440246582,
      "step": 8643
    },
    {
      "epoch": 1.131119653226466,
      "grad_norm": 4.905259132385254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30743712186813354,
      "logits/rejected": 0.23793677985668182,
      "logps/chosen": -428.3182678222656,
      "logps/rejected": -412.5636291503906,
      "loss": 0.3024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4898277521133423,
      "rewards/margins": 1.70072603225708,
      "rewards/rejected": -2.190553665161133,
      "step": 8644
    },
    {
      "epoch": 1.1312505111638178,
      "grad_norm": 5.705611705780029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17853499948978424,
      "logits/rejected": -0.12974998354911804,
      "logps/chosen": -434.0780029296875,
      "logps/rejected": -491.55322265625,
      "loss": 0.2569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7246686816215515,
      "rewards/margins": 1.9175121784210205,
      "rewards/rejected": -1.1928434371948242,
      "step": 8645
    },
    {
      "epoch": 1.1313813691011696,
      "grad_norm": 9.051140785217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24756623804569244,
      "logits/rejected": -0.21023133397102356,
      "logps/chosen": -569.1295166015625,
      "logps/rejected": -524.6463623046875,
      "loss": 0.6654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07497071474790573,
      "rewards/margins": 0.5991929173469543,
      "rewards/rejected": -0.6741635799407959,
      "step": 8646
    },
    {
      "epoch": 1.1315122270385214,
      "grad_norm": 10.102995872497559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24379853904247284,
      "logits/rejected": -0.3766075670719147,
      "logps/chosen": -427.821044921875,
      "logps/rejected": -378.7335205078125,
      "loss": 0.7135,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1046079397201538,
      "rewards/margins": 0.2352718561887741,
      "rewards/rejected": -1.3398798704147339,
      "step": 8647
    },
    {
      "epoch": 1.131643084975873,
      "grad_norm": 7.2449798583984375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2007879912853241,
      "logits/rejected": 0.08263794332742691,
      "logps/chosen": -419.7965393066406,
      "logps/rejected": -493.60186767578125,
      "loss": 0.4397,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17944909632205963,
      "rewards/margins": 0.7528902888298035,
      "rewards/rejected": -0.9323393106460571,
      "step": 8648
    },
    {
      "epoch": 1.1317739429132248,
      "grad_norm": 7.356621265411377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00444580614566803,
      "logits/rejected": -0.028745830059051514,
      "logps/chosen": -428.5509033203125,
      "logps/rejected": -412.4722900390625,
      "loss": 0.4818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3788570761680603,
      "rewards/margins": 0.7936056852340698,
      "rewards/rejected": -1.1724627017974854,
      "step": 8649
    },
    {
      "epoch": 1.1319048008505765,
      "grad_norm": 3.8826911449432373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004316940903663635,
      "logits/rejected": -0.12904217839241028,
      "logps/chosen": -621.79052734375,
      "logps/rejected": -609.4203491210938,
      "loss": 0.1726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6978846788406372,
      "rewards/margins": 2.298269271850586,
      "rewards/rejected": -1.6003843545913696,
      "step": 8650
    },
    {
      "epoch": 1.1320356587879283,
      "grad_norm": 7.897884368896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10397683829069138,
      "logits/rejected": -0.15778282284736633,
      "logps/chosen": -485.53271484375,
      "logps/rejected": -461.19384765625,
      "loss": 0.6768,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8637834787368774,
      "rewards/margins": 0.6950905323028564,
      "rewards/rejected": -1.5588738918304443,
      "step": 8651
    },
    {
      "epoch": 1.1321665167252801,
      "grad_norm": 9.554642677307129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1912807822227478,
      "logits/rejected": -0.3231586217880249,
      "logps/chosen": -402.70220947265625,
      "logps/rejected": -412.1500244140625,
      "loss": 0.7364,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.514324963092804,
      "rewards/margins": 0.6540481448173523,
      "rewards/rejected": -1.1683731079101562,
      "step": 8652
    },
    {
      "epoch": 1.132297374662632,
      "grad_norm": 6.322038173675537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1276986300945282,
      "logits/rejected": -0.4093235731124878,
      "logps/chosen": -588.5433349609375,
      "logps/rejected": -452.49365234375,
      "loss": 0.3215,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21100559830665588,
      "rewards/margins": 1.6293880939483643,
      "rewards/rejected": -1.8403939008712769,
      "step": 8653
    },
    {
      "epoch": 1.1324282325999837,
      "grad_norm": 7.474857330322266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06488746404647827,
      "logits/rejected": -0.02424820512533188,
      "logps/chosen": -348.1428527832031,
      "logps/rejected": -430.2021789550781,
      "loss": 0.5539,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0649006366729736,
      "rewards/margins": 0.6789194941520691,
      "rewards/rejected": -1.7438201904296875,
      "step": 8654
    },
    {
      "epoch": 1.1325590905373355,
      "grad_norm": 4.367674827575684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17203035950660706,
      "logits/rejected": -0.2394556999206543,
      "logps/chosen": -262.5689697265625,
      "logps/rejected": -329.84344482421875,
      "loss": 0.189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.033903956413269,
      "rewards/margins": 1.9472142457962036,
      "rewards/rejected": -2.9811179637908936,
      "step": 8655
    },
    {
      "epoch": 1.1326899484746873,
      "grad_norm": 6.710517883300781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.180209681391716,
      "logits/rejected": -0.15912213921546936,
      "logps/chosen": -315.7021789550781,
      "logps/rejected": -408.5183410644531,
      "loss": 0.452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5148334503173828,
      "rewards/margins": 1.070412516593933,
      "rewards/rejected": -1.585245966911316,
      "step": 8656
    },
    {
      "epoch": 1.1328208064120389,
      "grad_norm": 3.5831801891326904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3644576966762543,
      "logits/rejected": -0.15214258432388306,
      "logps/chosen": -369.03936767578125,
      "logps/rejected": -444.2048645019531,
      "loss": 0.1274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17426873743534088,
      "rewards/margins": 2.726193428039551,
      "rewards/rejected": -2.9004621505737305,
      "step": 8657
    },
    {
      "epoch": 1.1329516643493907,
      "grad_norm": 3.3666863441467285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10861047357320786,
      "logits/rejected": 0.07202146202325821,
      "logps/chosen": -487.6416931152344,
      "logps/rejected": -462.1309509277344,
      "loss": 0.1697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5778125524520874,
      "rewards/margins": 2.2155377864837646,
      "rewards/rejected": -2.7933504581451416,
      "step": 8658
    },
    {
      "epoch": 1.1330825222867424,
      "grad_norm": 3.2902047634124756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06514890491962433,
      "logits/rejected": -0.1867322325706482,
      "logps/chosen": -332.2718811035156,
      "logps/rejected": -310.38287353515625,
      "loss": 0.2624,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6554428339004517,
      "rewards/margins": 2.2967374324798584,
      "rewards/rejected": -2.9521801471710205,
      "step": 8659
    },
    {
      "epoch": 1.1332133802240942,
      "grad_norm": 8.507831573486328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01396845281124115,
      "logits/rejected": -0.18859818577766418,
      "logps/chosen": -481.2868957519531,
      "logps/rejected": -468.4680480957031,
      "loss": 0.5498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1307737827301025,
      "rewards/margins": 0.37046682834625244,
      "rewards/rejected": -1.5012404918670654,
      "step": 8660
    },
    {
      "epoch": 1.133344238161446,
      "grad_norm": 4.594447612762451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06845705956220627,
      "logits/rejected": -0.28891995549201965,
      "logps/chosen": -481.8901672363281,
      "logps/rejected": -313.6068420410156,
      "loss": 0.2223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4234766364097595,
      "rewards/margins": 2.0196774005889893,
      "rewards/rejected": -2.4431540966033936,
      "step": 8661
    },
    {
      "epoch": 1.1334750960987978,
      "grad_norm": 9.566681861877441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1979457288980484,
      "logits/rejected": -0.2845171391963959,
      "logps/chosen": -601.462890625,
      "logps/rejected": -552.6514892578125,
      "loss": 0.3949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4305448532104492,
      "rewards/margins": 1.463271141052246,
      "rewards/rejected": -1.8938159942626953,
      "step": 8662
    },
    {
      "epoch": 1.1336059540361494,
      "grad_norm": 5.482456207275391,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08280839025974274,
      "logits/rejected": -0.024907080456614494,
      "logps/chosen": -466.382080078125,
      "logps/rejected": -325.08416748046875,
      "loss": 0.2907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1234936714172363,
      "rewards/margins": 1.2099051475524902,
      "rewards/rejected": -2.3333988189697266,
      "step": 8663
    },
    {
      "epoch": 1.1337368119735012,
      "grad_norm": 13.118890762329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03663094714283943,
      "logits/rejected": 0.1731259524822235,
      "logps/chosen": -529.8347778320312,
      "logps/rejected": -537.4224853515625,
      "loss": 0.6333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.464284896850586,
      "rewards/margins": 1.0202029943466187,
      "rewards/rejected": -2.484488010406494,
      "step": 8664
    },
    {
      "epoch": 1.133867669910853,
      "grad_norm": 6.779398441314697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3343197703361511,
      "logits/rejected": 0.2134072333574295,
      "logps/chosen": -473.15313720703125,
      "logps/rejected": -679.9185791015625,
      "loss": 0.4704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3392379879951477,
      "rewards/margins": 1.7593519687652588,
      "rewards/rejected": -2.0985898971557617,
      "step": 8665
    },
    {
      "epoch": 1.1339985278482048,
      "grad_norm": 6.6986002922058105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.061221908777952194,
      "logits/rejected": -0.08564487099647522,
      "logps/chosen": -331.9333801269531,
      "logps/rejected": -287.71514892578125,
      "loss": 0.5019,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.255490779876709,
      "rewards/margins": 1.0933321714401245,
      "rewards/rejected": -2.348822832107544,
      "step": 8666
    },
    {
      "epoch": 1.1341293857855566,
      "grad_norm": 4.475338459014893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19208937883377075,
      "logits/rejected": -0.10175351053476334,
      "logps/chosen": -422.97052001953125,
      "logps/rejected": -418.5183410644531,
      "loss": 0.2902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11189860105514526,
      "rewards/margins": 1.7159276008605957,
      "rewards/rejected": -1.6040291786193848,
      "step": 8667
    },
    {
      "epoch": 1.1342602437229083,
      "grad_norm": 4.532854080200195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1720845103263855,
      "logits/rejected": 0.08122643083333969,
      "logps/chosen": -480.339111328125,
      "logps/rejected": -501.0957946777344,
      "loss": 0.2381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.282443642616272,
      "rewards/margins": 2.25846529006958,
      "rewards/rejected": -1.976021647453308,
      "step": 8668
    },
    {
      "epoch": 1.1343911016602601,
      "grad_norm": 6.180578708648682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01204722374677658,
      "logits/rejected": -0.1429334133863449,
      "logps/chosen": -467.5292663574219,
      "logps/rejected": -482.502197265625,
      "loss": 0.324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38325005769729614,
      "rewards/margins": 1.606740951538086,
      "rewards/rejected": -1.2234909534454346,
      "step": 8669
    },
    {
      "epoch": 1.134521959597612,
      "grad_norm": 8.32454776763916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012776188552379608,
      "logits/rejected": 0.007197216153144836,
      "logps/chosen": -412.4609680175781,
      "logps/rejected": -400.3218688964844,
      "loss": 0.3611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13506852090358734,
      "rewards/margins": 1.2285358905792236,
      "rewards/rejected": -1.3636043071746826,
      "step": 8670
    },
    {
      "epoch": 1.1346528175349637,
      "grad_norm": 8.042325973510742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0724971741437912,
      "logits/rejected": -0.10672634840011597,
      "logps/chosen": -405.50433349609375,
      "logps/rejected": -307.9176940917969,
      "loss": 0.3974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47667303681373596,
      "rewards/margins": 1.4937691688537598,
      "rewards/rejected": -1.9704421758651733,
      "step": 8671
    },
    {
      "epoch": 1.1347836754723153,
      "grad_norm": 5.7843756675720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05137297883629799,
      "logits/rejected": -0.09487925469875336,
      "logps/chosen": -424.38165283203125,
      "logps/rejected": -534.2787475585938,
      "loss": 0.268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5974642038345337,
      "rewards/margins": 1.4558840990066528,
      "rewards/rejected": -2.0533483028411865,
      "step": 8672
    },
    {
      "epoch": 1.134914533409667,
      "grad_norm": 7.213101863861084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058945801109075546,
      "logits/rejected": -0.18734358251094818,
      "logps/chosen": -407.1990966796875,
      "logps/rejected": -369.3077697753906,
      "loss": 0.5587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18549425899982452,
      "rewards/margins": 1.25350821018219,
      "rewards/rejected": -1.439002513885498,
      "step": 8673
    },
    {
      "epoch": 1.1350453913470189,
      "grad_norm": 11.099157333374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42668044567108154,
      "logits/rejected": -0.21320918202400208,
      "logps/chosen": -671.9454345703125,
      "logps/rejected": -549.7603149414062,
      "loss": 0.6578,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1759464740753174,
      "rewards/margins": 0.24532222747802734,
      "rewards/rejected": -1.4212687015533447,
      "step": 8674
    },
    {
      "epoch": 1.1351762492843707,
      "grad_norm": 11.001927375793457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026772532612085342,
      "logits/rejected": -0.007968127727508545,
      "logps/chosen": -395.9322509765625,
      "logps/rejected": -540.5945434570312,
      "loss": 0.7558,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4135381281375885,
      "rewards/margins": 0.915967583656311,
      "rewards/rejected": -1.3295058012008667,
      "step": 8675
    },
    {
      "epoch": 1.1353071072217225,
      "grad_norm": 10.50944995880127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13458049297332764,
      "logits/rejected": -0.09569916874170303,
      "logps/chosen": -377.28326416015625,
      "logps/rejected": -514.0128784179688,
      "loss": 0.9108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1195061206817627,
      "rewards/margins": 0.6796776056289673,
      "rewards/rejected": -1.7991838455200195,
      "step": 8676
    },
    {
      "epoch": 1.1354379651590742,
      "grad_norm": 4.719915390014648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09818418323993683,
      "logits/rejected": -0.11051484942436218,
      "logps/chosen": -351.26409912109375,
      "logps/rejected": -491.3611755371094,
      "loss": 0.3128,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17084094882011414,
      "rewards/margins": 1.9440796375274658,
      "rewards/rejected": -2.1149206161499023,
      "step": 8677
    },
    {
      "epoch": 1.135568823096426,
      "grad_norm": 6.761420726776123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041966356337070465,
      "logits/rejected": -0.12297860532999039,
      "logps/chosen": -399.4526672363281,
      "logps/rejected": -337.2232666015625,
      "loss": 0.415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18979458510875702,
      "rewards/margins": 0.9815933108329773,
      "rewards/rejected": -1.1713879108428955,
      "step": 8678
    },
    {
      "epoch": 1.1356996810337776,
      "grad_norm": 5.91206693649292,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30133941769599915,
      "logits/rejected": 0.2147236317396164,
      "logps/chosen": -395.3582763671875,
      "logps/rejected": -364.34991455078125,
      "loss": 0.3309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6925722360610962,
      "rewards/margins": 1.2883893251419067,
      "rewards/rejected": -1.9809613227844238,
      "step": 8679
    },
    {
      "epoch": 1.1358305389711294,
      "grad_norm": 5.402454376220703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022151019424200058,
      "logits/rejected": -0.031738899648189545,
      "logps/chosen": -400.88641357421875,
      "logps/rejected": -398.955322265625,
      "loss": 0.3787,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10627996921539307,
      "rewards/margins": 1.7981231212615967,
      "rewards/rejected": -1.9044030904769897,
      "step": 8680
    },
    {
      "epoch": 1.1359613969084812,
      "grad_norm": 9.755935668945312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13060638308525085,
      "logits/rejected": -0.006516765803098679,
      "logps/chosen": -529.9264526367188,
      "logps/rejected": -556.50537109375,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.149763584136963,
      "rewards/margins": 0.4935961067676544,
      "rewards/rejected": -1.643359899520874,
      "step": 8681
    },
    {
      "epoch": 1.136092254845833,
      "grad_norm": 6.318811893463135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03447030484676361,
      "logits/rejected": -0.08285307884216309,
      "logps/chosen": -514.2091064453125,
      "logps/rejected": -373.6846923828125,
      "loss": 0.4035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4917648434638977,
      "rewards/margins": 1.2100025415420532,
      "rewards/rejected": -1.7017674446105957,
      "step": 8682
    },
    {
      "epoch": 1.1362231127831848,
      "grad_norm": 6.775482654571533,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01839321479201317,
      "logits/rejected": -0.2775958776473999,
      "logps/chosen": -440.99053955078125,
      "logps/rejected": -640.2999877929688,
      "loss": 0.4212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2909746170043945,
      "rewards/margins": 1.1756067276000977,
      "rewards/rejected": -2.466581344604492,
      "step": 8683
    },
    {
      "epoch": 1.1363539707205366,
      "grad_norm": 6.02756929397583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25876936316490173,
      "logits/rejected": -0.3440667986869812,
      "logps/chosen": -630.8673706054688,
      "logps/rejected": -449.7880859375,
      "loss": 0.2651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6401077508926392,
      "rewards/margins": 2.231452226638794,
      "rewards/rejected": -1.5913443565368652,
      "step": 8684
    },
    {
      "epoch": 1.1364848286578884,
      "grad_norm": 6.987438201904297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18033577501773834,
      "logits/rejected": -0.3527454435825348,
      "logps/chosen": -398.5270080566406,
      "logps/rejected": -400.6536865234375,
      "loss": 0.5262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9702390432357788,
      "rewards/margins": 1.2260570526123047,
      "rewards/rejected": -2.196295976638794,
      "step": 8685
    },
    {
      "epoch": 1.1366156865952401,
      "grad_norm": 7.7072672843933105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16577625274658203,
      "logits/rejected": -0.1160501092672348,
      "logps/chosen": -529.1030883789062,
      "logps/rejected": -360.77899169921875,
      "loss": 0.4687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0222797393798828,
      "rewards/margins": 1.2022120952606201,
      "rewards/rejected": -2.224491834640503,
      "step": 8686
    },
    {
      "epoch": 1.1367465445325917,
      "grad_norm": 8.128006935119629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14535605907440186,
      "logits/rejected": -0.0726332888007164,
      "logps/chosen": -469.7139587402344,
      "logps/rejected": -449.1273193359375,
      "loss": 0.4975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4915485382080078,
      "rewards/margins": 0.773464560508728,
      "rewards/rejected": -1.2650130987167358,
      "step": 8687
    },
    {
      "epoch": 1.1368774024699435,
      "grad_norm": 8.594221115112305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4161142408847809,
      "logits/rejected": -0.3570609390735626,
      "logps/chosen": -353.713134765625,
      "logps/rejected": -334.7205810546875,
      "loss": 0.4037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.323817491531372,
      "rewards/margins": 1.5926735401153564,
      "rewards/rejected": -2.9164910316467285,
      "step": 8688
    },
    {
      "epoch": 1.1370082604072953,
      "grad_norm": 5.450127124786377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2023012340068817,
      "logits/rejected": -0.047505684196949005,
      "logps/chosen": -302.2445373535156,
      "logps/rejected": -253.51280212402344,
      "loss": 0.4528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2261312007904053,
      "rewards/margins": 1.098404049873352,
      "rewards/rejected": -2.324535369873047,
      "step": 8689
    },
    {
      "epoch": 1.137139118344647,
      "grad_norm": 13.558929443359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012718182057142258,
      "logits/rejected": 0.03268793970346451,
      "logps/chosen": -521.0426635742188,
      "logps/rejected": -339.1861572265625,
      "loss": 0.6379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9435181021690369,
      "rewards/margins": 1.5651774406433105,
      "rewards/rejected": -2.508695602416992,
      "step": 8690
    },
    {
      "epoch": 1.1372699762819989,
      "grad_norm": 7.039478302001953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2720567584037781,
      "logits/rejected": -0.5908322334289551,
      "logps/chosen": -354.5201721191406,
      "logps/rejected": -466.7918701171875,
      "loss": 0.3599,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7837089896202087,
      "rewards/margins": 1.9131977558135986,
      "rewards/rejected": -2.696906805038452,
      "step": 8691
    },
    {
      "epoch": 1.1374008342193507,
      "grad_norm": 7.848887920379639,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22409385442733765,
      "logits/rejected": -0.43981343507766724,
      "logps/chosen": -371.67120361328125,
      "logps/rejected": -435.44952392578125,
      "loss": 0.5319,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9877015352249146,
      "rewards/margins": 0.760809063911438,
      "rewards/rejected": -2.7485105991363525,
      "step": 8692
    },
    {
      "epoch": 1.1375316921567025,
      "grad_norm": 6.42428731918335,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01261311024427414,
      "logits/rejected": -0.25007960200309753,
      "logps/chosen": -631.5200805664062,
      "logps/rejected": -415.8387145996094,
      "loss": 0.3497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38228505849838257,
      "rewards/margins": 1.396323800086975,
      "rewards/rejected": -1.7786089181900024,
      "step": 8693
    },
    {
      "epoch": 1.137662550094054,
      "grad_norm": 7.385499477386475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013412967324256897,
      "logits/rejected": -0.02876855432987213,
      "logps/chosen": -806.4125366210938,
      "logps/rejected": -686.6510620117188,
      "loss": 0.3781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.068456269800663,
      "rewards/margins": 1.400539755821228,
      "rewards/rejected": -1.4689960479736328,
      "step": 8694
    },
    {
      "epoch": 1.1377934080314058,
      "grad_norm": 7.533767223358154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1444540023803711,
      "logits/rejected": -0.02664896287024021,
      "logps/chosen": -688.1400146484375,
      "logps/rejected": -510.7835998535156,
      "loss": 0.3975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7315278649330139,
      "rewards/margins": 1.0988709926605225,
      "rewards/rejected": -1.8303989171981812,
      "step": 8695
    },
    {
      "epoch": 1.1379242659687576,
      "grad_norm": 10.252739906311035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030356917530298233,
      "logits/rejected": -0.05569644272327423,
      "logps/chosen": -412.18524169921875,
      "logps/rejected": -349.8583984375,
      "loss": 0.5905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7644628286361694,
      "rewards/margins": 0.371066153049469,
      "rewards/rejected": -1.1355290412902832,
      "step": 8696
    },
    {
      "epoch": 1.1380551239061094,
      "grad_norm": 8.991931915283203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34414899349212646,
      "logits/rejected": -0.41251683235168457,
      "logps/chosen": -534.3551635742188,
      "logps/rejected": -416.5252380371094,
      "loss": 0.3691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6963432431221008,
      "rewards/margins": 1.3582099676132202,
      "rewards/rejected": -2.054553270339966,
      "step": 8697
    },
    {
      "epoch": 1.1381859818434612,
      "grad_norm": 4.91087532043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3060027062892914,
      "logits/rejected": -0.38369274139404297,
      "logps/chosen": -613.7470092773438,
      "logps/rejected": -590.9111328125,
      "loss": 0.2007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8524147272109985,
      "rewards/margins": 2.497349500656128,
      "rewards/rejected": -3.349764347076416,
      "step": 8698
    },
    {
      "epoch": 1.138316839780813,
      "grad_norm": 7.630808353424072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06604616343975067,
      "logits/rejected": -0.0006306832656264305,
      "logps/chosen": -501.69586181640625,
      "logps/rejected": -521.6729736328125,
      "loss": 0.3394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39028722047805786,
      "rewards/margins": 1.4609348773956299,
      "rewards/rejected": -1.851222038269043,
      "step": 8699
    },
    {
      "epoch": 1.1384476977181648,
      "grad_norm": 7.153336524963379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3421841561794281,
      "logits/rejected": -0.43244659900665283,
      "logps/chosen": -381.91717529296875,
      "logps/rejected": -592.6962890625,
      "loss": 0.2589,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9481396079063416,
      "rewards/margins": 2.4772145748138428,
      "rewards/rejected": -3.425354242324829,
      "step": 8700
    },
    {
      "epoch": 1.1385785556555166,
      "grad_norm": 7.722939491271973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24003487825393677,
      "logits/rejected": -0.04974767193198204,
      "logps/chosen": -496.3728942871094,
      "logps/rejected": -421.7067565917969,
      "loss": 0.4508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5385488867759705,
      "rewards/margins": 1.4419777393341064,
      "rewards/rejected": -1.9805265665054321,
      "step": 8701
    },
    {
      "epoch": 1.1387094135928681,
      "grad_norm": 7.109489440917969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2764109969139099,
      "logits/rejected": -0.10634993016719818,
      "logps/chosen": -503.8064270019531,
      "logps/rejected": -424.1685791015625,
      "loss": 0.4407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10884971916675568,
      "rewards/margins": 1.1886579990386963,
      "rewards/rejected": -1.2975077629089355,
      "step": 8702
    },
    {
      "epoch": 1.13884027153022,
      "grad_norm": 10.283257484436035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10966426879167557,
      "logits/rejected": -0.1128591001033783,
      "logps/chosen": -337.5332946777344,
      "logps/rejected": -428.03424072265625,
      "loss": 0.615,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4518142640590668,
      "rewards/margins": 0.6126039624214172,
      "rewards/rejected": -1.0644181966781616,
      "step": 8703
    },
    {
      "epoch": 1.1389711294675717,
      "grad_norm": 8.022393226623535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07433110475540161,
      "logits/rejected": -0.097978875041008,
      "logps/chosen": -327.48968505859375,
      "logps/rejected": -367.2406005859375,
      "loss": 0.5772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8475101590156555,
      "rewards/margins": 0.5041196942329407,
      "rewards/rejected": -1.3516298532485962,
      "step": 8704
    },
    {
      "epoch": 1.1391019874049235,
      "grad_norm": 5.844661712646484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05512708052992821,
      "logits/rejected": -0.2789274752140045,
      "logps/chosen": -614.3662109375,
      "logps/rejected": -449.9581298828125,
      "loss": 0.3045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44085538387298584,
      "rewards/margins": 1.4311238527297974,
      "rewards/rejected": -1.8719791173934937,
      "step": 8705
    },
    {
      "epoch": 1.1392328453422753,
      "grad_norm": 5.830442905426025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35265517234802246,
      "logits/rejected": -0.3016905188560486,
      "logps/chosen": -450.4680480957031,
      "logps/rejected": -424.89361572265625,
      "loss": 0.3074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7447105646133423,
      "rewards/margins": 1.446567416191101,
      "rewards/rejected": -2.1912782192230225,
      "step": 8706
    },
    {
      "epoch": 1.139363703279627,
      "grad_norm": 9.177634239196777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4136122167110443,
      "logits/rejected": -0.18590538203716278,
      "logps/chosen": -538.3650512695312,
      "logps/rejected": -597.084716796875,
      "loss": 0.4683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5440338253974915,
      "rewards/margins": 1.0137615203857422,
      "rewards/rejected": -1.5577954053878784,
      "step": 8707
    },
    {
      "epoch": 1.139494561216979,
      "grad_norm": 5.12956428527832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008293862454593182,
      "logits/rejected": 0.06350202858448029,
      "logps/chosen": -437.01507568359375,
      "logps/rejected": -433.86785888671875,
      "loss": 0.2281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3612678647041321,
      "rewards/margins": 2.0560076236724854,
      "rewards/rejected": -2.4172754287719727,
      "step": 8708
    },
    {
      "epoch": 1.1396254191543305,
      "grad_norm": 4.216294288635254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05530855059623718,
      "logits/rejected": -0.048589665442705154,
      "logps/chosen": -612.533203125,
      "logps/rejected": -400.29998779296875,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4830036163330078,
      "rewards/margins": 1.5154603719711304,
      "rewards/rejected": -1.9984638690948486,
      "step": 8709
    },
    {
      "epoch": 1.1397562770916823,
      "grad_norm": 9.330192565917969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003857061266899109,
      "logits/rejected": -0.06887011229991913,
      "logps/chosen": -425.635498046875,
      "logps/rejected": -483.4876403808594,
      "loss": 0.5041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2201848030090332,
      "rewards/margins": 1.146284818649292,
      "rewards/rejected": -2.366469621658325,
      "step": 8710
    },
    {
      "epoch": 1.139887135029034,
      "grad_norm": 6.746591091156006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09068458527326584,
      "logits/rejected": -0.08425478637218475,
      "logps/chosen": -242.98553466796875,
      "logps/rejected": -334.4677429199219,
      "loss": 0.5233,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2163081169128418,
      "rewards/margins": 0.8162261843681335,
      "rewards/rejected": -2.03253436088562,
      "step": 8711
    },
    {
      "epoch": 1.1400179929663858,
      "grad_norm": 12.143699645996094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4801051616668701,
      "logits/rejected": -0.44470736384391785,
      "logps/chosen": -374.989013671875,
      "logps/rejected": -451.4665222167969,
      "loss": 0.5641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5633693933486938,
      "rewards/margins": 1.1933313608169556,
      "rewards/rejected": -1.756700873374939,
      "step": 8712
    },
    {
      "epoch": 1.1401488509037376,
      "grad_norm": 6.0751166343688965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1461600363254547,
      "logits/rejected": -0.05472136661410332,
      "logps/chosen": -404.76080322265625,
      "logps/rejected": -350.3388977050781,
      "loss": 0.3421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9963798522949219,
      "rewards/margins": 1.3316912651062012,
      "rewards/rejected": -2.328071117401123,
      "step": 8713
    },
    {
      "epoch": 1.1402797088410894,
      "grad_norm": 6.774938583374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22206905484199524,
      "logits/rejected": -0.11679322272539139,
      "logps/chosen": -531.9423217773438,
      "logps/rejected": -547.5731201171875,
      "loss": 0.4117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16141818463802338,
      "rewards/margins": 1.1694607734680176,
      "rewards/rejected": -1.3308789730072021,
      "step": 8714
    },
    {
      "epoch": 1.1404105667784412,
      "grad_norm": 14.381139755249023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23276272416114807,
      "logits/rejected": -0.055082883685827255,
      "logps/chosen": -511.01666259765625,
      "logps/rejected": -477.7420349121094,
      "loss": 0.8645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.103386878967285,
      "rewards/margins": 0.4378358721733093,
      "rewards/rejected": -2.5412228107452393,
      "step": 8715
    },
    {
      "epoch": 1.140541424715793,
      "grad_norm": 3.429399013519287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20912973582744598,
      "logits/rejected": -0.4388856589794159,
      "logps/chosen": -339.232666015625,
      "logps/rejected": -484.5251770019531,
      "loss": 0.1851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7610426545143127,
      "rewards/margins": 2.786989212036133,
      "rewards/rejected": -3.548031806945801,
      "step": 8716
    },
    {
      "epoch": 1.1406722826531448,
      "grad_norm": 6.606662273406982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16818007826805115,
      "logits/rejected": -0.23964102566242218,
      "logps/chosen": -510.75262451171875,
      "logps/rejected": -467.4833984375,
      "loss": 0.289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28516826033592224,
      "rewards/margins": 1.5770461559295654,
      "rewards/rejected": -1.2918779850006104,
      "step": 8717
    },
    {
      "epoch": 1.1408031405904964,
      "grad_norm": 11.437535285949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24070410430431366,
      "logits/rejected": 0.10835259407758713,
      "logps/chosen": -577.9484252929688,
      "logps/rejected": -497.3985595703125,
      "loss": 0.5189,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8770146369934082,
      "rewards/margins": 1.206200122833252,
      "rewards/rejected": -2.08321475982666,
      "step": 8718
    },
    {
      "epoch": 1.1409339985278482,
      "grad_norm": 5.30441427230835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17000816762447357,
      "logits/rejected": -0.22319859266281128,
      "logps/chosen": -515.4593505859375,
      "logps/rejected": -461.0048828125,
      "loss": 0.2201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8894638419151306,
      "rewards/margins": 1.8047394752502441,
      "rewards/rejected": -2.6942036151885986,
      "step": 8719
    },
    {
      "epoch": 1.1410648564652,
      "grad_norm": 5.595860481262207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2339656949043274,
      "logits/rejected": -0.03532392531633377,
      "logps/chosen": -433.9105529785156,
      "logps/rejected": -409.66180419921875,
      "loss": 0.2615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4884945154190063,
      "rewards/margins": 1.8683282136917114,
      "rewards/rejected": -3.356822967529297,
      "step": 8720
    },
    {
      "epoch": 1.1411957144025517,
      "grad_norm": 7.827394008636475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1974470466375351,
      "logits/rejected": -0.4265930652618408,
      "logps/chosen": -385.70013427734375,
      "logps/rejected": -333.6824035644531,
      "loss": 0.3551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5588794946670532,
      "rewards/margins": 1.3386783599853516,
      "rewards/rejected": -1.8975578546524048,
      "step": 8721
    },
    {
      "epoch": 1.1413265723399035,
      "grad_norm": 7.6176228523254395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10654976218938828,
      "logits/rejected": -0.11610613763332367,
      "logps/chosen": -405.459228515625,
      "logps/rejected": -387.9219970703125,
      "loss": 0.6622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8993120193481445,
      "rewards/margins": 0.9925645589828491,
      "rewards/rejected": -1.8918765783309937,
      "step": 8722
    },
    {
      "epoch": 1.1414574302772553,
      "grad_norm": 8.129132270812988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29907333850860596,
      "logits/rejected": -0.28146737813949585,
      "logps/chosen": -456.8374938964844,
      "logps/rejected": -585.7593383789062,
      "loss": 0.4565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19631251692771912,
      "rewards/margins": 1.6008665561676025,
      "rewards/rejected": -1.797178864479065,
      "step": 8723
    },
    {
      "epoch": 1.141588288214607,
      "grad_norm": 8.092095375061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09567230939865112,
      "logits/rejected": -0.1671929955482483,
      "logps/chosen": -377.2439270019531,
      "logps/rejected": -372.62115478515625,
      "loss": 0.3471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1752567291259766,
      "rewards/margins": 1.3982625007629395,
      "rewards/rejected": -2.573519229888916,
      "step": 8724
    },
    {
      "epoch": 1.1417191461519587,
      "grad_norm": 6.961462020874023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1605997234582901,
      "logits/rejected": -0.13846462965011597,
      "logps/chosen": -382.535888671875,
      "logps/rejected": -419.68804931640625,
      "loss": 0.5815,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6173444390296936,
      "rewards/margins": 1.3355700969696045,
      "rewards/rejected": -1.9529145956039429,
      "step": 8725
    },
    {
      "epoch": 1.1418500040893105,
      "grad_norm": 7.054712772369385,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18752694129943848,
      "logits/rejected": -0.08557619154453278,
      "logps/chosen": -414.1039733886719,
      "logps/rejected": -468.5570068359375,
      "loss": 0.4498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6442655324935913,
      "rewards/margins": 1.8059858083724976,
      "rewards/rejected": -2.450251579284668,
      "step": 8726
    },
    {
      "epoch": 1.1419808620266623,
      "grad_norm": 7.982419013977051,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1151101142168045,
      "logits/rejected": 0.09177328646183014,
      "logps/chosen": -504.22344970703125,
      "logps/rejected": -385.7319030761719,
      "loss": 0.4918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10431347042322159,
      "rewards/margins": 1.560337781906128,
      "rewards/rejected": -1.6646513938903809,
      "step": 8727
    },
    {
      "epoch": 1.142111719964014,
      "grad_norm": 6.295222282409668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0998358428478241,
      "logits/rejected": -0.05288677662611008,
      "logps/chosen": -393.55194091796875,
      "logps/rejected": -498.02557373046875,
      "loss": 0.431,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.004507635720074177,
      "rewards/margins": 0.7047384977340698,
      "rewards/rejected": -0.7002307772636414,
      "step": 8728
    },
    {
      "epoch": 1.1422425779013659,
      "grad_norm": 6.037031650543213,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06277249753475189,
      "logits/rejected": -0.16416233777999878,
      "logps/chosen": -493.6131286621094,
      "logps/rejected": -368.7768249511719,
      "loss": 0.3592,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1375168561935425,
      "rewards/margins": 1.7215909957885742,
      "rewards/rejected": -2.8591079711914062,
      "step": 8729
    },
    {
      "epoch": 1.1423734358387176,
      "grad_norm": 9.821977615356445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018475396558642387,
      "logits/rejected": 0.05668702349066734,
      "logps/chosen": -400.8021545410156,
      "logps/rejected": -396.6190185546875,
      "loss": 0.7984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4467754662036896,
      "rewards/margins": 0.3051704168319702,
      "rewards/rejected": -0.7519459128379822,
      "step": 8730
    },
    {
      "epoch": 1.1425042937760694,
      "grad_norm": 8.688011169433594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.484346479177475,
      "logits/rejected": -0.5093281269073486,
      "logps/chosen": -348.6980895996094,
      "logps/rejected": -359.8846740722656,
      "loss": 0.5715,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.322061061859131,
      "rewards/margins": 0.5883774161338806,
      "rewards/rejected": -2.9104385375976562,
      "step": 8731
    },
    {
      "epoch": 1.1426351517134212,
      "grad_norm": 9.449542045593262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10552850365638733,
      "logits/rejected": -0.0813090056180954,
      "logps/chosen": -449.71636962890625,
      "logps/rejected": -398.3920593261719,
      "loss": 0.7365,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4121088981628418,
      "rewards/margins": 0.5031381845474243,
      "rewards/rejected": -1.9152470827102661,
      "step": 8732
    },
    {
      "epoch": 1.1427660096507728,
      "grad_norm": 5.296350002288818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18893024325370789,
      "logits/rejected": -0.5036519765853882,
      "logps/chosen": -428.9707336425781,
      "logps/rejected": -466.1829528808594,
      "loss": 0.3115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2833305597305298,
      "rewards/margins": 1.3667333126068115,
      "rewards/rejected": -1.6500636339187622,
      "step": 8733
    },
    {
      "epoch": 1.1428968675881246,
      "grad_norm": 8.196837425231934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11145221441984177,
      "logits/rejected": 0.09686104208230972,
      "logps/chosen": -501.50396728515625,
      "logps/rejected": -556.1104125976562,
      "loss": 0.4781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33314818143844604,
      "rewards/margins": 1.586575984954834,
      "rewards/rejected": -1.9197241067886353,
      "step": 8734
    },
    {
      "epoch": 1.1430277255254764,
      "grad_norm": 15.441695213317871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03418843448162079,
      "logits/rejected": 0.06610265374183655,
      "logps/chosen": -448.6382141113281,
      "logps/rejected": -388.95989990234375,
      "loss": 0.843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9503051042556763,
      "rewards/margins": 0.26803338527679443,
      "rewards/rejected": -2.21833872795105,
      "step": 8735
    },
    {
      "epoch": 1.1431585834628282,
      "grad_norm": 5.390294075012207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02294064499437809,
      "logits/rejected": -0.09452522546052933,
      "logps/chosen": -521.0350341796875,
      "logps/rejected": -496.623046875,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13081665337085724,
      "rewards/margins": 1.7916433811187744,
      "rewards/rejected": -1.6608266830444336,
      "step": 8736
    },
    {
      "epoch": 1.14328944140018,
      "grad_norm": 9.221659660339355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36297154426574707,
      "logits/rejected": -0.47241562604904175,
      "logps/chosen": -403.2636413574219,
      "logps/rejected": -390.8106689453125,
      "loss": 0.5888,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2441599369049072,
      "rewards/margins": 1.2578084468841553,
      "rewards/rejected": -2.5019683837890625,
      "step": 8737
    },
    {
      "epoch": 1.1434202993375318,
      "grad_norm": 10.392059326171875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11042141914367676,
      "logits/rejected": 0.1200917586684227,
      "logps/chosen": -461.3614501953125,
      "logps/rejected": -498.2243957519531,
      "loss": 0.6466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.679735004901886,
      "rewards/margins": 0.556218147277832,
      "rewards/rejected": -1.2359530925750732,
      "step": 8738
    },
    {
      "epoch": 1.1435511572748835,
      "grad_norm": 13.22103214263916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38067054748535156,
      "logits/rejected": -0.6680276393890381,
      "logps/chosen": -461.8242492675781,
      "logps/rejected": -415.17572021484375,
      "loss": 0.8857,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0985023975372314,
      "rewards/margins": 0.4404045045375824,
      "rewards/rejected": -2.538907051086426,
      "step": 8739
    },
    {
      "epoch": 1.1436820152122351,
      "grad_norm": 10.043418884277344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28669610619544983,
      "logits/rejected": -0.40470123291015625,
      "logps/chosen": -616.02587890625,
      "logps/rejected": -591.0830078125,
      "loss": 0.5819,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1474304348230362,
      "rewards/margins": 1.2468245029449463,
      "rewards/rejected": -1.3942551612854004,
      "step": 8740
    },
    {
      "epoch": 1.143812873149587,
      "grad_norm": 9.765351295471191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06468696147203445,
      "logits/rejected": -0.09372620284557343,
      "logps/chosen": -510.0882568359375,
      "logps/rejected": -432.4651794433594,
      "loss": 0.651,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4128166437149048,
      "rewards/margins": 0.21249981224536896,
      "rewards/rejected": -1.6253163814544678,
      "step": 8741
    },
    {
      "epoch": 1.1439437310869387,
      "grad_norm": 5.653021335601807,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02886660397052765,
      "logits/rejected": -0.10707642138004303,
      "logps/chosen": -439.90875244140625,
      "logps/rejected": -387.2939453125,
      "loss": 0.3319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18768447637557983,
      "rewards/margins": 1.2867199182510376,
      "rewards/rejected": -1.4744043350219727,
      "step": 8742
    },
    {
      "epoch": 1.1440745890242905,
      "grad_norm": 8.513097763061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19596974551677704,
      "logits/rejected": -0.20504020154476166,
      "logps/chosen": -516.0955810546875,
      "logps/rejected": -482.484375,
      "loss": 0.5854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4096551835536957,
      "rewards/margins": 1.005206823348999,
      "rewards/rejected": -1.4148619174957275,
      "step": 8743
    },
    {
      "epoch": 1.1442054469616423,
      "grad_norm": 9.295788764953613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.336396187543869,
      "logits/rejected": -0.3064417243003845,
      "logps/chosen": -548.7373657226562,
      "logps/rejected": -620.9730224609375,
      "loss": 0.5138,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5234531760215759,
      "rewards/margins": 1.0522098541259766,
      "rewards/rejected": -1.5756630897521973,
      "step": 8744
    },
    {
      "epoch": 1.144336304898994,
      "grad_norm": 6.294064521789551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32305392622947693,
      "logits/rejected": -0.24760641157627106,
      "logps/chosen": -231.31719970703125,
      "logps/rejected": -296.715576171875,
      "loss": 0.4121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.762427568435669,
      "rewards/margins": 0.9998777508735657,
      "rewards/rejected": -1.7623053789138794,
      "step": 8745
    },
    {
      "epoch": 1.1444671628363459,
      "grad_norm": 9.996320724487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1295994222164154,
      "logits/rejected": -0.1564178466796875,
      "logps/chosen": -437.7494201660156,
      "logps/rejected": -459.4151306152344,
      "loss": 0.5871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0744328498840332,
      "rewards/margins": 1.2202199697494507,
      "rewards/rejected": -2.2946527004241943,
      "step": 8746
    },
    {
      "epoch": 1.1445980207736977,
      "grad_norm": 6.409438133239746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04819001257419586,
      "logits/rejected": -0.027109414339065552,
      "logps/chosen": -472.5918273925781,
      "logps/rejected": -379.2054748535156,
      "loss": 0.4139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19602453708648682,
      "rewards/margins": 1.3584883213043213,
      "rewards/rejected": -1.1624637842178345,
      "step": 8747
    },
    {
      "epoch": 1.1447288787110492,
      "grad_norm": 8.800119400024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21942201256752014,
      "logits/rejected": -0.33393001556396484,
      "logps/chosen": -391.97039794921875,
      "logps/rejected": -402.35479736328125,
      "loss": 0.6698,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0447978973388672,
      "rewards/margins": 0.2191295176744461,
      "rewards/rejected": -1.2639275789260864,
      "step": 8748
    },
    {
      "epoch": 1.144859736648401,
      "grad_norm": 7.496055603027344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06838466972112656,
      "logits/rejected": -0.049502454698085785,
      "logps/chosen": -524.659912109375,
      "logps/rejected": -392.2703857421875,
      "loss": 0.5054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04167540371417999,
      "rewards/margins": 1.1593055725097656,
      "rewards/rejected": -1.1176302433013916,
      "step": 8749
    },
    {
      "epoch": 1.1449905945857528,
      "grad_norm": 11.564607620239258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07365237921476364,
      "logits/rejected": -0.06699119508266449,
      "logps/chosen": -642.3848876953125,
      "logps/rejected": -536.562744140625,
      "loss": 0.6559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0225250720977783,
      "rewards/margins": 0.4208179712295532,
      "rewards/rejected": -1.443342924118042,
      "step": 8750
    },
    {
      "epoch": 1.1451214525231046,
      "grad_norm": 6.257206916809082,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03947843611240387,
      "logits/rejected": -0.03269464150071144,
      "logps/chosen": -416.52703857421875,
      "logps/rejected": -367.28948974609375,
      "loss": 0.3565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0440561771392822,
      "rewards/margins": 1.5282572507858276,
      "rewards/rejected": -2.5723133087158203,
      "step": 8751
    },
    {
      "epoch": 1.1452523104604564,
      "grad_norm": 7.012812614440918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2368685007095337,
      "logits/rejected": -0.17551115155220032,
      "logps/chosen": -346.5107421875,
      "logps/rejected": -510.5443420410156,
      "loss": 0.3738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7136284708976746,
      "rewards/margins": 1.029289960861206,
      "rewards/rejected": -1.7429183721542358,
      "step": 8752
    },
    {
      "epoch": 1.1453831683978082,
      "grad_norm": 11.517934799194336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45530062913894653,
      "logits/rejected": -0.44531258940696716,
      "logps/chosen": -601.3355712890625,
      "logps/rejected": -578.6065673828125,
      "loss": 0.6244,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.589921712875366,
      "rewards/margins": 0.6713889837265015,
      "rewards/rejected": -3.261310577392578,
      "step": 8753
    },
    {
      "epoch": 1.14551402633516,
      "grad_norm": 7.497889041900635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14298051595687866,
      "logits/rejected": -0.18315991759300232,
      "logps/chosen": -436.44610595703125,
      "logps/rejected": -487.20257568359375,
      "loss": 0.4533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8264400362968445,
      "rewards/margins": 0.7683346271514893,
      "rewards/rejected": -1.594774603843689,
      "step": 8754
    },
    {
      "epoch": 1.1456448842725115,
      "grad_norm": 7.502689361572266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1356782466173172,
      "logits/rejected": -0.2463718205690384,
      "logps/chosen": -415.9270935058594,
      "logps/rejected": -406.244384765625,
      "loss": 0.4261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15246355533599854,
      "rewards/margins": 1.6702003479003906,
      "rewards/rejected": -1.5177369117736816,
      "step": 8755
    },
    {
      "epoch": 1.1457757422098633,
      "grad_norm": 7.136241912841797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06601080298423767,
      "logits/rejected": 0.07566182315349579,
      "logps/chosen": -463.063232421875,
      "logps/rejected": -442.63226318359375,
      "loss": 0.3624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3841090798377991,
      "rewards/margins": 0.9975210428237915,
      "rewards/rejected": -1.3816300630569458,
      "step": 8756
    },
    {
      "epoch": 1.1459066001472151,
      "grad_norm": 9.041478157043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2713308334350586,
      "logits/rejected": -0.1952235996723175,
      "logps/chosen": -414.8918151855469,
      "logps/rejected": -416.7779846191406,
      "loss": 0.5681,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1675504744052887,
      "rewards/margins": 0.5320345163345337,
      "rewards/rejected": -0.6995849609375,
      "step": 8757
    },
    {
      "epoch": 1.146037458084567,
      "grad_norm": 7.923239707946777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023666419088840485,
      "logits/rejected": -0.03129280358552933,
      "logps/chosen": -421.6405334472656,
      "logps/rejected": -418.7015686035156,
      "loss": 0.3615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5669968128204346,
      "rewards/margins": 1.7255967855453491,
      "rewards/rejected": -2.292593479156494,
      "step": 8758
    },
    {
      "epoch": 1.1461683160219187,
      "grad_norm": 6.411128997802734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1249702125787735,
      "logits/rejected": -0.20583663880825043,
      "logps/chosen": -433.3815002441406,
      "logps/rejected": -436.57952880859375,
      "loss": 0.3523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5260112881660461,
      "rewards/margins": 1.3685723543167114,
      "rewards/rejected": -1.8945837020874023,
      "step": 8759
    },
    {
      "epoch": 1.1462991739592705,
      "grad_norm": 2.8373289108276367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18408693373203278,
      "logits/rejected": -0.5055963397026062,
      "logps/chosen": -427.3449401855469,
      "logps/rejected": -749.8447265625,
      "loss": 0.0771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11689683794975281,
      "rewards/margins": 3.416616439819336,
      "rewards/rejected": -3.2997195720672607,
      "step": 8760
    },
    {
      "epoch": 1.1464300318966223,
      "grad_norm": 9.247051239013672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0441681444644928,
      "logits/rejected": -0.279255747795105,
      "logps/chosen": -289.0843505859375,
      "logps/rejected": -425.2315368652344,
      "loss": 0.5437,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.991257905960083,
      "rewards/margins": 1.1318399906158447,
      "rewards/rejected": -2.1230978965759277,
      "step": 8761
    },
    {
      "epoch": 1.146560889833974,
      "grad_norm": 10.91959285736084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.219456747174263,
      "logits/rejected": -0.09366640448570251,
      "logps/chosen": -378.6987609863281,
      "logps/rejected": -343.32818603515625,
      "loss": 0.5815,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3348981738090515,
      "rewards/margins": 0.5760017037391663,
      "rewards/rejected": -0.9109000563621521,
      "step": 8762
    },
    {
      "epoch": 1.1466917477713259,
      "grad_norm": 7.065214157104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15452109277248383,
      "logits/rejected": -0.28158432245254517,
      "logps/chosen": -466.567138671875,
      "logps/rejected": -311.958740234375,
      "loss": 0.4614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5476301312446594,
      "rewards/margins": 1.3803215026855469,
      "rewards/rejected": -1.9279518127441406,
      "step": 8763
    },
    {
      "epoch": 1.1468226057086774,
      "grad_norm": 10.253707885742188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09152142703533173,
      "logits/rejected": -0.12644971907138824,
      "logps/chosen": -323.28509521484375,
      "logps/rejected": -367.8122253417969,
      "loss": 0.5249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.586861789226532,
      "rewards/margins": 1.1709009408950806,
      "rewards/rejected": -1.7577629089355469,
      "step": 8764
    },
    {
      "epoch": 1.1469534636460292,
      "grad_norm": 8.955211639404297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07444237172603607,
      "logits/rejected": 0.15068091452121735,
      "logps/chosen": -322.7099609375,
      "logps/rejected": -342.15802001953125,
      "loss": 0.6686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7711134552955627,
      "rewards/margins": 0.2936248183250427,
      "rewards/rejected": -1.064738154411316,
      "step": 8765
    },
    {
      "epoch": 1.147084321583381,
      "grad_norm": 8.418014526367188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22509247064590454,
      "logits/rejected": -0.3062100112438202,
      "logps/chosen": -370.822998046875,
      "logps/rejected": -441.7229309082031,
      "loss": 0.4803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7731279134750366,
      "rewards/margins": 1.4525237083435059,
      "rewards/rejected": -2.225651502609253,
      "step": 8766
    },
    {
      "epoch": 1.1472151795207328,
      "grad_norm": 6.897037982940674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0163480993360281,
      "logits/rejected": -0.17639873921871185,
      "logps/chosen": -525.71923828125,
      "logps/rejected": -399.8460693359375,
      "loss": 0.4704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4031814634799957,
      "rewards/margins": 0.7839962244033813,
      "rewards/rejected": -1.1871776580810547,
      "step": 8767
    },
    {
      "epoch": 1.1473460374580846,
      "grad_norm": 9.203383445739746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18348269164562225,
      "logits/rejected": -0.017640741541981697,
      "logps/chosen": -400.19122314453125,
      "logps/rejected": -504.5924072265625,
      "loss": 0.661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5150184631347656,
      "rewards/margins": 0.7961292862892151,
      "rewards/rejected": -1.311147689819336,
      "step": 8768
    },
    {
      "epoch": 1.1474768953954364,
      "grad_norm": 6.475658893585205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.39107659459114075,
      "logits/rejected": 0.17389662563800812,
      "logps/chosen": -376.20379638671875,
      "logps/rejected": -374.68145751953125,
      "loss": 0.2997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4956423342227936,
      "rewards/margins": 2.507375717163086,
      "rewards/rejected": -2.011733293533325,
      "step": 8769
    },
    {
      "epoch": 1.147607753332788,
      "grad_norm": 5.9706292152404785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2003212720155716,
      "logits/rejected": -0.2528788447380066,
      "logps/chosen": -341.45257568359375,
      "logps/rejected": -303.52227783203125,
      "loss": 0.3241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6482282876968384,
      "rewards/margins": 1.6248786449432373,
      "rewards/rejected": -2.2731070518493652,
      "step": 8770
    },
    {
      "epoch": 1.1477386112701398,
      "grad_norm": 6.926327228546143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11703570932149887,
      "logits/rejected": -0.021211162209510803,
      "logps/chosen": -529.3096313476562,
      "logps/rejected": -597.5726928710938,
      "loss": 0.3284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3066783845424652,
      "rewards/margins": 1.9187259674072266,
      "rewards/rejected": -2.2254042625427246,
      "step": 8771
    },
    {
      "epoch": 1.1478694692074916,
      "grad_norm": 4.752737045288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18962359428405762,
      "logits/rejected": -0.41547781229019165,
      "logps/chosen": -446.8457336425781,
      "logps/rejected": -326.4195251464844,
      "loss": 0.2761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6578583121299744,
      "rewards/margins": 1.3255691528320312,
      "rewards/rejected": -1.9834274053573608,
      "step": 8772
    },
    {
      "epoch": 1.1480003271448433,
      "grad_norm": 5.409414291381836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24071604013442993,
      "logits/rejected": -0.1541898548603058,
      "logps/chosen": -311.57696533203125,
      "logps/rejected": -348.06414794921875,
      "loss": 0.4832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04136025905609131,
      "rewards/margins": 0.673589289188385,
      "rewards/rejected": -0.7149496078491211,
      "step": 8773
    },
    {
      "epoch": 1.1481311850821951,
      "grad_norm": 6.259084701538086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30459487438201904,
      "logits/rejected": -0.25069737434387207,
      "logps/chosen": -310.4764709472656,
      "logps/rejected": -264.6796569824219,
      "loss": 0.39,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3226091861724854,
      "rewards/margins": 1.33259117603302,
      "rewards/rejected": -2.655200481414795,
      "step": 8774
    },
    {
      "epoch": 1.148262043019547,
      "grad_norm": 4.960290908813477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06024099513888359,
      "logits/rejected": -0.15034912526607513,
      "logps/chosen": -455.4960021972656,
      "logps/rejected": -342.0577392578125,
      "loss": 0.2707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10115490853786469,
      "rewards/margins": 1.3989711999893188,
      "rewards/rejected": -1.5001261234283447,
      "step": 8775
    },
    {
      "epoch": 1.1483929009568987,
      "grad_norm": 3.2470932006835938,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03220561146736145,
      "logits/rejected": -0.02993294596672058,
      "logps/chosen": -362.63360595703125,
      "logps/rejected": -314.3170471191406,
      "loss": 0.2482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.031053349375724792,
      "rewards/margins": 1.8601350784301758,
      "rewards/rejected": -1.8290817737579346,
      "step": 8776
    },
    {
      "epoch": 1.1485237588942505,
      "grad_norm": 16.38642692565918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2502876818180084,
      "logits/rejected": -0.5266833305358887,
      "logps/chosen": -567.883544921875,
      "logps/rejected": -427.00579833984375,
      "loss": 0.7262,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9085603952407837,
      "rewards/margins": 0.6345821619033813,
      "rewards/rejected": -1.543142557144165,
      "step": 8777
    },
    {
      "epoch": 1.1486546168316023,
      "grad_norm": 7.036271095275879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08089567720890045,
      "logits/rejected": 0.10503207892179489,
      "logps/chosen": -467.483642578125,
      "logps/rejected": -425.5047302246094,
      "loss": 0.2972,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5577366352081299,
      "rewards/margins": 1.2820730209350586,
      "rewards/rejected": -1.8398096561431885,
      "step": 8778
    },
    {
      "epoch": 1.1487854747689539,
      "grad_norm": 7.342581748962402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014566898345947266,
      "logits/rejected": -0.034251946955919266,
      "logps/chosen": -715.2811279296875,
      "logps/rejected": -555.2786865234375,
      "loss": 0.3749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49879488348960876,
      "rewards/margins": 1.262355923652649,
      "rewards/rejected": -1.7611509561538696,
      "step": 8779
    },
    {
      "epoch": 1.1489163327063057,
      "grad_norm": 5.782257080078125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14556974172592163,
      "logits/rejected": -0.2367672324180603,
      "logps/chosen": -563.4437255859375,
      "logps/rejected": -570.50439453125,
      "loss": 0.2177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1380692720413208,
      "rewards/margins": 2.278390407562256,
      "rewards/rejected": -1.1403210163116455,
      "step": 8780
    },
    {
      "epoch": 1.1490471906436575,
      "grad_norm": 4.049276351928711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2560526728630066,
      "logits/rejected": -0.22505608201026917,
      "logps/chosen": -406.8028869628906,
      "logps/rejected": -581.8812866210938,
      "loss": 0.1991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23513832688331604,
      "rewards/margins": 2.364922523498535,
      "rewards/rejected": -2.60006046295166,
      "step": 8781
    },
    {
      "epoch": 1.1491780485810092,
      "grad_norm": 6.101728916168213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36244717240333557,
      "logits/rejected": -0.4531382620334625,
      "logps/chosen": -452.5269470214844,
      "logps/rejected": -479.0771484375,
      "loss": 0.3213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.220748171210289,
      "rewards/margins": 1.9293324947357178,
      "rewards/rejected": -2.150080680847168,
      "step": 8782
    },
    {
      "epoch": 1.149308906518361,
      "grad_norm": 10.884289741516113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13759902119636536,
      "logits/rejected": 0.033493854105472565,
      "logps/chosen": -534.7597045898438,
      "logps/rejected": -466.79913330078125,
      "loss": 0.842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8942590951919556,
      "rewards/margins": 0.35064831376075745,
      "rewards/rejected": -1.2449074983596802,
      "step": 8783
    },
    {
      "epoch": 1.1494397644557128,
      "grad_norm": 12.585332870483398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06139252707362175,
      "logits/rejected": -0.06861884146928787,
      "logps/chosen": -488.079345703125,
      "logps/rejected": -379.70721435546875,
      "loss": 0.8221,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1547119617462158,
      "rewards/margins": 0.4499373137950897,
      "rewards/rejected": -1.604649305343628,
      "step": 8784
    },
    {
      "epoch": 1.1495706223930646,
      "grad_norm": 7.805783748626709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24178531765937805,
      "logits/rejected": -0.45035338401794434,
      "logps/chosen": -503.2389221191406,
      "logps/rejected": -510.9239196777344,
      "loss": 0.5751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7025236487388611,
      "rewards/margins": 1.2542011737823486,
      "rewards/rejected": -1.9567248821258545,
      "step": 8785
    },
    {
      "epoch": 1.1497014803304162,
      "grad_norm": 1.1876037120819092,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04561448469758034,
      "logits/rejected": 0.09453041106462479,
      "logps/chosen": -358.963623046875,
      "logps/rejected": -466.587158203125,
      "loss": 0.0338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2556847929954529,
      "rewards/margins": 3.7820589542388916,
      "rewards/rejected": -3.526374340057373,
      "step": 8786
    },
    {
      "epoch": 1.149832338267768,
      "grad_norm": 9.170434951782227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14091894030570984,
      "logits/rejected": 0.005656305700540543,
      "logps/chosen": -340.9361572265625,
      "logps/rejected": -334.7716369628906,
      "loss": 0.4439,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44928497076034546,
      "rewards/margins": 1.5192410945892334,
      "rewards/rejected": -1.9685261249542236,
      "step": 8787
    },
    {
      "epoch": 1.1499631962051198,
      "grad_norm": 6.712588787078857,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023911409080028534,
      "logits/rejected": -0.07287989556789398,
      "logps/chosen": -331.2463073730469,
      "logps/rejected": -565.7535400390625,
      "loss": 0.3859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06529675424098969,
      "rewards/margins": 1.2971599102020264,
      "rewards/rejected": -1.3624565601348877,
      "step": 8788
    },
    {
      "epoch": 1.1500940541424716,
      "grad_norm": 9.036798477172852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22441360354423523,
      "logits/rejected": -0.24754619598388672,
      "logps/chosen": -627.8590087890625,
      "logps/rejected": -552.6515502929688,
      "loss": 0.5044,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40264466404914856,
      "rewards/margins": 1.0643959045410156,
      "rewards/rejected": -1.4670406579971313,
      "step": 8789
    },
    {
      "epoch": 1.1502249120798234,
      "grad_norm": 9.925703048706055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14002379775047302,
      "logits/rejected": -0.04034150391817093,
      "logps/chosen": -388.9964599609375,
      "logps/rejected": -440.7474365234375,
      "loss": 0.6814,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.579950749874115,
      "rewards/margins": 0.9377065300941467,
      "rewards/rejected": -1.5176573991775513,
      "step": 8790
    },
    {
      "epoch": 1.1503557700171751,
      "grad_norm": 12.291423797607422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09155018627643585,
      "logits/rejected": -0.18747365474700928,
      "logps/chosen": -565.7667236328125,
      "logps/rejected": -517.6694946289062,
      "loss": 0.7658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5013227462768555,
      "rewards/margins": 0.3239370584487915,
      "rewards/rejected": -0.825259804725647,
      "step": 8791
    },
    {
      "epoch": 1.150486627954527,
      "grad_norm": 6.931795120239258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006785117089748383,
      "logits/rejected": -0.07040250301361084,
      "logps/chosen": -462.5950927734375,
      "logps/rejected": -399.1768798828125,
      "loss": 0.5104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35203367471694946,
      "rewards/margins": 0.9434841275215149,
      "rewards/rejected": -1.2955178022384644,
      "step": 8792
    },
    {
      "epoch": 1.1506174858918787,
      "grad_norm": 3.513993740081787,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1294477880001068,
      "logits/rejected": 0.22712822258472443,
      "logps/chosen": -511.41937255859375,
      "logps/rejected": -535.423828125,
      "loss": 0.2037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03726083040237427,
      "rewards/margins": 2.384312629699707,
      "rewards/rejected": -2.3470516204833984,
      "step": 8793
    },
    {
      "epoch": 1.1507483438292303,
      "grad_norm": 5.368504047393799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04756253585219383,
      "logits/rejected": -0.08063724637031555,
      "logps/chosen": -519.5382690429688,
      "logps/rejected": -472.4073486328125,
      "loss": 0.3277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2705867290496826,
      "rewards/margins": 1.2950783967971802,
      "rewards/rejected": -1.5656652450561523,
      "step": 8794
    },
    {
      "epoch": 1.150879201766582,
      "grad_norm": 9.776787757873535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13549597561359406,
      "logits/rejected": 0.03821087256073952,
      "logps/chosen": -449.2154541015625,
      "logps/rejected": -474.6464538574219,
      "loss": 0.6703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2533075213432312,
      "rewards/margins": 0.5727967023849487,
      "rewards/rejected": -0.8261041641235352,
      "step": 8795
    },
    {
      "epoch": 1.1510100597039339,
      "grad_norm": 9.578509330749512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19924256205558777,
      "logits/rejected": -0.1600050926208496,
      "logps/chosen": -404.66278076171875,
      "logps/rejected": -461.91693115234375,
      "loss": 0.5935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.777542233467102,
      "rewards/margins": 0.7851626873016357,
      "rewards/rejected": -2.5627048015594482,
      "step": 8796
    },
    {
      "epoch": 1.1511409176412857,
      "grad_norm": 5.862721920013428,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29112112522125244,
      "logits/rejected": -0.4737028479576111,
      "logps/chosen": -438.3749694824219,
      "logps/rejected": -362.4635009765625,
      "loss": 0.4182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44663578271865845,
      "rewards/margins": 1.3640658855438232,
      "rewards/rejected": -1.8107014894485474,
      "step": 8797
    },
    {
      "epoch": 1.1512717755786375,
      "grad_norm": 8.820213317871094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07507951557636261,
      "logits/rejected": -0.0221659354865551,
      "logps/chosen": -379.1432800292969,
      "logps/rejected": -409.0082092285156,
      "loss": 0.6374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03693637624382973,
      "rewards/margins": 0.7292307615280151,
      "rewards/rejected": -0.7661671042442322,
      "step": 8798
    },
    {
      "epoch": 1.1514026335159893,
      "grad_norm": 5.924068450927734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20634165406227112,
      "logits/rejected": -0.4796532392501831,
      "logps/chosen": -526.7194213867188,
      "logps/rejected": -445.65203857421875,
      "loss": 0.2818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7964143753051758,
      "rewards/margins": 1.52388334274292,
      "rewards/rejected": -2.3202977180480957,
      "step": 8799
    },
    {
      "epoch": 1.151533491453341,
      "grad_norm": 4.429324626922607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018894361332058907,
      "logits/rejected": -0.09349827468395233,
      "logps/chosen": -318.8719482421875,
      "logps/rejected": -325.284423828125,
      "loss": 0.2996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47755375504493713,
      "rewards/margins": 1.8209632635116577,
      "rewards/rejected": -2.2985172271728516,
      "step": 8800
    },
    {
      "epoch": 1.1516643493906926,
      "grad_norm": 7.577212333679199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15501809120178223,
      "logits/rejected": 0.02177346684038639,
      "logps/chosen": -483.1835021972656,
      "logps/rejected": -440.1982421875,
      "loss": 0.4434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03455886244773865,
      "rewards/margins": 1.206822156906128,
      "rewards/rejected": -1.241381049156189,
      "step": 8801
    },
    {
      "epoch": 1.1517952073280444,
      "grad_norm": 7.023688316345215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3075522482395172,
      "logits/rejected": -0.0030130669474601746,
      "logps/chosen": -571.396728515625,
      "logps/rejected": -712.9227905273438,
      "loss": 0.4403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.614641010761261,
      "rewards/margins": 1.4025990962982178,
      "rewards/rejected": -2.017239809036255,
      "step": 8802
    },
    {
      "epoch": 1.1519260652653962,
      "grad_norm": 9.796449661254883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04415452852845192,
      "logits/rejected": 0.05558805540204048,
      "logps/chosen": -489.7772521972656,
      "logps/rejected": -565.4107666015625,
      "loss": 0.5786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11054231226444244,
      "rewards/margins": 1.0347344875335693,
      "rewards/rejected": -1.145276665687561,
      "step": 8803
    },
    {
      "epoch": 1.152056923202748,
      "grad_norm": 9.702040672302246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07871237397193909,
      "logits/rejected": -0.2188362181186676,
      "logps/chosen": -326.35272216796875,
      "logps/rejected": -439.1681823730469,
      "loss": 0.4905,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8818004727363586,
      "rewards/margins": 2.273707866668701,
      "rewards/rejected": -3.155508279800415,
      "step": 8804
    },
    {
      "epoch": 1.1521877811400998,
      "grad_norm": 6.891544342041016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13799093663692474,
      "logits/rejected": -0.24880486726760864,
      "logps/chosen": -560.1682739257812,
      "logps/rejected": -459.88824462890625,
      "loss": 0.462,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0873738527297974,
      "rewards/margins": 0.8026857376098633,
      "rewards/rejected": -1.8900595903396606,
      "step": 8805
    },
    {
      "epoch": 1.1523186390774516,
      "grad_norm": 5.172935962677002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0579071007668972,
      "logits/rejected": -0.15627270936965942,
      "logps/chosen": -469.99542236328125,
      "logps/rejected": -373.2202453613281,
      "loss": 0.1738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2558332085609436,
      "rewards/margins": 2.466731548309326,
      "rewards/rejected": -2.722564697265625,
      "step": 8806
    },
    {
      "epoch": 1.1524494970148034,
      "grad_norm": 10.08514404296875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13059094548225403,
      "logits/rejected": -0.008054979145526886,
      "logps/chosen": -402.89892578125,
      "logps/rejected": -384.2033386230469,
      "loss": 0.7112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2357620000839233,
      "rewards/margins": 0.4806860089302063,
      "rewards/rejected": -1.7164480686187744,
      "step": 8807
    },
    {
      "epoch": 1.1525803549521552,
      "grad_norm": 10.026700973510742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13072527945041656,
      "logits/rejected": -0.006666094064712524,
      "logps/chosen": -546.6588134765625,
      "logps/rejected": -442.56781005859375,
      "loss": 0.4369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.755193293094635,
      "rewards/margins": 2.510535717010498,
      "rewards/rejected": -3.265728712081909,
      "step": 8808
    },
    {
      "epoch": 1.152711212889507,
      "grad_norm": 9.70091724395752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019414303824305534,
      "logits/rejected": -0.03811958059668541,
      "logps/chosen": -358.9721374511719,
      "logps/rejected": -442.4514465332031,
      "loss": 0.6792,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5965622663497925,
      "rewards/margins": 0.5912872552871704,
      "rewards/rejected": -2.187849521636963,
      "step": 8809
    },
    {
      "epoch": 1.1528420708268585,
      "grad_norm": 10.530815124511719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20377452671527863,
      "logits/rejected": -0.36793574690818787,
      "logps/chosen": -437.3531494140625,
      "logps/rejected": -416.4838562011719,
      "loss": 0.6173,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5592626929283142,
      "rewards/margins": 0.48054423928260803,
      "rewards/rejected": -1.0398070812225342,
      "step": 8810
    },
    {
      "epoch": 1.1529729287642103,
      "grad_norm": 10.655791282653809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04253479093313217,
      "logits/rejected": -0.18389195203781128,
      "logps/chosen": -401.11346435546875,
      "logps/rejected": -309.91522216796875,
      "loss": 0.6971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8867889046669006,
      "rewards/margins": 0.9304273724555969,
      "rewards/rejected": -1.817216396331787,
      "step": 8811
    },
    {
      "epoch": 1.153103786701562,
      "grad_norm": 7.415294647216797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03320056200027466,
      "logits/rejected": -0.024937652051448822,
      "logps/chosen": -570.3574829101562,
      "logps/rejected": -533.38232421875,
      "loss": 0.3445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.003196120262146,
      "rewards/margins": 1.4987471103668213,
      "rewards/rejected": -2.501943349838257,
      "step": 8812
    },
    {
      "epoch": 1.153234644638914,
      "grad_norm": 7.8917646408081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027906669303774834,
      "logits/rejected": -0.07609549164772034,
      "logps/chosen": -513.2940063476562,
      "logps/rejected": -514.5797729492188,
      "loss": 0.3952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5363701581954956,
      "rewards/margins": 1.1464622020721436,
      "rewards/rejected": -1.6828324794769287,
      "step": 8813
    },
    {
      "epoch": 1.1533655025762657,
      "grad_norm": 5.568123817443848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034975677728652954,
      "logits/rejected": -0.024121224880218506,
      "logps/chosen": -459.72198486328125,
      "logps/rejected": -506.98443603515625,
      "loss": 0.2681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9725982546806335,
      "rewards/margins": 1.84561026096344,
      "rewards/rejected": -2.818208694458008,
      "step": 8814
    },
    {
      "epoch": 1.1534963605136175,
      "grad_norm": 4.5452070236206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20891933143138885,
      "logits/rejected": 0.043577536940574646,
      "logps/chosen": -559.6906127929688,
      "logps/rejected": -597.5479125976562,
      "loss": 0.2239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5812891721725464,
      "rewards/margins": 1.9775593280792236,
      "rewards/rejected": -2.5588481426239014,
      "step": 8815
    },
    {
      "epoch": 1.153627218450969,
      "grad_norm": 10.340428352355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23112472891807556,
      "logits/rejected": -0.37630611658096313,
      "logps/chosen": -378.7499694824219,
      "logps/rejected": -301.287841796875,
      "loss": 0.8085,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7351311445236206,
      "rewards/margins": 0.11299346387386322,
      "rewards/rejected": -1.8481245040893555,
      "step": 8816
    },
    {
      "epoch": 1.1537580763883208,
      "grad_norm": 9.722249984741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06694696098566055,
      "logits/rejected": 0.09917408227920532,
      "logps/chosen": -402.94989013671875,
      "logps/rejected": -352.4549560546875,
      "loss": 0.6432,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8020235300064087,
      "rewards/margins": 0.6867504119873047,
      "rewards/rejected": -1.4887739419937134,
      "step": 8817
    },
    {
      "epoch": 1.1538889343256726,
      "grad_norm": 6.494513511657715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.090643510222435,
      "logits/rejected": 0.07100354880094528,
      "logps/chosen": -426.1058349609375,
      "logps/rejected": -585.646728515625,
      "loss": 0.318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4219360053539276,
      "rewards/margins": 1.813617467880249,
      "rewards/rejected": -2.235553741455078,
      "step": 8818
    },
    {
      "epoch": 1.1540197922630244,
      "grad_norm": 6.350162029266357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10348796099424362,
      "logits/rejected": -0.12453025579452515,
      "logps/chosen": -380.23687744140625,
      "logps/rejected": -467.4354248046875,
      "loss": 0.475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7897424697875977,
      "rewards/margins": 1.0103626251220703,
      "rewards/rejected": -1.800105094909668,
      "step": 8819
    },
    {
      "epoch": 1.1541506502003762,
      "grad_norm": 13.394668579101562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09530211240053177,
      "logits/rejected": 0.13953222334384918,
      "logps/chosen": -431.75286865234375,
      "logps/rejected": -431.28070068359375,
      "loss": 0.7831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2265416383743286,
      "rewards/margins": 0.6577594876289368,
      "rewards/rejected": -1.8843013048171997,
      "step": 8820
    },
    {
      "epoch": 1.154281508137728,
      "grad_norm": 6.314382076263428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06219308078289032,
      "logits/rejected": 0.08145525306463242,
      "logps/chosen": -423.1700744628906,
      "logps/rejected": -450.61431884765625,
      "loss": 0.3587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6303101778030396,
      "rewards/margins": 1.8197015523910522,
      "rewards/rejected": -2.450011730194092,
      "step": 8821
    },
    {
      "epoch": 1.1544123660750798,
      "grad_norm": 5.146848678588867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10467016696929932,
      "logits/rejected": 0.2722916901111603,
      "logps/chosen": -483.6352233886719,
      "logps/rejected": -553.5045776367188,
      "loss": 0.3337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41685640811920166,
      "rewards/margins": 1.284805178642273,
      "rewards/rejected": -1.701661467552185,
      "step": 8822
    },
    {
      "epoch": 1.1545432240124316,
      "grad_norm": 8.308768272399902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3072946071624756,
      "logits/rejected": -0.22568000853061676,
      "logps/chosen": -543.5152587890625,
      "logps/rejected": -583.9261474609375,
      "loss": 0.5392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6221466064453125,
      "rewards/margins": 2.199449062347412,
      "rewards/rejected": -2.8215956687927246,
      "step": 8823
    },
    {
      "epoch": 1.1546740819497834,
      "grad_norm": 11.460685729980469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29947227239608765,
      "logits/rejected": -0.301263689994812,
      "logps/chosen": -530.7808837890625,
      "logps/rejected": -499.8895568847656,
      "loss": 0.5731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.636072039604187,
      "rewards/margins": 1.9662272930145264,
      "rewards/rejected": -3.602299451828003,
      "step": 8824
    },
    {
      "epoch": 1.154804939887135,
      "grad_norm": 9.930465698242188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11925704032182693,
      "logits/rejected": -0.24726419150829315,
      "logps/chosen": -495.4541320800781,
      "logps/rejected": -591.8195190429688,
      "loss": 0.8162,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7941704988479614,
      "rewards/margins": 0.9278424978256226,
      "rewards/rejected": -2.722012996673584,
      "step": 8825
    },
    {
      "epoch": 1.1549357978244867,
      "grad_norm": 3.8989264965057373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10722918808460236,
      "logits/rejected": -0.3283316195011139,
      "logps/chosen": -323.17315673828125,
      "logps/rejected": -338.65411376953125,
      "loss": 0.2178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.061999797821045,
      "rewards/margins": 1.8217507600784302,
      "rewards/rejected": -2.8837509155273438,
      "step": 8826
    },
    {
      "epoch": 1.1550666557618385,
      "grad_norm": 6.708301067352295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09031034260988235,
      "logits/rejected": -0.3242512047290802,
      "logps/chosen": -669.875244140625,
      "logps/rejected": -564.1138916015625,
      "loss": 0.3232,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21827983856201172,
      "rewards/margins": 1.4421457052230835,
      "rewards/rejected": -1.2238658666610718,
      "step": 8827
    },
    {
      "epoch": 1.1551975136991903,
      "grad_norm": 6.743606090545654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38746219873428345,
      "logits/rejected": -0.40307343006134033,
      "logps/chosen": -363.2513732910156,
      "logps/rejected": -320.66180419921875,
      "loss": 0.4715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1323119401931763,
      "rewards/margins": 0.9186661243438721,
      "rewards/rejected": -2.050977945327759,
      "step": 8828
    },
    {
      "epoch": 1.1553283716365421,
      "grad_norm": 5.8822503089904785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16990262269973755,
      "logits/rejected": 0.2562495768070221,
      "logps/chosen": -608.9222412109375,
      "logps/rejected": -507.1073913574219,
      "loss": 0.2839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9172252416610718,
      "rewards/margins": 2.1774325370788574,
      "rewards/rejected": -3.094658136367798,
      "step": 8829
    },
    {
      "epoch": 1.155459229573894,
      "grad_norm": 6.692580699920654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32852086424827576,
      "logits/rejected": -0.2884211540222168,
      "logps/chosen": -366.70965576171875,
      "logps/rejected": -418.1408386230469,
      "loss": 0.3435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4559280574321747,
      "rewards/margins": 1.5984444618225098,
      "rewards/rejected": -2.0543723106384277,
      "step": 8830
    },
    {
      "epoch": 1.1555900875112457,
      "grad_norm": 6.447526931762695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0653647854924202,
      "logits/rejected": -0.197560116648674,
      "logps/chosen": -451.17950439453125,
      "logps/rejected": -501.6711730957031,
      "loss": 0.2973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3130025565624237,
      "rewards/margins": 2.540759563446045,
      "rewards/rejected": -2.853762149810791,
      "step": 8831
    },
    {
      "epoch": 1.1557209454485973,
      "grad_norm": 11.743185043334961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0012751631438732147,
      "logits/rejected": -0.02147388644516468,
      "logps/chosen": -591.0346069335938,
      "logps/rejected": -581.1405029296875,
      "loss": 0.6961,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6732383966445923,
      "rewards/margins": 0.6890835762023926,
      "rewards/rejected": -2.3623220920562744,
      "step": 8832
    },
    {
      "epoch": 1.155851803385949,
      "grad_norm": 7.843837738037109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46484842896461487,
      "logits/rejected": -0.23170116543769836,
      "logps/chosen": -376.0679931640625,
      "logps/rejected": -531.5387573242188,
      "loss": 0.5798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7834261059761047,
      "rewards/margins": 0.9046552181243896,
      "rewards/rejected": -1.6880813837051392,
      "step": 8833
    },
    {
      "epoch": 1.1559826613233009,
      "grad_norm": 7.557377815246582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18450519442558289,
      "logits/rejected": -0.16372674703598022,
      "logps/chosen": -430.59918212890625,
      "logps/rejected": -328.6732482910156,
      "loss": 0.5581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5335319638252258,
      "rewards/margins": 1.1924781799316406,
      "rewards/rejected": -1.7260099649429321,
      "step": 8834
    },
    {
      "epoch": 1.1561135192606526,
      "grad_norm": 7.251558303833008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.020760998129844666,
      "logits/rejected": -0.020348988473415375,
      "logps/chosen": -568.3330078125,
      "logps/rejected": -605.41796875,
      "loss": 0.4548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09510535001754761,
      "rewards/margins": 1.2953170537948608,
      "rewards/rejected": -1.2002118825912476,
      "step": 8835
    },
    {
      "epoch": 1.1562443771980044,
      "grad_norm": 8.738069534301758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20481114089488983,
      "logits/rejected": -0.21601758897304535,
      "logps/chosen": -585.9739990234375,
      "logps/rejected": -486.01910400390625,
      "loss": 0.4611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18071192502975464,
      "rewards/margins": 1.5425076484680176,
      "rewards/rejected": -1.7232197523117065,
      "step": 8836
    },
    {
      "epoch": 1.1563752351353562,
      "grad_norm": 12.634565353393555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2420436292886734,
      "logits/rejected": -0.05928869917988777,
      "logps/chosen": -532.5553588867188,
      "logps/rejected": -518.1613159179688,
      "loss": 0.7845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7248978614807129,
      "rewards/margins": 0.9982742667198181,
      "rewards/rejected": -1.7231721878051758,
      "step": 8837
    },
    {
      "epoch": 1.156506093072708,
      "grad_norm": 3.5841307640075684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05777575075626373,
      "logits/rejected": 0.02868693694472313,
      "logps/chosen": -649.5455322265625,
      "logps/rejected": -516.6329345703125,
      "loss": 0.1545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15327510237693787,
      "rewards/margins": 2.0728936195373535,
      "rewards/rejected": -1.9196186065673828,
      "step": 8838
    },
    {
      "epoch": 1.1566369510100598,
      "grad_norm": 4.817321300506592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23743124306201935,
      "logits/rejected": -0.292841374874115,
      "logps/chosen": -463.4427185058594,
      "logps/rejected": -438.9706115722656,
      "loss": 0.3215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3724891245365143,
      "rewards/margins": 1.5704208612442017,
      "rewards/rejected": -1.9429099559783936,
      "step": 8839
    },
    {
      "epoch": 1.1567678089474114,
      "grad_norm": 9.955604553222656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1860499083995819,
      "logits/rejected": -0.2155899703502655,
      "logps/chosen": -409.917236328125,
      "logps/rejected": -389.35137939453125,
      "loss": 0.3524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9240036010742188,
      "rewards/margins": 1.8036185503005981,
      "rewards/rejected": -2.7276222705841064,
      "step": 8840
    },
    {
      "epoch": 1.1568986668847632,
      "grad_norm": 9.596368789672852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09624924510717392,
      "logits/rejected": -0.17880168557167053,
      "logps/chosen": -340.82904052734375,
      "logps/rejected": -293.2296447753906,
      "loss": 0.6571,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3516956567764282,
      "rewards/margins": 0.776405394077301,
      "rewards/rejected": -2.128101110458374,
      "step": 8841
    },
    {
      "epoch": 1.157029524822115,
      "grad_norm": 4.773112773895264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20832276344299316,
      "logits/rejected": -0.0848102867603302,
      "logps/chosen": -569.3932495117188,
      "logps/rejected": -400.8028259277344,
      "loss": 0.2073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2512888014316559,
      "rewards/margins": 1.733928918838501,
      "rewards/rejected": -1.4826401472091675,
      "step": 8842
    },
    {
      "epoch": 1.1571603827594668,
      "grad_norm": 9.830348014831543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20652759075164795,
      "logits/rejected": -0.33390337228775024,
      "logps/chosen": -501.2784423828125,
      "logps/rejected": -392.9726867675781,
      "loss": 0.5572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1162333488464355,
      "rewards/margins": 1.1618155241012573,
      "rewards/rejected": -2.2780487537384033,
      "step": 8843
    },
    {
      "epoch": 1.1572912406968185,
      "grad_norm": 8.841482162475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18914267420768738,
      "logits/rejected": -0.059502482414245605,
      "logps/chosen": -398.28289794921875,
      "logps/rejected": -515.8413696289062,
      "loss": 0.4977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6199058890342712,
      "rewards/margins": 1.1726053953170776,
      "rewards/rejected": -1.792511224746704,
      "step": 8844
    },
    {
      "epoch": 1.1574220986341703,
      "grad_norm": 8.877653121948242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03513453155755997,
      "logits/rejected": -0.011317066848278046,
      "logps/chosen": -444.8402404785156,
      "logps/rejected": -468.6921691894531,
      "loss": 0.8006,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29621267318725586,
      "rewards/margins": 0.85672527551651,
      "rewards/rejected": -1.1529380083084106,
      "step": 8845
    },
    {
      "epoch": 1.1575529565715221,
      "grad_norm": 5.239426612854004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08949802815914154,
      "logits/rejected": 0.1235433965921402,
      "logps/chosen": -370.6612243652344,
      "logps/rejected": -426.6783447265625,
      "loss": 0.2583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9401769638061523,
      "rewards/margins": 1.4565045833587646,
      "rewards/rejected": -2.396681547164917,
      "step": 8846
    },
    {
      "epoch": 1.1576838145088737,
      "grad_norm": 6.005852699279785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0375073105096817,
      "logits/rejected": 0.00011400878429412842,
      "logps/chosen": -438.9183654785156,
      "logps/rejected": -370.2309265136719,
      "loss": 0.2806,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3119693994522095,
      "rewards/margins": 1.718533992767334,
      "rewards/rejected": -2.030503511428833,
      "step": 8847
    },
    {
      "epoch": 1.1578146724462255,
      "grad_norm": 3.5983452796936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1068132221698761,
      "logits/rejected": -0.07482757419347763,
      "logps/chosen": -503.69097900390625,
      "logps/rejected": -420.28582763671875,
      "loss": 0.1524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19446620345115662,
      "rewards/margins": 2.8874917030334473,
      "rewards/rejected": -3.0819578170776367,
      "step": 8848
    },
    {
      "epoch": 1.1579455303835773,
      "grad_norm": 6.016905307769775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10132362693548203,
      "logits/rejected": -0.36743247509002686,
      "logps/chosen": -340.36175537109375,
      "logps/rejected": -366.89910888671875,
      "loss": 0.299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1839576959609985,
      "rewards/margins": 1.978684902191162,
      "rewards/rejected": -3.16264271736145,
      "step": 8849
    },
    {
      "epoch": 1.158076388320929,
      "grad_norm": 10.350059509277344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01716385781764984,
      "logits/rejected": 0.04650241136550903,
      "logps/chosen": -434.58978271484375,
      "logps/rejected": -472.531494140625,
      "loss": 0.7769,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8534990549087524,
      "rewards/margins": 0.5704689621925354,
      "rewards/rejected": -1.4239678382873535,
      "step": 8850
    },
    {
      "epoch": 1.1582072462582809,
      "grad_norm": 6.686267852783203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1991833746433258,
      "logits/rejected": -0.3259217441082001,
      "logps/chosen": -392.6733093261719,
      "logps/rejected": -431.9966125488281,
      "loss": 0.3491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1441187858581543,
      "rewards/margins": 1.4712467193603516,
      "rewards/rejected": -2.615365505218506,
      "step": 8851
    },
    {
      "epoch": 1.1583381041956327,
      "grad_norm": 7.400965690612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10716528445482254,
      "logits/rejected": 0.12316639721393585,
      "logps/chosen": -496.3165588378906,
      "logps/rejected": -417.01580810546875,
      "loss": 0.3266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5497795343399048,
      "rewards/margins": 1.5476188659667969,
      "rewards/rejected": -2.097398281097412,
      "step": 8852
    },
    {
      "epoch": 1.1584689621329844,
      "grad_norm": 10.558098793029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16292311251163483,
      "logits/rejected": 0.00658529344946146,
      "logps/chosen": -599.65380859375,
      "logps/rejected": -500.8787841796875,
      "loss": 0.6298,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5248367786407471,
      "rewards/margins": 0.9522775411605835,
      "rewards/rejected": -1.4771143198013306,
      "step": 8853
    },
    {
      "epoch": 1.1585998200703362,
      "grad_norm": 11.467574119567871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10194934904575348,
      "logits/rejected": -0.0205862894654274,
      "logps/chosen": -601.7689208984375,
      "logps/rejected": -595.3272094726562,
      "loss": 0.6795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7899230718612671,
      "rewards/margins": 1.5792053937911987,
      "rewards/rejected": -2.3691282272338867,
      "step": 8854
    },
    {
      "epoch": 1.1587306780076878,
      "grad_norm": 5.074971675872803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05508895963430405,
      "logits/rejected": -0.17204944789409637,
      "logps/chosen": -532.958251953125,
      "logps/rejected": -577.3394775390625,
      "loss": 0.28,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07166194915771484,
      "rewards/margins": 1.8107882738113403,
      "rewards/rejected": -1.739126205444336,
      "step": 8855
    },
    {
      "epoch": 1.1588615359450396,
      "grad_norm": 4.137347221374512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07234862446784973,
      "logits/rejected": 0.01604180783033371,
      "logps/chosen": -352.6099853515625,
      "logps/rejected": -442.0040283203125,
      "loss": 0.166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09010201692581177,
      "rewards/margins": 2.236449956893921,
      "rewards/rejected": -2.146347761154175,
      "step": 8856
    },
    {
      "epoch": 1.1589923938823914,
      "grad_norm": 9.422444343566895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011898711323738098,
      "logits/rejected": 0.06142684817314148,
      "logps/chosen": -558.648193359375,
      "logps/rejected": -720.828125,
      "loss": 0.4694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8426722288131714,
      "rewards/margins": 0.7644140124320984,
      "rewards/rejected": -1.607086181640625,
      "step": 8857
    },
    {
      "epoch": 1.1591232518197432,
      "grad_norm": 6.84122371673584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.31260645389556885,
      "logits/rejected": 0.05436040833592415,
      "logps/chosen": -498.2484436035156,
      "logps/rejected": -423.05499267578125,
      "loss": 0.5096,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3431532084941864,
      "rewards/margins": 0.8877195119857788,
      "rewards/rejected": -1.2308727502822876,
      "step": 8858
    },
    {
      "epoch": 1.159254109757095,
      "grad_norm": 15.782937049865723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26198330521583557,
      "logits/rejected": -0.43824514746665955,
      "logps/chosen": -647.1767578125,
      "logps/rejected": -612.2705078125,
      "loss": 0.891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.363311767578125,
      "rewards/margins": 0.4401928186416626,
      "rewards/rejected": -1.8035045862197876,
      "step": 8859
    },
    {
      "epoch": 1.1593849676944468,
      "grad_norm": 8.404786109924316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09466679394245148,
      "logits/rejected": -0.18009592592716217,
      "logps/chosen": -586.9827880859375,
      "logps/rejected": -636.5880737304688,
      "loss": 0.4581,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5581600069999695,
      "rewards/margins": 1.7690365314483643,
      "rewards/rejected": -2.3271965980529785,
      "step": 8860
    },
    {
      "epoch": 1.1595158256317986,
      "grad_norm": 7.892608165740967,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10678406804800034,
      "logits/rejected": 0.020945103839039803,
      "logps/chosen": -543.0118408203125,
      "logps/rejected": -542.2008056640625,
      "loss": 0.5803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7518717050552368,
      "rewards/margins": 0.9397174119949341,
      "rewards/rejected": -1.691589117050171,
      "step": 8861
    },
    {
      "epoch": 1.1596466835691501,
      "grad_norm": 6.809704303741455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14283809065818787,
      "logits/rejected": -0.0701570063829422,
      "logps/chosen": -594.608154296875,
      "logps/rejected": -352.23699951171875,
      "loss": 0.3128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4626409709453583,
      "rewards/margins": 2.5714540481567383,
      "rewards/rejected": -2.1088130474090576,
      "step": 8862
    },
    {
      "epoch": 1.159777541506502,
      "grad_norm": 8.265915870666504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008527325466275215,
      "logits/rejected": 0.022130127996206284,
      "logps/chosen": -333.8902893066406,
      "logps/rejected": -340.2228698730469,
      "loss": 0.5145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46548086404800415,
      "rewards/margins": 1.4051705598831177,
      "rewards/rejected": -1.8706516027450562,
      "step": 8863
    },
    {
      "epoch": 1.1599083994438537,
      "grad_norm": 9.162232398986816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01976597309112549,
      "logits/rejected": -0.0678330659866333,
      "logps/chosen": -501.0791931152344,
      "logps/rejected": -494.6155700683594,
      "loss": 0.5269,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7523725032806396,
      "rewards/margins": 1.1079628467559814,
      "rewards/rejected": -1.8603354692459106,
      "step": 8864
    },
    {
      "epoch": 1.1600392573812055,
      "grad_norm": 8.42317008972168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006221983581781387,
      "logits/rejected": -0.12526416778564453,
      "logps/chosen": -388.6842346191406,
      "logps/rejected": -403.7035827636719,
      "loss": 0.5477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2679266035556793,
      "rewards/margins": 1.021774172782898,
      "rewards/rejected": -1.2897007465362549,
      "step": 8865
    },
    {
      "epoch": 1.1601701153185573,
      "grad_norm": 7.846554279327393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20481538772583008,
      "logits/rejected": -0.21452221274375916,
      "logps/chosen": -511.5076904296875,
      "logps/rejected": -585.3450927734375,
      "loss": 0.4989,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.131261944770813,
      "rewards/margins": 1.1137800216674805,
      "rewards/rejected": -2.245041847229004,
      "step": 8866
    },
    {
      "epoch": 1.160300973255909,
      "grad_norm": 8.460734367370605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016209624707698822,
      "logits/rejected": -0.20392736792564392,
      "logps/chosen": -376.3908996582031,
      "logps/rejected": -398.8299865722656,
      "loss": 0.4887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3385844230651855,
      "rewards/margins": 1.0026730298995972,
      "rewards/rejected": -2.3412575721740723,
      "step": 8867
    },
    {
      "epoch": 1.1604318311932609,
      "grad_norm": 5.216545104980469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.038777850568294525,
      "logits/rejected": -0.20713379979133606,
      "logps/chosen": -449.2242736816406,
      "logps/rejected": -509.12591552734375,
      "loss": 0.3247,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6270958185195923,
      "rewards/margins": 1.2367936372756958,
      "rewards/rejected": -1.8638896942138672,
      "step": 8868
    },
    {
      "epoch": 1.1605626891306127,
      "grad_norm": 6.112732410430908,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04970882087945938,
      "logits/rejected": -0.015433602035045624,
      "logps/chosen": -521.1641235351562,
      "logps/rejected": -374.56976318359375,
      "loss": 0.3645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0926487445831299,
      "rewards/margins": 1.1790682077407837,
      "rewards/rejected": -2.271717071533203,
      "step": 8869
    },
    {
      "epoch": 1.1606935470679645,
      "grad_norm": 5.618098258972168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12988823652267456,
      "logits/rejected": -0.005478769540786743,
      "logps/chosen": -595.3892211914062,
      "logps/rejected": -522.782470703125,
      "loss": 0.4129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3793826997280121,
      "rewards/margins": 1.5796704292297363,
      "rewards/rejected": -1.9590530395507812,
      "step": 8870
    },
    {
      "epoch": 1.160824405005316,
      "grad_norm": 8.62476634979248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18179768323898315,
      "logits/rejected": -0.10047593712806702,
      "logps/chosen": -352.1070251464844,
      "logps/rejected": -338.5125732421875,
      "loss": 0.5943,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18490856885910034,
      "rewards/margins": 0.4176153540611267,
      "rewards/rejected": -0.602523922920227,
      "step": 8871
    },
    {
      "epoch": 1.1609552629426678,
      "grad_norm": 10.730971336364746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1249295324087143,
      "logits/rejected": 0.16550923883914948,
      "logps/chosen": -423.261962890625,
      "logps/rejected": -508.86785888671875,
      "loss": 0.7395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.764650583267212,
      "rewards/margins": 1.3386366367340088,
      "rewards/rejected": -3.1032874584198,
      "step": 8872
    },
    {
      "epoch": 1.1610861208800196,
      "grad_norm": 5.048041820526123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07185249775648117,
      "logits/rejected": 0.02851855382323265,
      "logps/chosen": -535.6000366210938,
      "logps/rejected": -550.0576782226562,
      "loss": 0.2957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.054585494101047516,
      "rewards/margins": 1.3095417022705078,
      "rewards/rejected": -1.2549562454223633,
      "step": 8873
    },
    {
      "epoch": 1.1612169788173714,
      "grad_norm": 8.182147026062012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21907074749469757,
      "logits/rejected": -0.45751112699508667,
      "logps/chosen": -491.5931701660156,
      "logps/rejected": -485.7126770019531,
      "loss": 0.4102,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49422574043273926,
      "rewards/margins": 1.632799506187439,
      "rewards/rejected": -2.1270253658294678,
      "step": 8874
    },
    {
      "epoch": 1.1613478367547232,
      "grad_norm": 13.494014739990234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09403926134109497,
      "logits/rejected": -0.2255409210920334,
      "logps/chosen": -516.3264770507812,
      "logps/rejected": -438.77471923828125,
      "loss": 0.6962,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5276283025741577,
      "rewards/margins": 1.0235087871551514,
      "rewards/rejected": -2.5511374473571777,
      "step": 8875
    },
    {
      "epoch": 1.161478694692075,
      "grad_norm": 6.909602165222168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3038472831249237,
      "logits/rejected": 0.11917967349290848,
      "logps/chosen": -492.7767333984375,
      "logps/rejected": -549.0945434570312,
      "loss": 0.3103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8368355631828308,
      "rewards/margins": 1.457331657409668,
      "rewards/rejected": -2.2941672801971436,
      "step": 8876
    },
    {
      "epoch": 1.1616095526294266,
      "grad_norm": 11.87552261352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39641106128692627,
      "logits/rejected": -0.3284362554550171,
      "logps/chosen": -442.37286376953125,
      "logps/rejected": -416.624755859375,
      "loss": 0.5225,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5767364501953125,
      "rewards/margins": 1.1313047409057617,
      "rewards/rejected": -1.7080411911010742,
      "step": 8877
    },
    {
      "epoch": 1.1617404105667783,
      "grad_norm": 9.366419792175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2362128496170044,
      "logits/rejected": -0.2004386931657791,
      "logps/chosen": -546.143798828125,
      "logps/rejected": -408.2005920410156,
      "loss": 0.4251,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42968976497650146,
      "rewards/margins": 1.2587363719940186,
      "rewards/rejected": -1.6884262561798096,
      "step": 8878
    },
    {
      "epoch": 1.1618712685041301,
      "grad_norm": 10.241311073303223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.095134437084198,
      "logits/rejected": -0.13731864094734192,
      "logps/chosen": -505.6415100097656,
      "logps/rejected": -392.6885070800781,
      "loss": 0.5193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.142441749572754,
      "rewards/margins": 0.7250226736068726,
      "rewards/rejected": -1.867464303970337,
      "step": 8879
    },
    {
      "epoch": 1.162002126441482,
      "grad_norm": 6.836503505706787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02027321420609951,
      "logits/rejected": -0.04826135188341141,
      "logps/chosen": -562.3670654296875,
      "logps/rejected": -590.6611328125,
      "loss": 0.3299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7628618478775024,
      "rewards/margins": 1.2013789415359497,
      "rewards/rejected": -1.9642409086227417,
      "step": 8880
    },
    {
      "epoch": 1.1621329843788337,
      "grad_norm": 6.7275824546813965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1683759242296219,
      "logits/rejected": -0.23918108642101288,
      "logps/chosen": -468.8018493652344,
      "logps/rejected": -324.872314453125,
      "loss": 0.4307,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3324264585971832,
      "rewards/margins": 1.8916946649551392,
      "rewards/rejected": -2.22412109375,
      "step": 8881
    },
    {
      "epoch": 1.1622638423161855,
      "grad_norm": 5.001576900482178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15904468297958374,
      "logits/rejected": -0.19204111397266388,
      "logps/chosen": -527.0765380859375,
      "logps/rejected": -627.3223876953125,
      "loss": 0.1942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6859065890312195,
      "rewards/margins": 1.8381562232971191,
      "rewards/rejected": -2.5240628719329834,
      "step": 8882
    },
    {
      "epoch": 1.1623947002535373,
      "grad_norm": 5.380568027496338,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024107351899147034,
      "logits/rejected": 0.06789721548557281,
      "logps/chosen": -477.5148620605469,
      "logps/rejected": -425.13525390625,
      "loss": 0.3345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3833850920200348,
      "rewards/margins": 1.4728237390518188,
      "rewards/rejected": -1.8562089204788208,
      "step": 8883
    },
    {
      "epoch": 1.162525558190889,
      "grad_norm": 3.6043455600738525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2097891867160797,
      "logits/rejected": -0.16229528188705444,
      "logps/chosen": -344.2785339355469,
      "logps/rejected": -309.5191955566406,
      "loss": 0.1903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.856022298336029,
      "rewards/margins": 2.318896532058716,
      "rewards/rejected": -3.1749186515808105,
      "step": 8884
    },
    {
      "epoch": 1.1626564161282409,
      "grad_norm": 6.011105537414551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042739078402519226,
      "logits/rejected": -0.1328536570072174,
      "logps/chosen": -310.5675354003906,
      "logps/rejected": -393.5754089355469,
      "loss": 0.2988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7167925834655762,
      "rewards/margins": 1.539057970046997,
      "rewards/rejected": -3.2558507919311523,
      "step": 8885
    },
    {
      "epoch": 1.1627872740655925,
      "grad_norm": 8.725866317749023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1017550379037857,
      "logits/rejected": -0.042181242257356644,
      "logps/chosen": -480.8641052246094,
      "logps/rejected": -316.16131591796875,
      "loss": 0.4429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5330206155776978,
      "rewards/margins": 0.9734151363372803,
      "rewards/rejected": -2.5064358711242676,
      "step": 8886
    },
    {
      "epoch": 1.1629181320029442,
      "grad_norm": 5.303892612457275,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16810615360736847,
      "logits/rejected": 0.17274785041809082,
      "logps/chosen": -363.56329345703125,
      "logps/rejected": -443.26947021484375,
      "loss": 0.2876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2430461645126343,
      "rewards/margins": 1.3508270978927612,
      "rewards/rejected": -2.5938732624053955,
      "step": 8887
    },
    {
      "epoch": 1.163048989940296,
      "grad_norm": 9.267354011535645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2712363600730896,
      "logits/rejected": 0.057532019913196564,
      "logps/chosen": -424.5664367675781,
      "logps/rejected": -314.94927978515625,
      "loss": 0.4329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.461217999458313,
      "rewards/margins": 0.8762035965919495,
      "rewards/rejected": -2.3374216556549072,
      "step": 8888
    },
    {
      "epoch": 1.1631798478776478,
      "grad_norm": 4.930084228515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07433576881885529,
      "logits/rejected": -0.06497330963611603,
      "logps/chosen": -452.53070068359375,
      "logps/rejected": -320.921875,
      "loss": 0.2771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20531949400901794,
      "rewards/margins": 1.6441770792007446,
      "rewards/rejected": -1.4388574361801147,
      "step": 8889
    },
    {
      "epoch": 1.1633107058149996,
      "grad_norm": 7.273593425750732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02030441351234913,
      "logits/rejected": 0.005315177142620087,
      "logps/chosen": -517.9649658203125,
      "logps/rejected": -439.3362731933594,
      "loss": 0.4697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.25997793674469,
      "rewards/margins": 0.8748893737792969,
      "rewards/rejected": -2.1348671913146973,
      "step": 8890
    },
    {
      "epoch": 1.1634415637523514,
      "grad_norm": 7.138156414031982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12518750131130219,
      "logits/rejected": -0.3635408878326416,
      "logps/chosen": -343.5483093261719,
      "logps/rejected": -441.7279968261719,
      "loss": 0.4863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3393041491508484,
      "rewards/margins": 1.2600716352462769,
      "rewards/rejected": -1.5993757247924805,
      "step": 8891
    },
    {
      "epoch": 1.1635724216897032,
      "grad_norm": 11.403815269470215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24272121489048004,
      "logits/rejected": 0.45527321100234985,
      "logps/chosen": -407.562255859375,
      "logps/rejected": -449.588623046875,
      "loss": 0.7354,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2324323654174805,
      "rewards/margins": 0.3461707830429077,
      "rewards/rejected": -1.5786031484603882,
      "step": 8892
    },
    {
      "epoch": 1.1637032796270548,
      "grad_norm": 11.553194046020508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.301872581243515,
      "logits/rejected": -0.2175036370754242,
      "logps/chosen": -266.5190124511719,
      "logps/rejected": -374.45611572265625,
      "loss": 1.0289,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.046261787414551,
      "rewards/margins": 0.23202091455459595,
      "rewards/rejected": -2.278282642364502,
      "step": 8893
    },
    {
      "epoch": 1.1638341375644066,
      "grad_norm": 10.742514610290527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2379082888364792,
      "logits/rejected": -0.28683528304100037,
      "logps/chosen": -496.0308532714844,
      "logps/rejected": -391.6163024902344,
      "loss": 0.4676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7267875671386719,
      "rewards/margins": 1.6632397174835205,
      "rewards/rejected": -2.3900272846221924,
      "step": 8894
    },
    {
      "epoch": 1.1639649955017584,
      "grad_norm": 5.578976631164551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19216486811637878,
      "logits/rejected": -0.3781971335411072,
      "logps/chosen": -424.9778747558594,
      "logps/rejected": -366.42803955078125,
      "loss": 0.315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3566983938217163,
      "rewards/margins": 1.2254538536071777,
      "rewards/rejected": -2.5821523666381836,
      "step": 8895
    },
    {
      "epoch": 1.1640958534391102,
      "grad_norm": 12.931050300598145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013505461625754833,
      "logits/rejected": -0.1511305719614029,
      "logps/chosen": -728.3080444335938,
      "logps/rejected": -587.58349609375,
      "loss": 0.953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3397003412246704,
      "rewards/margins": 0.5543208122253418,
      "rewards/rejected": -1.8940212726593018,
      "step": 8896
    },
    {
      "epoch": 1.164226711376462,
      "grad_norm": 5.758410930633545,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03214050829410553,
      "logits/rejected": 0.15690967440605164,
      "logps/chosen": -405.8536376953125,
      "logps/rejected": -379.0404052734375,
      "loss": 0.3577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3574506640434265,
      "rewards/margins": 1.2222765684127808,
      "rewards/rejected": -1.5797271728515625,
      "step": 8897
    },
    {
      "epoch": 1.1643575693138137,
      "grad_norm": 8.835753440856934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25139811635017395,
      "logits/rejected": -0.3091343641281128,
      "logps/chosen": -558.0463256835938,
      "logps/rejected": -734.7733154296875,
      "loss": 0.4852,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0804214477539062,
      "rewards/margins": 0.6094883680343628,
      "rewards/rejected": -1.689909815788269,
      "step": 8898
    },
    {
      "epoch": 1.1644884272511655,
      "grad_norm": 16.415739059448242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11565320193767548,
      "logits/rejected": -0.04311170056462288,
      "logps/chosen": -368.68475341796875,
      "logps/rejected": -339.6483459472656,
      "loss": 0.9188,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.079526424407959,
      "rewards/margins": 0.6981064081192017,
      "rewards/rejected": -2.7776331901550293,
      "step": 8899
    },
    {
      "epoch": 1.1646192851885173,
      "grad_norm": 5.510515213012695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12809617817401886,
      "logits/rejected": -0.29578065872192383,
      "logps/chosen": -547.3799438476562,
      "logps/rejected": -564.182373046875,
      "loss": 0.377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38572850823402405,
      "rewards/margins": 1.5297654867172241,
      "rewards/rejected": -1.9154939651489258,
      "step": 8900
    },
    {
      "epoch": 1.1647501431258689,
      "grad_norm": 32.224239349365234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15434177219867706,
      "logits/rejected": -0.24798177182674408,
      "logps/chosen": -524.3947143554688,
      "logps/rejected": -347.5686340332031,
      "loss": 0.9948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2205134630203247,
      "rewards/margins": 0.9146446585655212,
      "rewards/rejected": -2.1351583003997803,
      "step": 8901
    },
    {
      "epoch": 1.1648810010632207,
      "grad_norm": 10.172640800476074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03911055997014046,
      "logits/rejected": -0.16425876319408417,
      "logps/chosen": -556.1221313476562,
      "logps/rejected": -498.6581726074219,
      "loss": 0.5036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2982397079467773,
      "rewards/margins": 1.4531351327896118,
      "rewards/rejected": -2.7513749599456787,
      "step": 8902
    },
    {
      "epoch": 1.1650118590005725,
      "grad_norm": 8.304681777954102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21952202916145325,
      "logits/rejected": 0.07755638659000397,
      "logps/chosen": -323.4939270019531,
      "logps/rejected": -499.9723205566406,
      "loss": 0.3783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1848897784948349,
      "rewards/margins": 1.2584973573684692,
      "rewards/rejected": -1.4433872699737549,
      "step": 8903
    },
    {
      "epoch": 1.1651427169379243,
      "grad_norm": 8.855569839477539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47164681553840637,
      "logits/rejected": -0.4520453214645386,
      "logps/chosen": -388.2861328125,
      "logps/rejected": -596.718994140625,
      "loss": 0.5088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7877309322357178,
      "rewards/margins": 1.3237764835357666,
      "rewards/rejected": -3.1115076541900635,
      "step": 8904
    },
    {
      "epoch": 1.165273574875276,
      "grad_norm": 9.740490913391113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05923209339380264,
      "logits/rejected": -0.1361650675535202,
      "logps/chosen": -699.52197265625,
      "logps/rejected": -547.994140625,
      "loss": 0.6494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5391808152198792,
      "rewards/margins": 0.7450042366981506,
      "rewards/rejected": -1.2841849327087402,
      "step": 8905
    },
    {
      "epoch": 1.1654044328126278,
      "grad_norm": 8.998677253723145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.033674757927656174,
      "logits/rejected": -0.3119766116142273,
      "logps/chosen": -542.34375,
      "logps/rejected": -463.5959167480469,
      "loss": 0.5143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3313888907432556,
      "rewards/margins": 1.188596487045288,
      "rewards/rejected": -1.519985318183899,
      "step": 8906
    },
    {
      "epoch": 1.1655352907499796,
      "grad_norm": 6.7582244873046875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07182896137237549,
      "logits/rejected": -0.17719383537769318,
      "logps/chosen": -353.2803039550781,
      "logps/rejected": -408.2135314941406,
      "loss": 0.2398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6945488452911377,
      "rewards/margins": 2.150602102279663,
      "rewards/rejected": -3.84515118598938,
      "step": 8907
    },
    {
      "epoch": 1.1656661486873312,
      "grad_norm": 6.459764003753662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06533006578683853,
      "logits/rejected": -0.11657015979290009,
      "logps/chosen": -480.1639709472656,
      "logps/rejected": -505.2586669921875,
      "loss": 0.3142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7601598501205444,
      "rewards/margins": 1.8144077062606812,
      "rewards/rejected": -2.5745675563812256,
      "step": 8908
    },
    {
      "epoch": 1.165797006624683,
      "grad_norm": 5.19575834274292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0073546152561903,
      "logits/rejected": -0.2119290977716446,
      "logps/chosen": -592.0662231445312,
      "logps/rejected": -433.1387634277344,
      "loss": 0.3611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.396109014749527,
      "rewards/margins": 1.1243594884872437,
      "rewards/rejected": -1.5204683542251587,
      "step": 8909
    },
    {
      "epoch": 1.1659278645620348,
      "grad_norm": 8.037283897399902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25206124782562256,
      "logits/rejected": -0.15428400039672852,
      "logps/chosen": -414.3458251953125,
      "logps/rejected": -566.47509765625,
      "loss": 0.3726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1761584281921387,
      "rewards/margins": 2.029491662979126,
      "rewards/rejected": -3.2056503295898438,
      "step": 8910
    },
    {
      "epoch": 1.1660587224993866,
      "grad_norm": 7.710400104522705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1314018964767456,
      "logits/rejected": 0.12446523457765579,
      "logps/chosen": -430.7793273925781,
      "logps/rejected": -547.5693969726562,
      "loss": 0.5726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7748672366142273,
      "rewards/margins": 0.6107077598571777,
      "rewards/rejected": -1.3855749368667603,
      "step": 8911
    },
    {
      "epoch": 1.1661895804367384,
      "grad_norm": 3.0321662425994873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14296092092990875,
      "logits/rejected": -0.4004875123500824,
      "logps/chosen": -371.9927673339844,
      "logps/rejected": -343.1066589355469,
      "loss": 0.1895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2998698949813843,
      "rewards/margins": 2.079533100128174,
      "rewards/rejected": -3.3794026374816895,
      "step": 8912
    },
    {
      "epoch": 1.1663204383740902,
      "grad_norm": 7.5246500968933105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37938016653060913,
      "logits/rejected": -0.13955174386501312,
      "logps/chosen": -473.0730895996094,
      "logps/rejected": -335.1285705566406,
      "loss": 0.3289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4073452949523926,
      "rewards/margins": 1.7036004066467285,
      "rewards/rejected": -3.110945701599121,
      "step": 8913
    },
    {
      "epoch": 1.166451296311442,
      "grad_norm": 11.606619834899902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1440504491329193,
      "logits/rejected": -0.28761085867881775,
      "logps/chosen": -426.9847717285156,
      "logps/rejected": -421.7375793457031,
      "loss": 0.4615,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1408170461654663,
      "rewards/margins": 1.3843905925750732,
      "rewards/rejected": -2.525207757949829,
      "step": 8914
    },
    {
      "epoch": 1.1665821542487937,
      "grad_norm": 7.363675117492676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06221605837345123,
      "logits/rejected": -0.059216052293777466,
      "logps/chosen": -541.0332641601562,
      "logps/rejected": -492.9391784667969,
      "loss": 0.4489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6861755847930908,
      "rewards/margins": 1.216063380241394,
      "rewards/rejected": -2.9022393226623535,
      "step": 8915
    },
    {
      "epoch": 1.1667130121861455,
      "grad_norm": 5.466386795043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3096827566623688,
      "logits/rejected": -0.3628876805305481,
      "logps/chosen": -582.8218994140625,
      "logps/rejected": -485.31402587890625,
      "loss": 0.2553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4605777263641357,
      "rewards/margins": 1.6484431028366089,
      "rewards/rejected": -3.109020948410034,
      "step": 8916
    },
    {
      "epoch": 1.166843870123497,
      "grad_norm": 6.182705402374268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23900118470191956,
      "logits/rejected": 0.2842671871185303,
      "logps/chosen": -334.4201354980469,
      "logps/rejected": -407.7333068847656,
      "loss": 0.3515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5520271062850952,
      "rewards/margins": 1.0787150859832764,
      "rewards/rejected": -1.6307423114776611,
      "step": 8917
    },
    {
      "epoch": 1.166974728060849,
      "grad_norm": 7.449584484100342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26178544759750366,
      "logits/rejected": -0.2625187039375305,
      "logps/chosen": -455.94940185546875,
      "logps/rejected": -270.4599304199219,
      "loss": 0.4895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7517813444137573,
      "rewards/margins": 1.036719560623169,
      "rewards/rejected": -1.7885009050369263,
      "step": 8918
    },
    {
      "epoch": 1.1671055859982007,
      "grad_norm": 9.143301010131836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2216004729270935,
      "logits/rejected": 0.13074374198913574,
      "logps/chosen": -489.29351806640625,
      "logps/rejected": -573.4807739257812,
      "loss": 0.5081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.431784451007843,
      "rewards/margins": 1.411035418510437,
      "rewards/rejected": -1.842820167541504,
      "step": 8919
    },
    {
      "epoch": 1.1672364439355525,
      "grad_norm": 2.8791239261627197,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02435341477394104,
      "logits/rejected": -0.11547696590423584,
      "logps/chosen": -511.3748779296875,
      "logps/rejected": -556.8157348632812,
      "loss": 0.1699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2269139289855957,
      "rewards/margins": 2.619478940963745,
      "rewards/rejected": -3.846392869949341,
      "step": 8920
    },
    {
      "epoch": 1.1673673018729043,
      "grad_norm": 4.003112316131592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14836040139198303,
      "logits/rejected": -0.158442422747612,
      "logps/chosen": -371.1195068359375,
      "logps/rejected": -418.2919921875,
      "loss": 0.228,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7630390524864197,
      "rewards/margins": 1.7596958875656128,
      "rewards/rejected": -2.5227346420288086,
      "step": 8921
    },
    {
      "epoch": 1.167498159810256,
      "grad_norm": 6.529785633087158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20221300423145294,
      "logits/rejected": 0.008014857769012451,
      "logps/chosen": -464.4158935546875,
      "logps/rejected": -508.4571228027344,
      "loss": 0.3889,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9095048904418945,
      "rewards/margins": 1.1386014223098755,
      "rewards/rejected": -2.0481064319610596,
      "step": 8922
    },
    {
      "epoch": 1.1676290177476076,
      "grad_norm": 3.5928239822387695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0434579961001873,
      "logits/rejected": -0.08057931065559387,
      "logps/chosen": -596.7366333007812,
      "logps/rejected": -538.3187255859375,
      "loss": 0.189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36018607020378113,
      "rewards/margins": 1.8191696405410767,
      "rewards/rejected": -1.4589836597442627,
      "step": 8923
    },
    {
      "epoch": 1.1677598756849594,
      "grad_norm": 10.928573608398438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.224094420671463,
      "logits/rejected": -0.08055713772773743,
      "logps/chosen": -439.6465148925781,
      "logps/rejected": -388.2396545410156,
      "loss": 0.6211,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.490619421005249,
      "rewards/margins": 2.090320110321045,
      "rewards/rejected": -3.580939769744873,
      "step": 8924
    },
    {
      "epoch": 1.1678907336223112,
      "grad_norm": 8.773768424987793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08613759279251099,
      "logits/rejected": 0.06403502821922302,
      "logps/chosen": -486.5162048339844,
      "logps/rejected": -457.0557556152344,
      "loss": 0.5451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.17410793900489807,
      "rewards/margins": 1.3350226879119873,
      "rewards/rejected": -1.1609147787094116,
      "step": 8925
    },
    {
      "epoch": 1.168021591559663,
      "grad_norm": 12.465446472167969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.056153617799282074,
      "logits/rejected": -0.12033537030220032,
      "logps/chosen": -475.5770568847656,
      "logps/rejected": -430.22186279296875,
      "loss": 0.6598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6817165613174438,
      "rewards/margins": 0.9963574409484863,
      "rewards/rejected": -2.6780738830566406,
      "step": 8926
    },
    {
      "epoch": 1.1681524494970148,
      "grad_norm": 6.207586288452148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02012769505381584,
      "logits/rejected": -0.21186500787734985,
      "logps/chosen": -435.4031066894531,
      "logps/rejected": -448.1021423339844,
      "loss": 0.3182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6254991888999939,
      "rewards/margins": 1.8223440647125244,
      "rewards/rejected": -2.447843074798584,
      "step": 8927
    },
    {
      "epoch": 1.1682833074343666,
      "grad_norm": 4.870331764221191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0031452421098947525,
      "logits/rejected": -0.28675371408462524,
      "logps/chosen": -436.1163330078125,
      "logps/rejected": -472.23309326171875,
      "loss": 0.2666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6624200344085693,
      "rewards/margins": 1.6733431816101074,
      "rewards/rejected": -3.3357632160186768,
      "step": 8928
    },
    {
      "epoch": 1.1684141653717184,
      "grad_norm": 6.264674663543701,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07280056178569794,
      "logits/rejected": -0.02389633096754551,
      "logps/chosen": -382.1550598144531,
      "logps/rejected": -301.1446533203125,
      "loss": 0.3688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17146433889865875,
      "rewards/margins": 1.069427728652954,
      "rewards/rejected": -1.240891933441162,
      "step": 8929
    },
    {
      "epoch": 1.1685450233090702,
      "grad_norm": 11.814981460571289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05485190451145172,
      "logits/rejected": -0.40839883685112,
      "logps/chosen": -439.6378173828125,
      "logps/rejected": -522.8640747070312,
      "loss": 0.6908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2796555757522583,
      "rewards/margins": 1.197161078453064,
      "rewards/rejected": -2.4768166542053223,
      "step": 8930
    },
    {
      "epoch": 1.168675881246422,
      "grad_norm": 9.951131820678711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10137075930833817,
      "logits/rejected": -0.07700885087251663,
      "logps/chosen": -390.30999755859375,
      "logps/rejected": -402.3034973144531,
      "loss": 0.801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.017112374305725,
      "rewards/margins": 0.05424727499485016,
      "rewards/rejected": -1.071359634399414,
      "step": 8931
    },
    {
      "epoch": 1.1688067391837735,
      "grad_norm": 8.161110877990723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26361018419265747,
      "logits/rejected": -0.37193796038627625,
      "logps/chosen": -376.47052001953125,
      "logps/rejected": -394.7261962890625,
      "loss": 0.4926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2458748817443848,
      "rewards/margins": 1.177261233329773,
      "rewards/rejected": -2.4231362342834473,
      "step": 8932
    },
    {
      "epoch": 1.1689375971211253,
      "grad_norm": 5.698119640350342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012302577495574951,
      "logits/rejected": -0.021471258252859116,
      "logps/chosen": -361.8285827636719,
      "logps/rejected": -332.93304443359375,
      "loss": 0.22,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0401983261108398,
      "rewards/margins": 1.9673233032226562,
      "rewards/rejected": -3.007521867752075,
      "step": 8933
    },
    {
      "epoch": 1.1690684550584771,
      "grad_norm": 3.6681442260742188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19555087387561798,
      "logits/rejected": 0.07955916970968246,
      "logps/chosen": -376.40264892578125,
      "logps/rejected": -351.19970703125,
      "loss": 0.2089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2527568340301514,
      "rewards/margins": 2.115483045578003,
      "rewards/rejected": -3.368239641189575,
      "step": 8934
    },
    {
      "epoch": 1.169199312995829,
      "grad_norm": 13.334976196289062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15641799569129944,
      "logits/rejected": -0.13730548322200775,
      "logps/chosen": -702.4592895507812,
      "logps/rejected": -487.6960144042969,
      "loss": 0.5928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.187606692314148,
      "rewards/margins": 1.3158042430877686,
      "rewards/rejected": -2.503411293029785,
      "step": 8935
    },
    {
      "epoch": 1.1693301709331807,
      "grad_norm": 5.689432144165039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.059750206768512726,
      "logits/rejected": -0.3141423761844635,
      "logps/chosen": -451.6914367675781,
      "logps/rejected": -326.39813232421875,
      "loss": 0.3501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9530715942382812,
      "rewards/margins": 1.5307751893997192,
      "rewards/rejected": -2.48384690284729,
      "step": 8936
    },
    {
      "epoch": 1.1694610288705325,
      "grad_norm": 9.24917221069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32388901710510254,
      "logits/rejected": -0.18797513842582703,
      "logps/chosen": -591.0192260742188,
      "logps/rejected": -495.74713134765625,
      "loss": 0.4294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2042943835258484,
      "rewards/margins": 2.171173095703125,
      "rewards/rejected": -2.375467538833618,
      "step": 8937
    },
    {
      "epoch": 1.1695918868078843,
      "grad_norm": 7.396122932434082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06111142784357071,
      "logits/rejected": -0.14985226094722748,
      "logps/chosen": -383.0275573730469,
      "logps/rejected": -425.73443603515625,
      "loss": 0.3553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43593502044677734,
      "rewards/margins": 1.6073471307754517,
      "rewards/rejected": -2.0432820320129395,
      "step": 8938
    },
    {
      "epoch": 1.1697227447452359,
      "grad_norm": 10.229361534118652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4055894613265991,
      "logits/rejected": -0.4944153130054474,
      "logps/chosen": -567.1075439453125,
      "logps/rejected": -370.4951171875,
      "loss": 0.541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7722878456115723,
      "rewards/margins": 0.8794317245483398,
      "rewards/rejected": -2.651719570159912,
      "step": 8939
    },
    {
      "epoch": 1.1698536026825876,
      "grad_norm": 5.4539031982421875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.38661813735961914,
      "logits/rejected": 0.1969960480928421,
      "logps/chosen": -566.9177856445312,
      "logps/rejected": -463.91015625,
      "loss": 0.2628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2312333732843399,
      "rewards/margins": 1.6442935466766357,
      "rewards/rejected": -1.875527024269104,
      "step": 8940
    },
    {
      "epoch": 1.1699844606199394,
      "grad_norm": 5.296257972717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07683782279491425,
      "logits/rejected": -0.16251495480537415,
      "logps/chosen": -330.18963623046875,
      "logps/rejected": -468.7319641113281,
      "loss": 0.1775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6660407781600952,
      "rewards/margins": 2.181408405303955,
      "rewards/rejected": -2.84744930267334,
      "step": 8941
    },
    {
      "epoch": 1.1701153185572912,
      "grad_norm": 6.592857360839844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18366298079490662,
      "logits/rejected": -0.01529720425605774,
      "logps/chosen": -467.51434326171875,
      "logps/rejected": -479.13861083984375,
      "loss": 0.3043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6198775768280029,
      "rewards/margins": 1.8858835697174072,
      "rewards/rejected": -2.50576114654541,
      "step": 8942
    },
    {
      "epoch": 1.170246176494643,
      "grad_norm": 8.76384162902832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14088985323905945,
      "logits/rejected": -0.24369415640830994,
      "logps/chosen": -566.0297241210938,
      "logps/rejected": -693.796630859375,
      "loss": 0.415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.398851752281189,
      "rewards/margins": 1.5267179012298584,
      "rewards/rejected": -2.925569534301758,
      "step": 8943
    },
    {
      "epoch": 1.1703770344319948,
      "grad_norm": 6.504701614379883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19619038701057434,
      "logits/rejected": -0.0749107077717781,
      "logps/chosen": -474.40740966796875,
      "logps/rejected": -457.10723876953125,
      "loss": 0.3295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13566474616527557,
      "rewards/margins": 1.2741549015045166,
      "rewards/rejected": -1.4098199605941772,
      "step": 8944
    },
    {
      "epoch": 1.1705078923693466,
      "grad_norm": 6.419368743896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01027124747633934,
      "logits/rejected": -0.17332960665225983,
      "logps/chosen": -626.6153564453125,
      "logps/rejected": -434.4177551269531,
      "loss": 0.3592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9039438366889954,
      "rewards/margins": 1.1030170917510986,
      "rewards/rejected": -2.006960868835449,
      "step": 8945
    },
    {
      "epoch": 1.1706387503066984,
      "grad_norm": 7.556610107421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39115890860557556,
      "logits/rejected": -0.25772884488105774,
      "logps/chosen": -617.99560546875,
      "logps/rejected": -333.5729675292969,
      "loss": 0.5224,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.739864706993103,
      "rewards/margins": 1.816866397857666,
      "rewards/rejected": -2.5567312240600586,
      "step": 8946
    },
    {
      "epoch": 1.17076960824405,
      "grad_norm": 9.000112533569336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09026755392551422,
      "logits/rejected": 0.009874701499938965,
      "logps/chosen": -499.8699951171875,
      "logps/rejected": -513.2987670898438,
      "loss": 0.4191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5695309042930603,
      "rewards/margins": 1.420433759689331,
      "rewards/rejected": -1.9899646043777466,
      "step": 8947
    },
    {
      "epoch": 1.1709004661814018,
      "grad_norm": 9.73342514038086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24365639686584473,
      "logits/rejected": 0.23453004658222198,
      "logps/chosen": -454.01531982421875,
      "logps/rejected": -518.193359375,
      "loss": 0.6837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.597760796546936,
      "rewards/margins": 0.5123996734619141,
      "rewards/rejected": -1.11016047000885,
      "step": 8948
    },
    {
      "epoch": 1.1710313241187535,
      "grad_norm": 13.030861854553223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23872560262680054,
      "logits/rejected": -0.2667944133281708,
      "logps/chosen": -572.4627075195312,
      "logps/rejected": -537.9057006835938,
      "loss": 0.8101,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5890886783599854,
      "rewards/margins": 0.7824589014053345,
      "rewards/rejected": -2.3715474605560303,
      "step": 8949
    },
    {
      "epoch": 1.1711621820561053,
      "grad_norm": 4.020479202270508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17243847250938416,
      "logits/rejected": -0.14807745814323425,
      "logps/chosen": -427.2713317871094,
      "logps/rejected": -402.562744140625,
      "loss": 0.209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42072218656539917,
      "rewards/margins": 2.121711254119873,
      "rewards/rejected": -2.542433500289917,
      "step": 8950
    },
    {
      "epoch": 1.1712930399934571,
      "grad_norm": 10.369058609008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19319939613342285,
      "logits/rejected": -0.40061652660369873,
      "logps/chosen": -344.2836608886719,
      "logps/rejected": -333.7276611328125,
      "loss": 0.4618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2269575595855713,
      "rewards/margins": 1.8738608360290527,
      "rewards/rejected": -4.100818157196045,
      "step": 8951
    },
    {
      "epoch": 1.171423897930809,
      "grad_norm": 7.887402534484863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03152564540505409,
      "logits/rejected": -0.19968946278095245,
      "logps/chosen": -550.302978515625,
      "logps/rejected": -445.210205078125,
      "loss": 0.3821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.307667851448059,
      "rewards/margins": 1.2611840963363647,
      "rewards/rejected": -2.568851947784424,
      "step": 8952
    },
    {
      "epoch": 1.1715547558681607,
      "grad_norm": 4.890471935272217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25022387504577637,
      "logits/rejected": -0.21021795272827148,
      "logps/chosen": -323.6166687011719,
      "logps/rejected": -326.9532165527344,
      "loss": 0.3372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7567949295043945,
      "rewards/margins": 1.1060800552368164,
      "rewards/rejected": -1.862874984741211,
      "step": 8953
    },
    {
      "epoch": 1.1716856138055123,
      "grad_norm": 13.629464149475098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32233861088752747,
      "logits/rejected": -0.2393297255039215,
      "logps/chosen": -458.4223937988281,
      "logps/rejected": -377.2930908203125,
      "loss": 0.6457,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6040523052215576,
      "rewards/margins": 1.138134479522705,
      "rewards/rejected": -2.7421867847442627,
      "step": 8954
    },
    {
      "epoch": 1.171816471742864,
      "grad_norm": 10.96471118927002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007591679692268372,
      "logits/rejected": 0.18952181935310364,
      "logps/chosen": -361.2120056152344,
      "logps/rejected": -384.7680358886719,
      "loss": 0.621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2833409309387207,
      "rewards/margins": 0.6484243869781494,
      "rewards/rejected": -1.9317651987075806,
      "step": 8955
    },
    {
      "epoch": 1.1719473296802159,
      "grad_norm": 8.189827919006348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1604381799697876,
      "logits/rejected": -0.14577032625675201,
      "logps/chosen": -484.37890625,
      "logps/rejected": -658.8997802734375,
      "loss": 0.656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1723694801330566,
      "rewards/margins": 1.1587202548980713,
      "rewards/rejected": -2.331089973449707,
      "step": 8956
    },
    {
      "epoch": 1.1720781876175677,
      "grad_norm": 7.114649772644043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07657104730606079,
      "logits/rejected": -0.10861881077289581,
      "logps/chosen": -410.7628173828125,
      "logps/rejected": -539.0550537109375,
      "loss": 0.3608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4297468066215515,
      "rewards/margins": 1.0050684213638306,
      "rewards/rejected": -1.4348152875900269,
      "step": 8957
    },
    {
      "epoch": 1.1722090455549194,
      "grad_norm": 7.07727575302124,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1697716861963272,
      "logits/rejected": -0.24059973657131195,
      "logps/chosen": -384.02685546875,
      "logps/rejected": -485.9061584472656,
      "loss": 0.4547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2192500829696655,
      "rewards/margins": 1.0902481079101562,
      "rewards/rejected": -2.3094983100891113,
      "step": 8958
    },
    {
      "epoch": 1.1723399034922712,
      "grad_norm": 15.030933380126953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01523011177778244,
      "logits/rejected": -0.10922490060329437,
      "logps/chosen": -457.78399658203125,
      "logps/rejected": -433.5304870605469,
      "loss": 1.1671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0712220668792725,
      "rewards/margins": -0.04031461477279663,
      "rewards/rejected": -2.03090763092041,
      "step": 8959
    },
    {
      "epoch": 1.172470761429623,
      "grad_norm": 8.951963424682617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17512068152427673,
      "logits/rejected": -0.07514570653438568,
      "logps/chosen": -642.3367919921875,
      "logps/rejected": -506.92645263671875,
      "loss": 0.571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6324100494384766,
      "rewards/margins": 1.7573838233947754,
      "rewards/rejected": -2.389793872833252,
      "step": 8960
    },
    {
      "epoch": 1.1726016193669748,
      "grad_norm": 8.402064323425293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2274242788553238,
      "logits/rejected": -0.26651352643966675,
      "logps/chosen": -439.2530517578125,
      "logps/rejected": -477.73358154296875,
      "loss": 0.4334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8562172055244446,
      "rewards/margins": 0.9068679809570312,
      "rewards/rejected": -1.763085126876831,
      "step": 8961
    },
    {
      "epoch": 1.1727324773043266,
      "grad_norm": 5.860184192657471,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003622381016612053,
      "logits/rejected": -0.0874280035495758,
      "logps/chosen": -691.5806884765625,
      "logps/rejected": -409.8695983886719,
      "loss": 0.328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.353455126285553,
      "rewards/margins": 1.8592123985290527,
      "rewards/rejected": -2.212667465209961,
      "step": 8962
    },
    {
      "epoch": 1.1728633352416782,
      "grad_norm": 11.17052936553955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005591914057731628,
      "logits/rejected": 0.04386032372713089,
      "logps/chosen": -433.0299072265625,
      "logps/rejected": -449.8982849121094,
      "loss": 0.821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.243388295173645,
      "rewards/margins": 0.5739070177078247,
      "rewards/rejected": -1.8172953128814697,
      "step": 8963
    },
    {
      "epoch": 1.17299419317903,
      "grad_norm": 5.936987400054932,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24226315319538116,
      "logits/rejected": 0.3468531370162964,
      "logps/chosen": -570.5506591796875,
      "logps/rejected": -495.6581115722656,
      "loss": 0.326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24263188242912292,
      "rewards/margins": 1.3406453132629395,
      "rewards/rejected": -1.5832773447036743,
      "step": 8964
    },
    {
      "epoch": 1.1731250511163818,
      "grad_norm": 11.799010276794434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3744056820869446,
      "logits/rejected": -0.39706167578697205,
      "logps/chosen": -510.885009765625,
      "logps/rejected": -380.6426086425781,
      "loss": 0.837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9580312967300415,
      "rewards/margins": 0.7719197273254395,
      "rewards/rejected": -1.7299511432647705,
      "step": 8965
    },
    {
      "epoch": 1.1732559090537336,
      "grad_norm": 7.646667003631592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23256650567054749,
      "logits/rejected": -0.473690003156662,
      "logps/chosen": -615.2117919921875,
      "logps/rejected": -584.759521484375,
      "loss": 0.5062,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9682728052139282,
      "rewards/margins": 0.8705105185508728,
      "rewards/rejected": -1.8387832641601562,
      "step": 8966
    },
    {
      "epoch": 1.1733867669910854,
      "grad_norm": 4.749510288238525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02802763134241104,
      "logits/rejected": -0.025328733026981354,
      "logps/chosen": -507.9602355957031,
      "logps/rejected": -528.1909790039062,
      "loss": 0.322,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8193437457084656,
      "rewards/margins": 1.2782032489776611,
      "rewards/rejected": -2.0975470542907715,
      "step": 8967
    },
    {
      "epoch": 1.1735176249284371,
      "grad_norm": 7.573700904846191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08594518899917603,
      "logits/rejected": -0.056705113500356674,
      "logps/chosen": -703.3107299804688,
      "logps/rejected": -444.3196716308594,
      "loss": 0.4054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43555670976638794,
      "rewards/margins": 1.0992964506149292,
      "rewards/rejected": -1.5348533391952515,
      "step": 8968
    },
    {
      "epoch": 1.1736484828657887,
      "grad_norm": 10.26928424835205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0015046373009681702,
      "logits/rejected": -0.011054553091526031,
      "logps/chosen": -353.38525390625,
      "logps/rejected": -336.7944641113281,
      "loss": 0.6161,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5437290668487549,
      "rewards/margins": 1.4169588088989258,
      "rewards/rejected": -1.9606879949569702,
      "step": 8969
    },
    {
      "epoch": 1.1737793408031405,
      "grad_norm": 11.786985397338867,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06581567227840424,
      "logits/rejected": -0.04879327863454819,
      "logps/chosen": -507.28118896484375,
      "logps/rejected": -316.48980712890625,
      "loss": 0.6179,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6827049255371094,
      "rewards/margins": 1.1423344612121582,
      "rewards/rejected": -1.825039267539978,
      "step": 8970
    },
    {
      "epoch": 1.1739101987404923,
      "grad_norm": 6.674193859100342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2976585328578949,
      "logits/rejected": -0.3901488780975342,
      "logps/chosen": -411.3755798339844,
      "logps/rejected": -340.94012451171875,
      "loss": 0.309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.839491069316864,
      "rewards/margins": 1.399834156036377,
      "rewards/rejected": -2.2393252849578857,
      "step": 8971
    },
    {
      "epoch": 1.174041056677844,
      "grad_norm": 6.1153130531311035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.038240887224674225,
      "logits/rejected": -0.40373682975769043,
      "logps/chosen": -406.6972961425781,
      "logps/rejected": -438.47406005859375,
      "loss": 0.2781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3986161947250366,
      "rewards/margins": 1.7566496133804321,
      "rewards/rejected": -3.1552658081054688,
      "step": 8972
    },
    {
      "epoch": 1.1741719146151959,
      "grad_norm": 11.12799072265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08894241601228714,
      "logits/rejected": -0.08352956175804138,
      "logps/chosen": -510.94256591796875,
      "logps/rejected": -599.14111328125,
      "loss": 0.5414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.394757628440857,
      "rewards/margins": 1.0794790983200073,
      "rewards/rejected": -2.4742367267608643,
      "step": 8973
    },
    {
      "epoch": 1.1743027725525477,
      "grad_norm": 5.470124244689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22429487109184265,
      "logits/rejected": -0.406817227602005,
      "logps/chosen": -625.0613403320312,
      "logps/rejected": -587.719970703125,
      "loss": 0.451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05074883997440338,
      "rewards/margins": 1.40703547000885,
      "rewards/rejected": -1.356286644935608,
      "step": 8974
    },
    {
      "epoch": 1.1744336304898995,
      "grad_norm": 10.244589805603027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29649218916893005,
      "logits/rejected": -0.125288188457489,
      "logps/chosen": -434.1839599609375,
      "logps/rejected": -486.2735290527344,
      "loss": 0.7317,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4859791994094849,
      "rewards/margins": 0.24668198823928833,
      "rewards/rejected": -1.7326611280441284,
      "step": 8975
    },
    {
      "epoch": 1.1745644884272513,
      "grad_norm": 5.139639377593994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12545408308506012,
      "logits/rejected": -0.015274377539753914,
      "logps/chosen": -432.6809997558594,
      "logps/rejected": -506.07440185546875,
      "loss": 0.2492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.022899866104126,
      "rewards/margins": 1.5511687994003296,
      "rewards/rejected": -2.574068784713745,
      "step": 8976
    },
    {
      "epoch": 1.174695346364603,
      "grad_norm": 11.245978355407715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18288740515708923,
      "logits/rejected": -0.3789272606372833,
      "logps/chosen": -358.007080078125,
      "logps/rejected": -366.9570007324219,
      "loss": 0.7177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1957697868347168,
      "rewards/margins": 0.8859884738922119,
      "rewards/rejected": -2.0817580223083496,
      "step": 8977
    },
    {
      "epoch": 1.1748262043019546,
      "grad_norm": 2.161642074584961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26922670006752014,
      "logits/rejected": -0.3149225115776062,
      "logps/chosen": -415.5228271484375,
      "logps/rejected": -371.84619140625,
      "loss": 0.11,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7007524967193604,
      "rewards/margins": 2.629042625427246,
      "rewards/rejected": -3.3297951221466064,
      "step": 8978
    },
    {
      "epoch": 1.1749570622393064,
      "grad_norm": 8.863592147827148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06550471484661102,
      "logits/rejected": -0.08247649669647217,
      "logps/chosen": -321.0915222167969,
      "logps/rejected": -302.0735778808594,
      "loss": 0.6081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3139162063598633,
      "rewards/margins": 0.34327906370162964,
      "rewards/rejected": -1.6571953296661377,
      "step": 8979
    },
    {
      "epoch": 1.1750879201766582,
      "grad_norm": 8.583780288696289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15580858290195465,
      "logits/rejected": 0.18545149266719818,
      "logps/chosen": -358.28857421875,
      "logps/rejected": -447.92645263671875,
      "loss": 0.5453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6840442419052124,
      "rewards/margins": 1.8749150037765503,
      "rewards/rejected": -2.5589592456817627,
      "step": 8980
    },
    {
      "epoch": 1.17521877811401,
      "grad_norm": 12.393691062927246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12180734425783157,
      "logits/rejected": -0.16859793663024902,
      "logps/chosen": -314.3711242675781,
      "logps/rejected": -297.3881530761719,
      "loss": 1.1134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0062673091888428,
      "rewards/margins": -0.3399728536605835,
      "rewards/rejected": -0.6662945747375488,
      "step": 8981
    },
    {
      "epoch": 1.1753496360513618,
      "grad_norm": 6.743036270141602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19541729986667633,
      "logits/rejected": -0.24274909496307373,
      "logps/chosen": -393.80755615234375,
      "logps/rejected": -291.1690673828125,
      "loss": 0.4026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22429655492305756,
      "rewards/margins": 1.7574434280395508,
      "rewards/rejected": -1.533146858215332,
      "step": 8982
    },
    {
      "epoch": 1.1754804939887136,
      "grad_norm": 5.707071304321289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20610906183719635,
      "logits/rejected": -0.10761101543903351,
      "logps/chosen": -363.49322509765625,
      "logps/rejected": -445.1316833496094,
      "loss": 0.306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8262546062469482,
      "rewards/margins": 1.623266577720642,
      "rewards/rejected": -3.449521064758301,
      "step": 8983
    },
    {
      "epoch": 1.1756113519260654,
      "grad_norm": 9.457605361938477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002579338848590851,
      "logits/rejected": 0.10236835479736328,
      "logps/chosen": -377.5477294921875,
      "logps/rejected": -411.0076904296875,
      "loss": 0.6663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6905558109283447,
      "rewards/margins": 0.6158121824264526,
      "rewards/rejected": -1.3063678741455078,
      "step": 8984
    },
    {
      "epoch": 1.175742209863417,
      "grad_norm": 6.057209491729736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010700628161430359,
      "logits/rejected": -0.2931842803955078,
      "logps/chosen": -472.90509033203125,
      "logps/rejected": -602.992919921875,
      "loss": 0.3922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47169458866119385,
      "rewards/margins": 1.6902559995651245,
      "rewards/rejected": -2.1619508266448975,
      "step": 8985
    },
    {
      "epoch": 1.1758730678007687,
      "grad_norm": 7.1018595695495605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012011770159006119,
      "logits/rejected": -0.13846614956855774,
      "logps/chosen": -365.1041259765625,
      "logps/rejected": -412.55523681640625,
      "loss": 0.4283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7044050693511963,
      "rewards/margins": 1.3528084754943848,
      "rewards/rejected": -3.057213306427002,
      "step": 8986
    },
    {
      "epoch": 1.1760039257381205,
      "grad_norm": 10.126749992370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13147982954978943,
      "logits/rejected": -0.16450054943561554,
      "logps/chosen": -557.3720703125,
      "logps/rejected": -445.13153076171875,
      "loss": 0.7879,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.888733446598053,
      "rewards/margins": 0.2535920739173889,
      "rewards/rejected": -1.142325520515442,
      "step": 8987
    },
    {
      "epoch": 1.1761347836754723,
      "grad_norm": 6.883511543273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007031694054603577,
      "logits/rejected": -0.051393672823905945,
      "logps/chosen": -424.5956115722656,
      "logps/rejected": -643.27734375,
      "loss": 0.3368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2961183786392212,
      "rewards/margins": 1.4165414571762085,
      "rewards/rejected": -1.7126599550247192,
      "step": 8988
    },
    {
      "epoch": 1.176265641612824,
      "grad_norm": 6.920693874359131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2951173782348633,
      "logits/rejected": 0.2935003638267517,
      "logps/chosen": -367.09600830078125,
      "logps/rejected": -436.32415771484375,
      "loss": 0.4237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5210176706314087,
      "rewards/margins": 1.3621010780334473,
      "rewards/rejected": -1.8831186294555664,
      "step": 8989
    },
    {
      "epoch": 1.176396499550176,
      "grad_norm": 9.437151908874512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03830612823367119,
      "logits/rejected": -0.2460901439189911,
      "logps/chosen": -431.55615234375,
      "logps/rejected": -507.82830810546875,
      "loss": 0.346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21617695689201355,
      "rewards/margins": 2.7910614013671875,
      "rewards/rejected": -2.5748841762542725,
      "step": 8990
    },
    {
      "epoch": 1.1765273574875277,
      "grad_norm": 7.206657409667969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013247653841972351,
      "logits/rejected": 0.052299946546554565,
      "logps/chosen": -321.58245849609375,
      "logps/rejected": -502.2073669433594,
      "loss": 0.4886,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3930917978286743,
      "rewards/margins": 1.0438199043273926,
      "rewards/rejected": -2.4369115829467773,
      "step": 8991
    },
    {
      "epoch": 1.1766582154248795,
      "grad_norm": 6.421928882598877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23292693495750427,
      "logits/rejected": 0.04719569534063339,
      "logps/chosen": -413.86224365234375,
      "logps/rejected": -530.279052734375,
      "loss": 0.4699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29492032527923584,
      "rewards/margins": 0.793984591960907,
      "rewards/rejected": -1.088904857635498,
      "step": 8992
    },
    {
      "epoch": 1.176789073362231,
      "grad_norm": 7.63463020324707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006158499978482723,
      "logits/rejected": -0.26836562156677246,
      "logps/chosen": -407.7847900390625,
      "logps/rejected": -593.6891479492188,
      "loss": 0.5132,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8327279686927795,
      "rewards/margins": 1.2262367010116577,
      "rewards/rejected": -2.058964729309082,
      "step": 8993
    },
    {
      "epoch": 1.1769199312995828,
      "grad_norm": 7.8362040519714355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09987500309944153,
      "logits/rejected": 0.056394312530756,
      "logps/chosen": -326.99200439453125,
      "logps/rejected": -303.576904296875,
      "loss": 0.5598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8878363370895386,
      "rewards/margins": 0.5029683709144592,
      "rewards/rejected": -1.3908047676086426,
      "step": 8994
    },
    {
      "epoch": 1.1770507892369346,
      "grad_norm": 8.48058795928955,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1946997344493866,
      "logits/rejected": 0.07785729318857193,
      "logps/chosen": -629.5921020507812,
      "logps/rejected": -542.4660034179688,
      "loss": 0.5744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03079316020011902,
      "rewards/margins": 1.2129569053649902,
      "rewards/rejected": -1.2437502145767212,
      "step": 8995
    },
    {
      "epoch": 1.1771816471742864,
      "grad_norm": 5.8012003898620605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09292027354240417,
      "logits/rejected": 0.04384507238864899,
      "logps/chosen": -539.9032592773438,
      "logps/rejected": -524.8286743164062,
      "loss": 0.3317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1063644289970398,
      "rewards/margins": 1.414053201675415,
      "rewards/rejected": -1.5204176902770996,
      "step": 8996
    },
    {
      "epoch": 1.1773125051116382,
      "grad_norm": 8.413849830627441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25592148303985596,
      "logits/rejected": -0.018956532701849937,
      "logps/chosen": -642.4354858398438,
      "logps/rejected": -455.6379699707031,
      "loss": 0.4487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3007015585899353,
      "rewards/margins": 1.9035999774932861,
      "rewards/rejected": -1.6028984785079956,
      "step": 8997
    },
    {
      "epoch": 1.17744336304899,
      "grad_norm": 7.478333473205566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0892275720834732,
      "logits/rejected": -0.1746515929698944,
      "logps/chosen": -642.9788818359375,
      "logps/rejected": -383.4898681640625,
      "loss": 0.4596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02793104574084282,
      "rewards/margins": 1.2723380327224731,
      "rewards/rejected": -1.244406819343567,
      "step": 8998
    },
    {
      "epoch": 1.1775742209863418,
      "grad_norm": 3.5456764698028564,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04906640574336052,
      "logits/rejected": -0.08619840443134308,
      "logps/chosen": -508.15032958984375,
      "logps/rejected": -339.9015808105469,
      "loss": 0.2029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.026427462697029114,
      "rewards/margins": 1.8977398872375488,
      "rewards/rejected": -1.924167275428772,
      "step": 8999
    },
    {
      "epoch": 1.1777050789236934,
      "grad_norm": 10.556015014648438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041879087686538696,
      "logits/rejected": -0.056683506816625595,
      "logps/chosen": -390.7892150878906,
      "logps/rejected": -436.4224853515625,
      "loss": 0.7655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7429453134536743,
      "rewards/margins": 0.20003223419189453,
      "rewards/rejected": -0.9429774880409241,
      "step": 9000
    },
    {
      "epoch": 1.1778359368610452,
      "grad_norm": 5.167228698730469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.369914710521698,
      "logits/rejected": 0.15281620621681213,
      "logps/chosen": -493.49847412109375,
      "logps/rejected": -519.6005249023438,
      "loss": 0.2282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03878059983253479,
      "rewards/margins": 1.6654540300369263,
      "rewards/rejected": -1.7042348384857178,
      "step": 9001
    },
    {
      "epoch": 1.177966794798397,
      "grad_norm": 7.922078609466553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2232966125011444,
      "logits/rejected": -0.20659375190734863,
      "logps/chosen": -467.6525573730469,
      "logps/rejected": -383.6372985839844,
      "loss": 0.4729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5735113620758057,
      "rewards/margins": 1.1677910089492798,
      "rewards/rejected": -1.741302251815796,
      "step": 9002
    },
    {
      "epoch": 1.1780976527357487,
      "grad_norm": 6.637890815734863,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08011207729578018,
      "logits/rejected": 0.17201706767082214,
      "logps/chosen": -487.87310791015625,
      "logps/rejected": -458.92913818359375,
      "loss": 0.3109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.32248151302337646,
      "rewards/margins": 1.3895392417907715,
      "rewards/rejected": -1.7120208740234375,
      "step": 9003
    },
    {
      "epoch": 1.1782285106731005,
      "grad_norm": 6.519869327545166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09296610206365585,
      "logits/rejected": 0.09223880618810654,
      "logps/chosen": -485.3277282714844,
      "logps/rejected": -516.0279541015625,
      "loss": 0.3604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2918505668640137,
      "rewards/margins": 1.554266095161438,
      "rewards/rejected": -2.846116781234741,
      "step": 9004
    },
    {
      "epoch": 1.1783593686104523,
      "grad_norm": 9.114083290100098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2707027494907379,
      "logits/rejected": -0.3074454665184021,
      "logps/chosen": -643.956787109375,
      "logps/rejected": -520.179931640625,
      "loss": 0.537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5549547076225281,
      "rewards/margins": 1.1089693307876587,
      "rewards/rejected": -1.6639240980148315,
      "step": 9005
    },
    {
      "epoch": 1.178490226547804,
      "grad_norm": 12.179815292358398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11892607808113098,
      "logits/rejected": -0.07777240127325058,
      "logps/chosen": -581.90380859375,
      "logps/rejected": -560.8634033203125,
      "loss": 0.7588,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3686954379081726,
      "rewards/margins": 0.45368844270706177,
      "rewards/rejected": -0.8223838210105896,
      "step": 9006
    },
    {
      "epoch": 1.178621084485156,
      "grad_norm": 6.461392879486084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11813969910144806,
      "logits/rejected": -0.09089470654726028,
      "logps/chosen": -511.5838623046875,
      "logps/rejected": -597.2716674804688,
      "loss": 0.2719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08456233143806458,
      "rewards/margins": 1.714478611946106,
      "rewards/rejected": -1.6299164295196533,
      "step": 9007
    },
    {
      "epoch": 1.1787519424225075,
      "grad_norm": 11.066888809204102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.001160264015197754,
      "logits/rejected": -0.14428022503852844,
      "logps/chosen": -461.1697082519531,
      "logps/rejected": -428.7815856933594,
      "loss": 0.5472,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.136827290058136,
      "rewards/margins": 1.134124994277954,
      "rewards/rejected": -1.2709523439407349,
      "step": 9008
    },
    {
      "epoch": 1.1788828003598593,
      "grad_norm": 7.6845502853393555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10620705038309097,
      "logits/rejected": 0.09692278504371643,
      "logps/chosen": -385.3645324707031,
      "logps/rejected": -391.3717041015625,
      "loss": 0.4716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9100496172904968,
      "rewards/margins": 1.344167947769165,
      "rewards/rejected": -2.2542176246643066,
      "step": 9009
    },
    {
      "epoch": 1.179013658297211,
      "grad_norm": 4.9180216789245605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2167971134185791,
      "logits/rejected": 0.18931053578853607,
      "logps/chosen": -427.89691162109375,
      "logps/rejected": -502.6042175292969,
      "loss": 0.2316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2103549838066101,
      "rewards/margins": 2.228114604949951,
      "rewards/rejected": -2.0177595615386963,
      "step": 9010
    },
    {
      "epoch": 1.1791445162345628,
      "grad_norm": 6.238917350769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3159869909286499,
      "logits/rejected": -0.4053376615047455,
      "logps/chosen": -474.8124084472656,
      "logps/rejected": -325.1463317871094,
      "loss": 0.3964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7589154839515686,
      "rewards/margins": 1.4538452625274658,
      "rewards/rejected": -2.2127609252929688,
      "step": 9011
    },
    {
      "epoch": 1.1792753741719146,
      "grad_norm": 9.05528450012207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.29675203561782837,
      "logits/rejected": -0.07871876657009125,
      "logps/chosen": -476.10528564453125,
      "logps/rejected": -445.3513488769531,
      "loss": 0.5283,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22201921045780182,
      "rewards/margins": 0.9283219575881958,
      "rewards/rejected": -1.150341272354126,
      "step": 9012
    },
    {
      "epoch": 1.1794062321092664,
      "grad_norm": 5.727935314178467,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015237417072057724,
      "logits/rejected": 0.016440097242593765,
      "logps/chosen": -484.48944091796875,
      "logps/rejected": -415.27642822265625,
      "loss": 0.2633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10880821943283081,
      "rewards/margins": 1.702984094619751,
      "rewards/rejected": -1.811792254447937,
      "step": 9013
    },
    {
      "epoch": 1.1795370900466182,
      "grad_norm": 6.598636627197266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14634442329406738,
      "logits/rejected": -0.2051677256822586,
      "logps/chosen": -326.1715393066406,
      "logps/rejected": -361.2222595214844,
      "loss": 0.4583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7202492952346802,
      "rewards/margins": 1.3185538053512573,
      "rewards/rejected": -2.0388028621673584,
      "step": 9014
    },
    {
      "epoch": 1.1796679479839698,
      "grad_norm": 7.319973468780518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010682184249162674,
      "logits/rejected": 0.017664674669504166,
      "logps/chosen": -371.9958190917969,
      "logps/rejected": -372.7956237792969,
      "loss": 0.3729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0436537265777588,
      "rewards/margins": 1.2026797533035278,
      "rewards/rejected": -2.246333360671997,
      "step": 9015
    },
    {
      "epoch": 1.1797988059213216,
      "grad_norm": 15.222575187683105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2971995770931244,
      "logits/rejected": -0.11722436547279358,
      "logps/chosen": -383.7550354003906,
      "logps/rejected": -501.4062805175781,
      "loss": 1.2607,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0349888801574707,
      "rewards/margins": -0.3825412690639496,
      "rewards/rejected": -1.6524474620819092,
      "step": 9016
    },
    {
      "epoch": 1.1799296638586734,
      "grad_norm": 7.837031364440918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003292553126811981,
      "logits/rejected": -0.04428606480360031,
      "logps/chosen": -277.92425537109375,
      "logps/rejected": -275.2209167480469,
      "loss": 0.5318,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19825343787670135,
      "rewards/margins": 0.7675685286521912,
      "rewards/rejected": -0.9658220410346985,
      "step": 9017
    },
    {
      "epoch": 1.1800605217960252,
      "grad_norm": 7.992967128753662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2687421143054962,
      "logits/rejected": -0.3336288332939148,
      "logps/chosen": -501.08709716796875,
      "logps/rejected": -347.125244140625,
      "loss": 0.5926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5287075042724609,
      "rewards/margins": 0.8361663818359375,
      "rewards/rejected": -1.3648738861083984,
      "step": 9018
    },
    {
      "epoch": 1.180191379733377,
      "grad_norm": 8.453856468200684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.258612722158432,
      "logits/rejected": -0.21686969697475433,
      "logps/chosen": -523.029541015625,
      "logps/rejected": -391.9640197753906,
      "loss": 0.4137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8716728091239929,
      "rewards/margins": 0.8813852071762085,
      "rewards/rejected": -1.7530580759048462,
      "step": 9019
    },
    {
      "epoch": 1.1803222376707287,
      "grad_norm": 4.694998741149902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.26257041096687317,
      "logits/rejected": -0.13750939071178436,
      "logps/chosen": -434.7137756347656,
      "logps/rejected": -364.3056335449219,
      "loss": 0.3153,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4935096800327301,
      "rewards/margins": 1.3979887962341309,
      "rewards/rejected": -1.8914984464645386,
      "step": 9020
    },
    {
      "epoch": 1.1804530956080805,
      "grad_norm": 7.822259902954102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21501019597053528,
      "logits/rejected": -0.11610403656959534,
      "logps/chosen": -429.3685607910156,
      "logps/rejected": -402.5720520019531,
      "loss": 0.6009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8273731470108032,
      "rewards/margins": 0.9354470372200012,
      "rewards/rejected": -1.7628202438354492,
      "step": 9021
    },
    {
      "epoch": 1.1805839535454323,
      "grad_norm": 5.734870433807373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10554257035255432,
      "logits/rejected": 0.1202068030834198,
      "logps/chosen": -401.12176513671875,
      "logps/rejected": -364.6774597167969,
      "loss": 0.387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6597614288330078,
      "rewards/margins": 1.5102866888046265,
      "rewards/rejected": -2.170048236846924,
      "step": 9022
    },
    {
      "epoch": 1.1807148114827841,
      "grad_norm": 5.921420097351074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10210394859313965,
      "logits/rejected": -0.02120698057115078,
      "logps/chosen": -495.5364990234375,
      "logps/rejected": -599.1788330078125,
      "loss": 0.2676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32800617814064026,
      "rewards/margins": 2.1126837730407715,
      "rewards/rejected": -1.784677505493164,
      "step": 9023
    },
    {
      "epoch": 1.1808456694201357,
      "grad_norm": 8.294687271118164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3483280539512634,
      "logits/rejected": 0.26100555062294006,
      "logps/chosen": -491.55169677734375,
      "logps/rejected": -512.2713623046875,
      "loss": 0.4841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03596724569797516,
      "rewards/margins": 1.061202883720398,
      "rewards/rejected": -1.097170114517212,
      "step": 9024
    },
    {
      "epoch": 1.1809765273574875,
      "grad_norm": 16.68971824645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03957824036478996,
      "logits/rejected": -0.21578267216682434,
      "logps/chosen": -707.4684448242188,
      "logps/rejected": -395.61138916015625,
      "loss": 1.0049,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9425254464149475,
      "rewards/margins": 0.415639728307724,
      "rewards/rejected": -1.3581650257110596,
      "step": 9025
    },
    {
      "epoch": 1.1811073852948393,
      "grad_norm": 6.12841272354126,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07155345380306244,
      "logits/rejected": -0.06743865460157394,
      "logps/chosen": -346.0199890136719,
      "logps/rejected": -490.67120361328125,
      "loss": 0.3705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.892054557800293,
      "rewards/margins": 1.4387155771255493,
      "rewards/rejected": -2.3307700157165527,
      "step": 9026
    },
    {
      "epoch": 1.181238243232191,
      "grad_norm": 11.889547348022461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2833704948425293,
      "logits/rejected": -0.19922083616256714,
      "logps/chosen": -406.3213195800781,
      "logps/rejected": -348.3023681640625,
      "loss": 0.7002,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8658711314201355,
      "rewards/margins": 0.6154712438583374,
      "rewards/rejected": -1.4813426733016968,
      "step": 9027
    },
    {
      "epoch": 1.1813691011695429,
      "grad_norm": 7.402679920196533,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11503173410892487,
      "logits/rejected": -0.28007832169532776,
      "logps/chosen": -509.4154052734375,
      "logps/rejected": -467.27508544921875,
      "loss": 0.3482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35298240184783936,
      "rewards/margins": 1.3497189283370972,
      "rewards/rejected": -0.9967365264892578,
      "step": 9028
    },
    {
      "epoch": 1.1814999591068946,
      "grad_norm": 6.496361255645752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15009447932243347,
      "logits/rejected": -0.08658988028764725,
      "logps/chosen": -468.24530029296875,
      "logps/rejected": -425.0708312988281,
      "loss": 0.3745,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7232702970504761,
      "rewards/margins": 1.6399973630905151,
      "rewards/rejected": -2.363267660140991,
      "step": 9029
    },
    {
      "epoch": 1.1816308170442462,
      "grad_norm": 9.258808135986328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1883634328842163,
      "logits/rejected": 0.20262618362903595,
      "logps/chosen": -366.4355163574219,
      "logps/rejected": -432.2962951660156,
      "loss": 0.7036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25056901574134827,
      "rewards/margins": 0.4438798129558563,
      "rewards/rejected": -0.6944488883018494,
      "step": 9030
    },
    {
      "epoch": 1.181761674981598,
      "grad_norm": 6.816600799560547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.243498295545578,
      "logits/rejected": -0.23214739561080933,
      "logps/chosen": -494.6458435058594,
      "logps/rejected": -419.224609375,
      "loss": 0.3683,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5191035866737366,
      "rewards/margins": 1.2967592477798462,
      "rewards/rejected": -1.815863013267517,
      "step": 9031
    },
    {
      "epoch": 1.1818925329189498,
      "grad_norm": 7.855220317840576,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14266103506088257,
      "logits/rejected": -0.370657742023468,
      "logps/chosen": -510.1217041015625,
      "logps/rejected": -540.6316528320312,
      "loss": 0.4293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4957062005996704,
      "rewards/margins": 1.5837440490722656,
      "rewards/rejected": -2.0794501304626465,
      "step": 9032
    },
    {
      "epoch": 1.1820233908563016,
      "grad_norm": 12.702500343322754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05193556845188141,
      "logits/rejected": -0.08178698271512985,
      "logps/chosen": -449.65386962890625,
      "logps/rejected": -421.23443603515625,
      "loss": 1.1576,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8775949478149414,
      "rewards/margins": -0.19656237959861755,
      "rewards/rejected": -0.6810325384140015,
      "step": 9033
    },
    {
      "epoch": 1.1821542487936534,
      "grad_norm": 5.598853588104248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2470969706773758,
      "logits/rejected": -0.2078295201063156,
      "logps/chosen": -264.04571533203125,
      "logps/rejected": -334.13446044921875,
      "loss": 0.4193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.233474850654602,
      "rewards/margins": 1.4076749086380005,
      "rewards/rejected": -2.6411499977111816,
      "step": 9034
    },
    {
      "epoch": 1.1822851067310052,
      "grad_norm": 7.178747177124023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03609194979071617,
      "logits/rejected": -0.04839498549699783,
      "logps/chosen": -581.3380126953125,
      "logps/rejected": -382.2825622558594,
      "loss": 0.4837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43930232524871826,
      "rewards/margins": 1.278030514717102,
      "rewards/rejected": -1.7173327207565308,
      "step": 9035
    },
    {
      "epoch": 1.182415964668357,
      "grad_norm": 8.546791076660156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07964134961366653,
      "logits/rejected": 0.024702489376068115,
      "logps/chosen": -511.5459899902344,
      "logps/rejected": -439.172119140625,
      "loss": 0.4877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3193734884262085,
      "rewards/margins": 0.7455544471740723,
      "rewards/rejected": -1.0649279356002808,
      "step": 9036
    },
    {
      "epoch": 1.1825468226057088,
      "grad_norm": 7.724174976348877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0020368918776512146,
      "logits/rejected": 0.00695975124835968,
      "logps/chosen": -459.1410217285156,
      "logps/rejected": -458.2574462890625,
      "loss": 0.5786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9929018020629883,
      "rewards/margins": 0.5778356790542603,
      "rewards/rejected": -1.570737600326538,
      "step": 9037
    },
    {
      "epoch": 1.1826776805430605,
      "grad_norm": 14.870967864990234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18537232279777527,
      "logits/rejected": -0.053413379937410355,
      "logps/chosen": -496.2720642089844,
      "logps/rejected": -430.0816955566406,
      "loss": 0.346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8244230151176453,
      "rewards/margins": 1.617323398590088,
      "rewards/rejected": -2.441746473312378,
      "step": 9038
    },
    {
      "epoch": 1.1828085384804121,
      "grad_norm": 8.904684066772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04228638857603073,
      "logits/rejected": -0.023940905928611755,
      "logps/chosen": -496.3521728515625,
      "logps/rejected": -492.8374938964844,
      "loss": 0.5797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1028330326080322,
      "rewards/margins": 1.769539713859558,
      "rewards/rejected": -2.8723723888397217,
      "step": 9039
    },
    {
      "epoch": 1.182939396417764,
      "grad_norm": 5.970340728759766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1838693767786026,
      "logits/rejected": -0.12610873579978943,
      "logps/chosen": -634.8168334960938,
      "logps/rejected": -587.152099609375,
      "loss": 0.3103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5297117233276367,
      "rewards/margins": 1.5133305788040161,
      "rewards/rejected": -2.0430424213409424,
      "step": 9040
    },
    {
      "epoch": 1.1830702543551157,
      "grad_norm": 10.767820358276367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18412242829799652,
      "logits/rejected": -0.23572593927383423,
      "logps/chosen": -438.5324401855469,
      "logps/rejected": -497.55584716796875,
      "loss": 0.7285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8597049117088318,
      "rewards/margins": 0.8416255116462708,
      "rewards/rejected": -1.701330542564392,
      "step": 9041
    },
    {
      "epoch": 1.1832011122924675,
      "grad_norm": 10.161298751831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1802002340555191,
      "logits/rejected": -0.17328916490077972,
      "logps/chosen": -514.1636352539062,
      "logps/rejected": -465.0755920410156,
      "loss": 0.524,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7826330661773682,
      "rewards/margins": 1.441589117050171,
      "rewards/rejected": -2.224222183227539,
      "step": 9042
    },
    {
      "epoch": 1.1833319702298193,
      "grad_norm": 6.578789710998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03691381961107254,
      "logits/rejected": 0.02455686777830124,
      "logps/chosen": -422.51934814453125,
      "logps/rejected": -476.0061950683594,
      "loss": 0.3682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4534895122051239,
      "rewards/margins": 1.6603434085845947,
      "rewards/rejected": -2.113832712173462,
      "step": 9043
    },
    {
      "epoch": 1.183462828167171,
      "grad_norm": 8.913819313049316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022470364347100258,
      "logits/rejected": -0.029024390503764153,
      "logps/chosen": -493.53948974609375,
      "logps/rejected": -461.7352294921875,
      "loss": 0.6028,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8818886280059814,
      "rewards/margins": 0.6484174728393555,
      "rewards/rejected": -1.5303059816360474,
      "step": 9044
    },
    {
      "epoch": 1.1835936861045229,
      "grad_norm": 5.178323745727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2900511622428894,
      "logits/rejected": -0.34695011377334595,
      "logps/chosen": -408.3141174316406,
      "logps/rejected": -456.53363037109375,
      "loss": 0.2526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32286375761032104,
      "rewards/margins": 1.8234823942184448,
      "rewards/rejected": -2.1463465690612793,
      "step": 9045
    },
    {
      "epoch": 1.1837245440418744,
      "grad_norm": 6.68455696105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18188050389289856,
      "logits/rejected": -0.032976411283016205,
      "logps/chosen": -398.6219177246094,
      "logps/rejected": -456.57269287109375,
      "loss": 0.3632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5828450918197632,
      "rewards/margins": 1.2487890720367432,
      "rewards/rejected": -2.831634044647217,
      "step": 9046
    },
    {
      "epoch": 1.1838554019792262,
      "grad_norm": 8.222058296203613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0232650525867939,
      "logits/rejected": -0.08782664686441422,
      "logps/chosen": -357.8860168457031,
      "logps/rejected": -320.76458740234375,
      "loss": 0.766,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1140565872192383,
      "rewards/margins": 0.9720653891563416,
      "rewards/rejected": -2.0861217975616455,
      "step": 9047
    },
    {
      "epoch": 1.183986259916578,
      "grad_norm": 12.057539939880371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33533206582069397,
      "logits/rejected": -0.21649610996246338,
      "logps/chosen": -469.38909912109375,
      "logps/rejected": -503.5592956542969,
      "loss": 0.852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7630233764648438,
      "rewards/margins": 0.32665789127349854,
      "rewards/rejected": -1.0896812677383423,
      "step": 9048
    },
    {
      "epoch": 1.1841171178539298,
      "grad_norm": 3.8919296264648438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20306971669197083,
      "logits/rejected": 0.23476463556289673,
      "logps/chosen": -534.1654663085938,
      "logps/rejected": -557.1376953125,
      "loss": 0.2252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2629114091396332,
      "rewards/margins": 1.9380220174789429,
      "rewards/rejected": -2.2009334564208984,
      "step": 9049
    },
    {
      "epoch": 1.1842479757912816,
      "grad_norm": 10.72928237915039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14386260509490967,
      "logits/rejected": -0.007787999697029591,
      "logps/chosen": -433.67474365234375,
      "logps/rejected": -394.3570556640625,
      "loss": 1.0063,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8185758590698242,
      "rewards/margins": 0.47956717014312744,
      "rewards/rejected": -1.2981430292129517,
      "step": 9050
    },
    {
      "epoch": 1.1843788337286334,
      "grad_norm": 10.676043510437012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01897636242210865,
      "logits/rejected": -0.037857525050640106,
      "logps/chosen": -367.5677490234375,
      "logps/rejected": -493.72357177734375,
      "loss": 0.7631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3359472453594208,
      "rewards/margins": 1.42812180519104,
      "rewards/rejected": -1.7640691995620728,
      "step": 9051
    },
    {
      "epoch": 1.1845096916659852,
      "grad_norm": 8.091734886169434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020180467516183853,
      "logits/rejected": -0.13068322837352753,
      "logps/chosen": -393.7796630859375,
      "logps/rejected": -497.421875,
      "loss": 0.2236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6741598844528198,
      "rewards/margins": 2.0849733352661133,
      "rewards/rejected": -2.7591333389282227,
      "step": 9052
    },
    {
      "epoch": 1.184640549603337,
      "grad_norm": 7.230746269226074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05618421733379364,
      "logits/rejected": -0.1053195521235466,
      "logps/chosen": -505.7662353515625,
      "logps/rejected": -584.1829833984375,
      "loss": 0.4155,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43455278873443604,
      "rewards/margins": 1.056072473526001,
      "rewards/rejected": -1.4906251430511475,
      "step": 9053
    },
    {
      "epoch": 1.1847714075406885,
      "grad_norm": 46.739315032958984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0006319433450698853,
      "logits/rejected": -0.1763007640838623,
      "logps/chosen": -464.86529541015625,
      "logps/rejected": -456.28302001953125,
      "loss": 0.9256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4794557094573975,
      "rewards/margins": 0.9916021227836609,
      "rewards/rejected": -2.471057891845703,
      "step": 9054
    },
    {
      "epoch": 1.1849022654780403,
      "grad_norm": 8.465721130371094,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17233966290950775,
      "logits/rejected": 0.16019347310066223,
      "logps/chosen": -546.0490112304688,
      "logps/rejected": -562.2835693359375,
      "loss": 0.5352,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0515495538711548,
      "rewards/margins": 0.9758339524269104,
      "rewards/rejected": -2.02738356590271,
      "step": 9055
    },
    {
      "epoch": 1.1850331234153921,
      "grad_norm": 11.429755210876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29970812797546387,
      "logits/rejected": -0.37244585156440735,
      "logps/chosen": -441.0910339355469,
      "logps/rejected": -486.86932373046875,
      "loss": 0.6744,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1875312328338623,
      "rewards/margins": 0.9116537570953369,
      "rewards/rejected": -2.099184989929199,
      "step": 9056
    },
    {
      "epoch": 1.185163981352744,
      "grad_norm": 6.377712726593018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02889493852853775,
      "logits/rejected": -0.4629688560962677,
      "logps/chosen": -478.21148681640625,
      "logps/rejected": -467.2482604980469,
      "loss": 0.1674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7446434497833252,
      "rewards/margins": 2.796264410018921,
      "rewards/rejected": -3.540907859802246,
      "step": 9057
    },
    {
      "epoch": 1.1852948392900957,
      "grad_norm": 8.408500671386719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31319665908813477,
      "logits/rejected": -0.1781618893146515,
      "logps/chosen": -403.06890869140625,
      "logps/rejected": -501.8359375,
      "loss": 0.4943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9682608842849731,
      "rewards/margins": 0.6398867964744568,
      "rewards/rejected": -1.6081477403640747,
      "step": 9058
    },
    {
      "epoch": 1.1854256972274475,
      "grad_norm": 9.385298728942871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4461120069026947,
      "logits/rejected": -0.31831610202789307,
      "logps/chosen": -470.0644226074219,
      "logps/rejected": -379.3407287597656,
      "loss": 0.5979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7301653623580933,
      "rewards/margins": 0.49182695150375366,
      "rewards/rejected": -2.221992254257202,
      "step": 9059
    },
    {
      "epoch": 1.1855565551647993,
      "grad_norm": 3.947143077850342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1427554339170456,
      "logits/rejected": -0.2230129837989807,
      "logps/chosen": -354.0311279296875,
      "logps/rejected": -349.2740173339844,
      "loss": 0.2278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7066888809204102,
      "rewards/margins": 1.5008450746536255,
      "rewards/rejected": -2.207533836364746,
      "step": 9060
    },
    {
      "epoch": 1.1856874131021509,
      "grad_norm": 8.05223560333252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06170657277107239,
      "logits/rejected": -0.16184204816818237,
      "logps/chosen": -382.23870849609375,
      "logps/rejected": -646.1503295898438,
      "loss": 0.357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8394157886505127,
      "rewards/margins": 1.1616111993789673,
      "rewards/rejected": -3.0010271072387695,
      "step": 9061
    },
    {
      "epoch": 1.1858182710395027,
      "grad_norm": 7.4448747634887695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06266307085752487,
      "logits/rejected": -0.1237909272313118,
      "logps/chosen": -582.66064453125,
      "logps/rejected": -464.2806701660156,
      "loss": 0.39,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7033664584159851,
      "rewards/margins": 1.5506348609924316,
      "rewards/rejected": -2.2540013790130615,
      "step": 9062
    },
    {
      "epoch": 1.1859491289768544,
      "grad_norm": 31.274900436401367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13675300776958466,
      "logits/rejected": -0.31386426091194153,
      "logps/chosen": -757.804443359375,
      "logps/rejected": -674.1875610351562,
      "loss": 0.7664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5273908376693726,
      "rewards/margins": 1.7737654447555542,
      "rewards/rejected": -3.3011560440063477,
      "step": 9063
    },
    {
      "epoch": 1.1860799869142062,
      "grad_norm": 6.10352087020874,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16573333740234375,
      "logits/rejected": 0.10096465796232224,
      "logps/chosen": -422.94830322265625,
      "logps/rejected": -370.6392822265625,
      "loss": 0.3112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9694108963012695,
      "rewards/margins": 1.6408631801605225,
      "rewards/rejected": -2.610274314880371,
      "step": 9064
    },
    {
      "epoch": 1.186210844851558,
      "grad_norm": 7.9364118576049805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08896525949239731,
      "logits/rejected": -0.188497394323349,
      "logps/chosen": -394.43743896484375,
      "logps/rejected": -334.382568359375,
      "loss": 0.3493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7871637940406799,
      "rewards/margins": 1.6678566932678223,
      "rewards/rejected": -2.4550201892852783,
      "step": 9065
    },
    {
      "epoch": 1.1863417027889098,
      "grad_norm": 8.344038009643555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05109943822026253,
      "logits/rejected": -0.1886047124862671,
      "logps/chosen": -825.9181518554688,
      "logps/rejected": -673.2589111328125,
      "loss": 0.4004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8726302981376648,
      "rewards/margins": 1.476549744606018,
      "rewards/rejected": -2.349179983139038,
      "step": 9066
    },
    {
      "epoch": 1.1864725607262616,
      "grad_norm": 4.777356147766113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15365059673786163,
      "logits/rejected": -0.11575357615947723,
      "logps/chosen": -488.43115234375,
      "logps/rejected": -398.5000915527344,
      "loss": 0.2667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019148249179124832,
      "rewards/margins": 1.3242852687835693,
      "rewards/rejected": -1.3434336185455322,
      "step": 9067
    },
    {
      "epoch": 1.1866034186636134,
      "grad_norm": 12.240545272827148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07913646101951599,
      "logits/rejected": -0.12543073296546936,
      "logps/chosen": -635.146240234375,
      "logps/rejected": -661.65625,
      "loss": 0.7761,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2792220711708069,
      "rewards/margins": 0.33213192224502563,
      "rewards/rejected": -0.6113541126251221,
      "step": 9068
    },
    {
      "epoch": 1.1867342766009652,
      "grad_norm": 5.85393762588501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09200529754161835,
      "logits/rejected": -0.22000059485435486,
      "logps/chosen": -511.2902526855469,
      "logps/rejected": -481.48626708984375,
      "loss": 0.3636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18493595719337463,
      "rewards/margins": 1.7379963397979736,
      "rewards/rejected": -1.9229322671890259,
      "step": 9069
    },
    {
      "epoch": 1.1868651345383168,
      "grad_norm": 4.169113636016846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05532989650964737,
      "logits/rejected": -0.271410197019577,
      "logps/chosen": -408.083251953125,
      "logps/rejected": -394.8265380859375,
      "loss": 0.1643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1463772058486938,
      "rewards/margins": 2.2306199073791504,
      "rewards/rejected": -3.3769969940185547,
      "step": 9070
    },
    {
      "epoch": 1.1869959924756686,
      "grad_norm": 7.769981384277344,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16421574354171753,
      "logits/rejected": 0.01020309329032898,
      "logps/chosen": -524.88525390625,
      "logps/rejected": -655.3305053710938,
      "loss": 0.3617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1842955350875854,
      "rewards/margins": 1.5534838438034058,
      "rewards/rejected": -2.737779378890991,
      "step": 9071
    },
    {
      "epoch": 1.1871268504130204,
      "grad_norm": 8.42896556854248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023816686123609543,
      "logits/rejected": -0.15305298566818237,
      "logps/chosen": -348.0140686035156,
      "logps/rejected": -500.7346496582031,
      "loss": 0.3364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0481549501419067,
      "rewards/margins": 1.63589608669281,
      "rewards/rejected": -2.684051275253296,
      "step": 9072
    },
    {
      "epoch": 1.1872577083503721,
      "grad_norm": 8.114450454711914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18501493334770203,
      "logits/rejected": -0.46134325861930847,
      "logps/chosen": -561.389404296875,
      "logps/rejected": -856.5344848632812,
      "loss": 0.3813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9731777906417847,
      "rewards/margins": 1.71055269241333,
      "rewards/rejected": -2.683730125427246,
      "step": 9073
    },
    {
      "epoch": 1.187388566287724,
      "grad_norm": 5.042609214782715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04016409069299698,
      "logits/rejected": -0.009137660264968872,
      "logps/chosen": -403.92041015625,
      "logps/rejected": -354.3551940917969,
      "loss": 0.2074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39953556656837463,
      "rewards/margins": 2.114072322845459,
      "rewards/rejected": -2.5136075019836426,
      "step": 9074
    },
    {
      "epoch": 1.1875194242250757,
      "grad_norm": 9.614697456359863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1465568244457245,
      "logits/rejected": -0.15959686040878296,
      "logps/chosen": -321.4415588378906,
      "logps/rejected": -307.1353759765625,
      "loss": 0.6179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2197232246398926,
      "rewards/margins": 1.479492425918579,
      "rewards/rejected": -2.6992156505584717,
      "step": 9075
    },
    {
      "epoch": 1.1876502821624273,
      "grad_norm": 6.88480281829834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09514163434505463,
      "logits/rejected": 0.0536809042096138,
      "logps/chosen": -366.9534912109375,
      "logps/rejected": -315.7119445800781,
      "loss": 0.4191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.061797022819519,
      "rewards/margins": 1.0952255725860596,
      "rewards/rejected": -2.157022714614868,
      "step": 9076
    },
    {
      "epoch": 1.187781140099779,
      "grad_norm": 5.940779685974121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04124925285577774,
      "logits/rejected": -0.2826841473579407,
      "logps/chosen": -397.44915771484375,
      "logps/rejected": -426.58514404296875,
      "loss": 0.2425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7788854241371155,
      "rewards/margins": 1.9141367673873901,
      "rewards/rejected": -2.6930220127105713,
      "step": 9077
    },
    {
      "epoch": 1.1879119980371309,
      "grad_norm": 8.762394905090332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10273878276348114,
      "logits/rejected": -0.3306260108947754,
      "logps/chosen": -632.87353515625,
      "logps/rejected": -536.5956420898438,
      "loss": 0.3636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0443897247314453,
      "rewards/margins": 2.0760257244110107,
      "rewards/rejected": -3.120415449142456,
      "step": 9078
    },
    {
      "epoch": 1.1880428559744827,
      "grad_norm": 5.47877311706543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10123756527900696,
      "logits/rejected": 0.1317102015018463,
      "logps/chosen": -489.08941650390625,
      "logps/rejected": -531.2103881835938,
      "loss": 0.229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8572379946708679,
      "rewards/margins": 1.769726037979126,
      "rewards/rejected": -2.6269638538360596,
      "step": 9079
    },
    {
      "epoch": 1.1881737139118345,
      "grad_norm": 8.732511520385742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06879081577062607,
      "logits/rejected": -0.23870441317558289,
      "logps/chosen": -521.1132202148438,
      "logps/rejected": -434.9895935058594,
      "loss": 0.5505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38033831119537354,
      "rewards/margins": 1.6816442012786865,
      "rewards/rejected": -2.0619826316833496,
      "step": 9080
    },
    {
      "epoch": 1.1883045718491863,
      "grad_norm": 10.896172523498535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20047767460346222,
      "logits/rejected": -0.14576809108257294,
      "logps/chosen": -257.2760314941406,
      "logps/rejected": -256.90283203125,
      "loss": 0.6197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2412426471710205,
      "rewards/margins": 0.7955214381217957,
      "rewards/rejected": -2.036764144897461,
      "step": 9081
    },
    {
      "epoch": 1.188435429786538,
      "grad_norm": 4.750142574310303,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11271253228187561,
      "logits/rejected": -0.0034207552671432495,
      "logps/chosen": -528.797119140625,
      "logps/rejected": -607.200927734375,
      "loss": 0.2344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6343792080879211,
      "rewards/margins": 2.3538753986358643,
      "rewards/rejected": -2.9882545471191406,
      "step": 9082
    },
    {
      "epoch": 1.1885662877238898,
      "grad_norm": 10.420419692993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2270612269639969,
      "logits/rejected": -0.35838526487350464,
      "logps/chosen": -499.9807434082031,
      "logps/rejected": -492.0829772949219,
      "loss": 0.4014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0528228282928467,
      "rewards/margins": 1.002997875213623,
      "rewards/rejected": -3.0558207035064697,
      "step": 9083
    },
    {
      "epoch": 1.1886971456612416,
      "grad_norm": 8.110152244567871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08636502921581268,
      "logits/rejected": -0.07192213833332062,
      "logps/chosen": -450.8087158203125,
      "logps/rejected": -353.0078430175781,
      "loss": 0.4738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.819746732711792,
      "rewards/margins": 1.1746561527252197,
      "rewards/rejected": -1.9944028854370117,
      "step": 9084
    },
    {
      "epoch": 1.1888280035985932,
      "grad_norm": 5.10605001449585,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08511486649513245,
      "logits/rejected": -0.24090807139873505,
      "logps/chosen": -462.97747802734375,
      "logps/rejected": -359.2003479003906,
      "loss": 0.2436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.256856083869934,
      "rewards/margins": 1.566960334777832,
      "rewards/rejected": -2.8238162994384766,
      "step": 9085
    },
    {
      "epoch": 1.188958861535945,
      "grad_norm": 21.641408920288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25126585364341736,
      "logits/rejected": -0.31650441884994507,
      "logps/chosen": -553.8588256835938,
      "logps/rejected": -474.5695495605469,
      "loss": 1.2622,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.803809404373169,
      "rewards/margins": -0.2550979256629944,
      "rewards/rejected": -2.5487117767333984,
      "step": 9086
    },
    {
      "epoch": 1.1890897194732968,
      "grad_norm": 3.213596820831299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16741088032722473,
      "logits/rejected": -0.27684417366981506,
      "logps/chosen": -582.6696166992188,
      "logps/rejected": -522.5361938476562,
      "loss": 0.1215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.44977322220802307,
      "rewards/margins": 2.7863504886627197,
      "rewards/rejected": -3.23612380027771,
      "step": 9087
    },
    {
      "epoch": 1.1892205774106486,
      "grad_norm": 10.239655494689941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15035147964954376,
      "logits/rejected": 0.11736051738262177,
      "logps/chosen": -635.779541015625,
      "logps/rejected": -547.53662109375,
      "loss": 0.6435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8503944277763367,
      "rewards/margins": 0.7736982107162476,
      "rewards/rejected": -1.624092698097229,
      "step": 9088
    },
    {
      "epoch": 1.1893514353480004,
      "grad_norm": 7.509174346923828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.449060320854187,
      "logits/rejected": -0.24759326875209808,
      "logps/chosen": -467.2000427246094,
      "logps/rejected": -398.7110900878906,
      "loss": 0.3981,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1400575637817383,
      "rewards/margins": 1.4206340312957764,
      "rewards/rejected": -2.5606915950775146,
      "step": 9089
    },
    {
      "epoch": 1.1894822932853522,
      "grad_norm": 13.878217697143555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3800523579120636,
      "logits/rejected": -0.44149330258369446,
      "logps/chosen": -377.60321044921875,
      "logps/rejected": -426.9117736816406,
      "loss": 0.7477,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.404711365699768,
      "rewards/margins": 0.5940662622451782,
      "rewards/rejected": -1.9987773895263672,
      "step": 9090
    },
    {
      "epoch": 1.189613151222704,
      "grad_norm": 7.489017009735107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009520512074232101,
      "logits/rejected": -0.2647371292114258,
      "logps/chosen": -554.49609375,
      "logps/rejected": -439.6322326660156,
      "loss": 0.379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5613237023353577,
      "rewards/margins": 1.9118881225585938,
      "rewards/rejected": -2.4732117652893066,
      "step": 9091
    },
    {
      "epoch": 1.1897440091600555,
      "grad_norm": 18.370376586914062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011202894151210785,
      "logits/rejected": -0.20743365585803986,
      "logps/chosen": -514.593017578125,
      "logps/rejected": -395.2453308105469,
      "loss": 0.9311,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2609148025512695,
      "rewards/margins": 0.25814688205718994,
      "rewards/rejected": -1.5190616846084595,
      "step": 9092
    },
    {
      "epoch": 1.1898748670974073,
      "grad_norm": 5.021406650543213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07931122183799744,
      "logits/rejected": -0.1465860903263092,
      "logps/chosen": -373.8609619140625,
      "logps/rejected": -359.4522705078125,
      "loss": 0.2911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.502576470375061,
      "rewards/margins": 1.839025855064392,
      "rewards/rejected": -3.341602325439453,
      "step": 9093
    },
    {
      "epoch": 1.190005725034759,
      "grad_norm": 8.3942289352417,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09093905985355377,
      "logits/rejected": -0.04789164289832115,
      "logps/chosen": -481.8617858886719,
      "logps/rejected": -442.71075439453125,
      "loss": 0.4025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.414920836687088,
      "rewards/margins": 1.9214327335357666,
      "rewards/rejected": -1.5065118074417114,
      "step": 9094
    },
    {
      "epoch": 1.190136582972111,
      "grad_norm": 13.262300491333008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4016917645931244,
      "logits/rejected": -0.14510729908943176,
      "logps/chosen": -439.30291748046875,
      "logps/rejected": -404.89111328125,
      "loss": 0.6521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9346832036972046,
      "rewards/margins": 0.9800524711608887,
      "rewards/rejected": -2.9147353172302246,
      "step": 9095
    },
    {
      "epoch": 1.1902674409094627,
      "grad_norm": 6.589329242706299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05415955185890198,
      "logits/rejected": 0.2052459716796875,
      "logps/chosen": -520.4921875,
      "logps/rejected": -502.30078125,
      "loss": 0.3489,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1387364864349365,
      "rewards/margins": 1.3770519495010376,
      "rewards/rejected": -3.5157880783081055,
      "step": 9096
    },
    {
      "epoch": 1.1903982988468145,
      "grad_norm": 7.415420055389404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1767742782831192,
      "logits/rejected": -0.05376450717449188,
      "logps/chosen": -500.9320983886719,
      "logps/rejected": -520.9076538085938,
      "loss": 0.4199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1951005458831787,
      "rewards/margins": 1.244109034538269,
      "rewards/rejected": -2.439209461212158,
      "step": 9097
    },
    {
      "epoch": 1.1905291567841663,
      "grad_norm": 7.929971694946289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039946313947439194,
      "logits/rejected": -0.22211864590644836,
      "logps/chosen": -428.12835693359375,
      "logps/rejected": -527.0282592773438,
      "loss": 0.5324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7529453039169312,
      "rewards/margins": 1.7572648525238037,
      "rewards/rejected": -3.5102100372314453,
      "step": 9098
    },
    {
      "epoch": 1.190660014721518,
      "grad_norm": 13.625225067138672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2759469747543335,
      "logits/rejected": -0.327817440032959,
      "logps/chosen": -413.95635986328125,
      "logps/rejected": -479.30743408203125,
      "loss": 0.8222,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9692625999450684,
      "rewards/margins": 0.9872263073921204,
      "rewards/rejected": -2.956488847732544,
      "step": 9099
    },
    {
      "epoch": 1.1907908726588696,
      "grad_norm": 6.325543403625488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06626273691654205,
      "logits/rejected": -0.004382459446787834,
      "logps/chosen": -617.7506103515625,
      "logps/rejected": -512.6676025390625,
      "loss": 0.2609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39517706632614136,
      "rewards/margins": 1.790328860282898,
      "rewards/rejected": -2.1855058670043945,
      "step": 9100
    },
    {
      "epoch": 1.1909217305962214,
      "grad_norm": 6.924274921417236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4185154139995575,
      "logits/rejected": -0.2014188915491104,
      "logps/chosen": -426.7403564453125,
      "logps/rejected": -473.4217834472656,
      "loss": 0.4201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1635074615478516,
      "rewards/margins": 1.099478840827942,
      "rewards/rejected": -2.262986421585083,
      "step": 9101
    },
    {
      "epoch": 1.1910525885335732,
      "grad_norm": 6.709712505340576,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013680409640073776,
      "logits/rejected": -0.20517614483833313,
      "logps/chosen": -475.92901611328125,
      "logps/rejected": -384.46466064453125,
      "loss": 0.4167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19228383898735046,
      "rewards/margins": 1.2207084894180298,
      "rewards/rejected": -1.4129923582077026,
      "step": 9102
    },
    {
      "epoch": 1.191183446470925,
      "grad_norm": 8.53660774230957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2076094150543213,
      "logits/rejected": -0.2501051127910614,
      "logps/chosen": -549.76220703125,
      "logps/rejected": -415.55316162109375,
      "loss": 0.4177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4994972348213196,
      "rewards/margins": 1.1273733377456665,
      "rewards/rejected": -1.6268706321716309,
      "step": 9103
    },
    {
      "epoch": 1.1913143044082768,
      "grad_norm": 7.563310623168945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2574460506439209,
      "logits/rejected": -0.10144753009080887,
      "logps/chosen": -609.9761962890625,
      "logps/rejected": -604.96533203125,
      "loss": 0.4174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5039211511611938,
      "rewards/margins": 1.2704795598983765,
      "rewards/rejected": -1.7744005918502808,
      "step": 9104
    },
    {
      "epoch": 1.1914451623456286,
      "grad_norm": 7.679068565368652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43747830390930176,
      "logits/rejected": -0.45807063579559326,
      "logps/chosen": -417.1416015625,
      "logps/rejected": -658.3230590820312,
      "loss": 0.4036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8078430891036987,
      "rewards/margins": 1.1516917943954468,
      "rewards/rejected": -1.959534764289856,
      "step": 9105
    },
    {
      "epoch": 1.1915760202829804,
      "grad_norm": 4.303219318389893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5787752866744995,
      "logits/rejected": -0.32841548323631287,
      "logps/chosen": -349.372314453125,
      "logps/rejected": -275.8969421386719,
      "loss": 0.2357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.025229811668396,
      "rewards/margins": 1.7303886413574219,
      "rewards/rejected": -2.7556183338165283,
      "step": 9106
    },
    {
      "epoch": 1.191706878220332,
      "grad_norm": 8.562115669250488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2758193016052246,
      "logits/rejected": -0.35984688997268677,
      "logps/chosen": -516.8603515625,
      "logps/rejected": -559.7313842773438,
      "loss": 0.417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9794701337814331,
      "rewards/margins": 1.4820280075073242,
      "rewards/rejected": -2.4614980220794678,
      "step": 9107
    },
    {
      "epoch": 1.1918377361576837,
      "grad_norm": 7.860743999481201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12125622481107712,
      "logits/rejected": -0.1856541931629181,
      "logps/chosen": -578.3715209960938,
      "logps/rejected": -515.3818969726562,
      "loss": 0.4778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.357277512550354,
      "rewards/margins": 0.9751066565513611,
      "rewards/rejected": -1.3323841094970703,
      "step": 9108
    },
    {
      "epoch": 1.1919685940950355,
      "grad_norm": 9.580201148986816,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12757450342178345,
      "logits/rejected": 0.20799130201339722,
      "logps/chosen": -339.0782165527344,
      "logps/rejected": -339.2196044921875,
      "loss": 0.7808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3688011169433594,
      "rewards/margins": 0.29557138681411743,
      "rewards/rejected": -1.6643725633621216,
      "step": 9109
    },
    {
      "epoch": 1.1920994520323873,
      "grad_norm": 8.806497573852539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20574206113815308,
      "logits/rejected": 0.17999640107154846,
      "logps/chosen": -491.95281982421875,
      "logps/rejected": -477.1238708496094,
      "loss": 0.4711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7580625414848328,
      "rewards/margins": 0.8915320038795471,
      "rewards/rejected": -1.6495945453643799,
      "step": 9110
    },
    {
      "epoch": 1.192230309969739,
      "grad_norm": 3.2415788173675537,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05249898508191109,
      "logits/rejected": -0.014965085312724113,
      "logps/chosen": -485.1675109863281,
      "logps/rejected": -386.343017578125,
      "loss": 0.1427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2228420376777649,
      "rewards/margins": 2.599872589111328,
      "rewards/rejected": -2.8227145671844482,
      "step": 9111
    },
    {
      "epoch": 1.192361167907091,
      "grad_norm": 6.8952250480651855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019864335656166077,
      "logits/rejected": -0.3097713887691498,
      "logps/chosen": -596.8837280273438,
      "logps/rejected": -517.6336059570312,
      "loss": 0.4546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3137981593608856,
      "rewards/margins": 0.885929524898529,
      "rewards/rejected": -1.1997275352478027,
      "step": 9112
    },
    {
      "epoch": 1.1924920258444427,
      "grad_norm": 5.623714923858643,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0026880651712417603,
      "logits/rejected": -0.31473666429519653,
      "logps/chosen": -511.7512512207031,
      "logps/rejected": -490.6408386230469,
      "loss": 0.3052,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7056595087051392,
      "rewards/margins": 1.3887916803359985,
      "rewards/rejected": -2.0944511890411377,
      "step": 9113
    },
    {
      "epoch": 1.1926228837817945,
      "grad_norm": 8.31582260131836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14620335400104523,
      "logits/rejected": -0.1449943333864212,
      "logps/chosen": -546.9114990234375,
      "logps/rejected": -395.6142578125,
      "loss": 0.3712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17355063557624817,
      "rewards/margins": 2.416186809539795,
      "rewards/rejected": -2.5897376537323,
      "step": 9114
    },
    {
      "epoch": 1.1927537417191463,
      "grad_norm": 8.943202018737793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2044641226530075,
      "logits/rejected": -0.06895937025547028,
      "logps/chosen": -401.852783203125,
      "logps/rejected": -467.8921813964844,
      "loss": 0.5366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0777424573898315,
      "rewards/margins": 1.2795920372009277,
      "rewards/rejected": -2.3573341369628906,
      "step": 9115
    },
    {
      "epoch": 1.1928845996564978,
      "grad_norm": 8.208746910095215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20463047921657562,
      "logits/rejected": -0.24981950223445892,
      "logps/chosen": -594.2036743164062,
      "logps/rejected": -399.6744689941406,
      "loss": 0.3256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1342874765396118,
      "rewards/margins": 1.404421329498291,
      "rewards/rejected": -2.5387086868286133,
      "step": 9116
    },
    {
      "epoch": 1.1930154575938496,
      "grad_norm": 10.487174987792969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15222059190273285,
      "logits/rejected": -0.04968853294849396,
      "logps/chosen": -412.56536865234375,
      "logps/rejected": -359.2203674316406,
      "loss": 0.7051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6736855506896973,
      "rewards/margins": 0.7942256927490234,
      "rewards/rejected": -2.4679112434387207,
      "step": 9117
    },
    {
      "epoch": 1.1931463155312014,
      "grad_norm": 4.1246514320373535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2222132384777069,
      "logits/rejected": 0.22834262251853943,
      "logps/chosen": -369.25103759765625,
      "logps/rejected": -328.9378967285156,
      "loss": 0.2664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0712958574295044,
      "rewards/margins": 1.9057952165603638,
      "rewards/rejected": -1.9770911931991577,
      "step": 9118
    },
    {
      "epoch": 1.1932771734685532,
      "grad_norm": 6.258331298828125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2931225597858429,
      "logits/rejected": -0.12585532665252686,
      "logps/chosen": -399.3533630371094,
      "logps/rejected": -439.4459228515625,
      "loss": 0.2764,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6897459626197815,
      "rewards/margins": 1.7204456329345703,
      "rewards/rejected": -2.410191535949707,
      "step": 9119
    },
    {
      "epoch": 1.193408031405905,
      "grad_norm": 6.805421352386475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07503798604011536,
      "logits/rejected": -0.20166252553462982,
      "logps/chosen": -367.7629089355469,
      "logps/rejected": -314.9398193359375,
      "loss": 0.4884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9097363352775574,
      "rewards/margins": 1.3228176832199097,
      "rewards/rejected": -2.2325539588928223,
      "step": 9120
    },
    {
      "epoch": 1.1935388893432568,
      "grad_norm": 6.448980808258057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15972144901752472,
      "logits/rejected": -0.15642301738262177,
      "logps/chosen": -380.85260009765625,
      "logps/rejected": -424.6130676269531,
      "loss": 0.397,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3401707410812378,
      "rewards/margins": 1.5943982601165771,
      "rewards/rejected": -2.9345693588256836,
      "step": 9121
    },
    {
      "epoch": 1.1936697472806084,
      "grad_norm": 7.727387428283691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06903387606143951,
      "logits/rejected": -0.2691044211387634,
      "logps/chosen": -471.8133850097656,
      "logps/rejected": -522.5111083984375,
      "loss": 0.4384,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7383197546005249,
      "rewards/margins": 1.4970670938491821,
      "rewards/rejected": -2.235386848449707,
      "step": 9122
    },
    {
      "epoch": 1.1938006052179602,
      "grad_norm": 9.55190372467041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04467019438743591,
      "logits/rejected": -0.3146640360355377,
      "logps/chosen": -418.492431640625,
      "logps/rejected": -368.98199462890625,
      "loss": 0.7777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9144710302352905,
      "rewards/margins": 0.7912026047706604,
      "rewards/rejected": -2.7056736946105957,
      "step": 9123
    },
    {
      "epoch": 1.193931463155312,
      "grad_norm": 16.717819213867188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44284719228744507,
      "logits/rejected": -0.3238633871078491,
      "logps/chosen": -335.6704406738281,
      "logps/rejected": -384.40802001953125,
      "loss": 0.959,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9703441858291626,
      "rewards/margins": -0.11899504065513611,
      "rewards/rejected": -0.8513491153717041,
      "step": 9124
    },
    {
      "epoch": 1.1940623210926637,
      "grad_norm": 8.172064781188965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16737578809261322,
      "logits/rejected": -0.30537912249565125,
      "logps/chosen": -301.2248840332031,
      "logps/rejected": -383.09332275390625,
      "loss": 0.3827,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.181884765625,
      "rewards/margins": 1.007657527923584,
      "rewards/rejected": -1.1895421743392944,
      "step": 9125
    },
    {
      "epoch": 1.1941931790300155,
      "grad_norm": 8.445365905761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13355150818824768,
      "logits/rejected": -0.1608683168888092,
      "logps/chosen": -352.1407470703125,
      "logps/rejected": -422.80975341796875,
      "loss": 0.4921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2639527320861816,
      "rewards/margins": 1.0043212175369263,
      "rewards/rejected": -2.2682740688323975,
      "step": 9126
    },
    {
      "epoch": 1.1943240369673673,
      "grad_norm": 16.484079360961914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48435261845588684,
      "logits/rejected": -0.30502140522003174,
      "logps/chosen": -454.94464111328125,
      "logps/rejected": -371.10540771484375,
      "loss": 1.0626,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.865926742553711,
      "rewards/margins": 0.479442834854126,
      "rewards/rejected": -2.345369338989258,
      "step": 9127
    },
    {
      "epoch": 1.1944548949047191,
      "grad_norm": 9.36958122253418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012257270514965057,
      "logits/rejected": -0.06284669041633606,
      "logps/chosen": -516.8033447265625,
      "logps/rejected": -490.67523193359375,
      "loss": 0.3754,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8173079490661621,
      "rewards/margins": 1.7515969276428223,
      "rewards/rejected": -2.5689046382904053,
      "step": 9128
    },
    {
      "epoch": 1.194585752842071,
      "grad_norm": 7.973827362060547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12181859463453293,
      "logits/rejected": -0.1405855119228363,
      "logps/chosen": -566.0714111328125,
      "logps/rejected": -435.8175354003906,
      "loss": 0.4179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.293043851852417,
      "rewards/margins": 1.2438627481460571,
      "rewards/rejected": -2.5369064807891846,
      "step": 9129
    },
    {
      "epoch": 1.1947166107794227,
      "grad_norm": 8.916509628295898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32495981454849243,
      "logits/rejected": -0.3704196512699127,
      "logps/chosen": -371.2458801269531,
      "logps/rejected": -489.4281921386719,
      "loss": 0.5039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10803385823965073,
      "rewards/margins": 1.995132327079773,
      "rewards/rejected": -2.103166341781616,
      "step": 9130
    },
    {
      "epoch": 1.1948474687167743,
      "grad_norm": 1.9013447761535645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04538097232580185,
      "logits/rejected": -0.05229022353887558,
      "logps/chosen": -387.3708190917969,
      "logps/rejected": -479.60919189453125,
      "loss": 0.0798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01183851808309555,
      "rewards/margins": 3.0437839031219482,
      "rewards/rejected": -3.055622100830078,
      "step": 9131
    },
    {
      "epoch": 1.194978326654126,
      "grad_norm": 5.77883243560791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1991267204284668,
      "logits/rejected": -0.2842060327529907,
      "logps/chosen": -575.6416625976562,
      "logps/rejected": -540.4666748046875,
      "loss": 0.2637,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.675734519958496,
      "rewards/margins": 1.6986455917358398,
      "rewards/rejected": -3.374380350112915,
      "step": 9132
    },
    {
      "epoch": 1.1951091845914779,
      "grad_norm": 7.957622528076172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31197842955589294,
      "logits/rejected": -0.3240739703178406,
      "logps/chosen": -529.1572875976562,
      "logps/rejected": -330.3487854003906,
      "loss": 0.4912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6941400766372681,
      "rewards/margins": 1.085741639137268,
      "rewards/rejected": -1.7798817157745361,
      "step": 9133
    },
    {
      "epoch": 1.1952400425288296,
      "grad_norm": 11.358054161071777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22268153727054596,
      "logits/rejected": -0.1839197725057602,
      "logps/chosen": -379.8641357421875,
      "logps/rejected": -325.0793762207031,
      "loss": 0.7587,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.354670763015747,
      "rewards/margins": 0.3604215979576111,
      "rewards/rejected": -1.7150921821594238,
      "step": 9134
    },
    {
      "epoch": 1.1953709004661814,
      "grad_norm": 8.39833927154541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2439422309398651,
      "logits/rejected": -0.2666514813899994,
      "logps/chosen": -540.2134399414062,
      "logps/rejected": -452.2834167480469,
      "loss": 0.3761,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1974594593048096,
      "rewards/margins": 2.1275100708007812,
      "rewards/rejected": -3.3249692916870117,
      "step": 9135
    },
    {
      "epoch": 1.1955017584035332,
      "grad_norm": 8.680584907531738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03095487505197525,
      "logits/rejected": -0.16931402683258057,
      "logps/chosen": -382.2519226074219,
      "logps/rejected": -415.55712890625,
      "loss": 0.5297,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5380973815917969,
      "rewards/margins": 0.8659336566925049,
      "rewards/rejected": -1.4040310382843018,
      "step": 9136
    },
    {
      "epoch": 1.195632616340885,
      "grad_norm": 9.084482192993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3356424570083618,
      "logits/rejected": -0.19977328181266785,
      "logps/chosen": -463.6065368652344,
      "logps/rejected": -472.01251220703125,
      "loss": 0.5311,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.230028748512268,
      "rewards/margins": 2.0772817134857178,
      "rewards/rejected": -3.3073103427886963,
      "step": 9137
    },
    {
      "epoch": 1.1957634742782366,
      "grad_norm": 6.4795403480529785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07285340875387192,
      "logits/rejected": -0.04151160269975662,
      "logps/chosen": -446.19866943359375,
      "logps/rejected": -425.2763671875,
      "loss": 0.2979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3606818914413452,
      "rewards/margins": 1.3916680812835693,
      "rewards/rejected": -1.752350091934204,
      "step": 9138
    },
    {
      "epoch": 1.1958943322155884,
      "grad_norm": 6.039937496185303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10867089033126831,
      "logits/rejected": -0.15521809458732605,
      "logps/chosen": -403.51531982421875,
      "logps/rejected": -400.0380554199219,
      "loss": 0.2971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.166282296180725,
      "rewards/margins": 1.2010517120361328,
      "rewards/rejected": -2.3673338890075684,
      "step": 9139
    },
    {
      "epoch": 1.1960251901529402,
      "grad_norm": 7.945528030395508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06245718523859978,
      "logits/rejected": 0.04503421485424042,
      "logps/chosen": -482.0110778808594,
      "logps/rejected": -532.8779296875,
      "loss": 0.477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.830961287021637,
      "rewards/margins": 0.8894062638282776,
      "rewards/rejected": -1.720367431640625,
      "step": 9140
    },
    {
      "epoch": 1.196156048090292,
      "grad_norm": 8.29885482788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09695026278495789,
      "logits/rejected": -0.21821168065071106,
      "logps/chosen": -240.15228271484375,
      "logps/rejected": -323.78253173828125,
      "loss": 0.3121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6853194832801819,
      "rewards/margins": 2.1839513778686523,
      "rewards/rejected": -2.8692710399627686,
      "step": 9141
    },
    {
      "epoch": 1.1962869060276438,
      "grad_norm": 6.004024505615234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27272292971611023,
      "logits/rejected": -0.24199546873569489,
      "logps/chosen": -370.3651428222656,
      "logps/rejected": -411.1540832519531,
      "loss": 0.2477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6741451621055603,
      "rewards/margins": 1.6012399196624756,
      "rewards/rejected": -2.2753851413726807,
      "step": 9142
    },
    {
      "epoch": 1.1964177639649956,
      "grad_norm": 4.596853256225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03830741345882416,
      "logits/rejected": 0.09308148920536041,
      "logps/chosen": -452.2242736816406,
      "logps/rejected": -496.3067626953125,
      "loss": 0.2353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1997956037521362,
      "rewards/margins": 1.8035157918930054,
      "rewards/rejected": -3.0033111572265625,
      "step": 9143
    },
    {
      "epoch": 1.1965486219023473,
      "grad_norm": 6.818634510040283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043082576245069504,
      "logits/rejected": -0.13317610323429108,
      "logps/chosen": -599.3338623046875,
      "logps/rejected": -481.83953857421875,
      "loss": 0.3747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22694969177246094,
      "rewards/margins": 1.556982159614563,
      "rewards/rejected": -1.3300323486328125,
      "step": 9144
    },
    {
      "epoch": 1.1966794798396991,
      "grad_norm": 10.159221649169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15444843471050262,
      "logits/rejected": -0.11366879940032959,
      "logps/chosen": -549.9376831054688,
      "logps/rejected": -610.5803833007812,
      "loss": 0.6254,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.085170030593872,
      "rewards/margins": 0.5403186678886414,
      "rewards/rejected": -1.6254886388778687,
      "step": 9145
    },
    {
      "epoch": 1.1968103377770507,
      "grad_norm": 3.943758964538574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04862681403756142,
      "logits/rejected": -0.08550611883401871,
      "logps/chosen": -360.7318420410156,
      "logps/rejected": -338.34747314453125,
      "loss": 0.2123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3172546327114105,
      "rewards/margins": 1.8654168844223022,
      "rewards/rejected": -2.182671546936035,
      "step": 9146
    },
    {
      "epoch": 1.1969411957144025,
      "grad_norm": 5.672837257385254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10307981073856354,
      "logits/rejected": -0.44294580817222595,
      "logps/chosen": -430.8005676269531,
      "logps/rejected": -331.4776916503906,
      "loss": 0.277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3386211395263672,
      "rewards/margins": 1.771740198135376,
      "rewards/rejected": -3.110361337661743,
      "step": 9147
    },
    {
      "epoch": 1.1970720536517543,
      "grad_norm": 10.635882377624512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5229136347770691,
      "logits/rejected": -0.6593222618103027,
      "logps/chosen": -498.97393798828125,
      "logps/rejected": -464.22100830078125,
      "loss": 0.4851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4617195129394531,
      "rewards/margins": 1.0809590816497803,
      "rewards/rejected": -2.5426788330078125,
      "step": 9148
    },
    {
      "epoch": 1.197202911589106,
      "grad_norm": 4.3537726402282715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08918353915214539,
      "logits/rejected": -0.17852112650871277,
      "logps/chosen": -506.5070495605469,
      "logps/rejected": -430.6822814941406,
      "loss": 0.1972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2929248809814453,
      "rewards/margins": 2.065521240234375,
      "rewards/rejected": -2.358445882797241,
      "step": 9149
    },
    {
      "epoch": 1.1973337695264579,
      "grad_norm": 9.29021167755127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2916569411754608,
      "logits/rejected": -0.19695189595222473,
      "logps/chosen": -410.9582214355469,
      "logps/rejected": -450.1560363769531,
      "loss": 0.524,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38510096073150635,
      "rewards/margins": 1.2618061304092407,
      "rewards/rejected": -1.646907091140747,
      "step": 9150
    },
    {
      "epoch": 1.1974646274638097,
      "grad_norm": 5.8462605476379395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23046378791332245,
      "logits/rejected": -0.24858170747756958,
      "logps/chosen": -349.85552978515625,
      "logps/rejected": -384.91339111328125,
      "loss": 0.3022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5448824167251587,
      "rewards/margins": 1.829418659210205,
      "rewards/rejected": -1.284536361694336,
      "step": 9151
    },
    {
      "epoch": 1.1975954854011615,
      "grad_norm": 6.143537998199463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0038754642009735107,
      "logits/rejected": -0.008330628275871277,
      "logps/chosen": -704.9713745117188,
      "logps/rejected": -612.809814453125,
      "loss": 0.3392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.023500919342041,
      "rewards/margins": 1.5865845680236816,
      "rewards/rejected": -2.6100852489471436,
      "step": 9152
    },
    {
      "epoch": 1.197726343338513,
      "grad_norm": 8.778636932373047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23882392048835754,
      "logits/rejected": -0.11355267465114594,
      "logps/chosen": -514.6289672851562,
      "logps/rejected": -461.50518798828125,
      "loss": 0.3968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7662588357925415,
      "rewards/margins": 1.232733964920044,
      "rewards/rejected": -1.9989928007125854,
      "step": 9153
    },
    {
      "epoch": 1.1978572012758648,
      "grad_norm": 9.87641716003418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12120398879051208,
      "logits/rejected": -0.08809836953878403,
      "logps/chosen": -503.25457763671875,
      "logps/rejected": -502.1048583984375,
      "loss": 0.4173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8967863321304321,
      "rewards/margins": 1.4033355712890625,
      "rewards/rejected": -3.300121784210205,
      "step": 9154
    },
    {
      "epoch": 1.1979880592132166,
      "grad_norm": 6.719931125640869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22175613045692444,
      "logits/rejected": -0.5043169260025024,
      "logps/chosen": -636.321044921875,
      "logps/rejected": -553.9700927734375,
      "loss": 0.5695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5396313667297363,
      "rewards/margins": 0.9818706512451172,
      "rewards/rejected": -2.5215020179748535,
      "step": 9155
    },
    {
      "epoch": 1.1981189171505684,
      "grad_norm": 10.854837417602539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08138097077608109,
      "logits/rejected": -0.10919195413589478,
      "logps/chosen": -424.11322021484375,
      "logps/rejected": -429.6429443359375,
      "loss": 0.6729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1146221160888672,
      "rewards/margins": 0.26624220609664917,
      "rewards/rejected": -1.3808643817901611,
      "step": 9156
    },
    {
      "epoch": 1.1982497750879202,
      "grad_norm": 6.458920955657959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15228977799415588,
      "logits/rejected": -0.5094640851020813,
      "logps/chosen": -409.990478515625,
      "logps/rejected": -563.8181762695312,
      "loss": 0.474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19189892709255219,
      "rewards/margins": 1.0114316940307617,
      "rewards/rejected": -1.203330636024475,
      "step": 9157
    },
    {
      "epoch": 1.198380633025272,
      "grad_norm": 4.260407447814941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22886160016059875,
      "logits/rejected": -0.1832752823829651,
      "logps/chosen": -633.5587158203125,
      "logps/rejected": -521.6367797851562,
      "loss": 0.2176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39859598875045776,
      "rewards/margins": 2.3096837997436523,
      "rewards/rejected": -2.708279609680176,
      "step": 9158
    },
    {
      "epoch": 1.1985114909626238,
      "grad_norm": 7.198769569396973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17099720239639282,
      "logits/rejected": 0.028096653521060944,
      "logps/chosen": -599.9374389648438,
      "logps/rejected": -820.853759765625,
      "loss": 0.2535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06077232211828232,
      "rewards/margins": 2.359856367111206,
      "rewards/rejected": -2.299084186553955,
      "step": 9159
    },
    {
      "epoch": 1.1986423488999756,
      "grad_norm": 9.280677795410156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04053649306297302,
      "logits/rejected": -0.12236376106739044,
      "logps/chosen": -261.8763122558594,
      "logps/rejected": -411.24237060546875,
      "loss": 0.4441,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6176756620407104,
      "rewards/margins": 2.10524320602417,
      "rewards/rejected": -3.72291898727417,
      "step": 9160
    },
    {
      "epoch": 1.1987732068373271,
      "grad_norm": 14.731473922729492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0820498988032341,
      "logits/rejected": -0.19966717064380646,
      "logps/chosen": -498.4049072265625,
      "logps/rejected": -394.68743896484375,
      "loss": 0.7416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1497318744659424,
      "rewards/margins": 0.35060688853263855,
      "rewards/rejected": -1.5003387928009033,
      "step": 9161
    },
    {
      "epoch": 1.198904064774679,
      "grad_norm": 8.598251342773438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.259998619556427,
      "logits/rejected": -0.11655841767787933,
      "logps/chosen": -449.1522216796875,
      "logps/rejected": -363.72662353515625,
      "loss": 0.3888,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8291381597518921,
      "rewards/margins": 1.709463119506836,
      "rewards/rejected": -2.5386013984680176,
      "step": 9162
    },
    {
      "epoch": 1.1990349227120307,
      "grad_norm": 5.487895488739014,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02194739133119583,
      "logits/rejected": 0.07086382806301117,
      "logps/chosen": -403.51275634765625,
      "logps/rejected": -395.40264892578125,
      "loss": 0.3396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02829436957836151,
      "rewards/margins": 1.2168840169906616,
      "rewards/rejected": -1.1885896921157837,
      "step": 9163
    },
    {
      "epoch": 1.1991657806493825,
      "grad_norm": 7.048746585845947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11869767308235168,
      "logits/rejected": -0.1861635148525238,
      "logps/chosen": -384.7745056152344,
      "logps/rejected": -550.7299194335938,
      "loss": 0.2611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8229830265045166,
      "rewards/margins": 2.826076030731201,
      "rewards/rejected": -3.6490590572357178,
      "step": 9164
    },
    {
      "epoch": 1.1992966385867343,
      "grad_norm": 7.34096097946167,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05896498262882233,
      "logits/rejected": -0.12751995027065277,
      "logps/chosen": -445.1458740234375,
      "logps/rejected": -448.3731994628906,
      "loss": 0.4668,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5898195505142212,
      "rewards/margins": 0.8089742064476013,
      "rewards/rejected": -2.3987936973571777,
      "step": 9165
    },
    {
      "epoch": 1.199427496524086,
      "grad_norm": 5.925567150115967,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0651901513338089,
      "logits/rejected": -0.2698984146118164,
      "logps/chosen": -517.5233154296875,
      "logps/rejected": -483.40362548828125,
      "loss": 0.2668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2814280986785889,
      "rewards/margins": 2.420107841491699,
      "rewards/rejected": -3.701535701751709,
      "step": 9166
    },
    {
      "epoch": 1.1995583544614379,
      "grad_norm": 4.218836307525635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14074143767356873,
      "logits/rejected": -0.36372801661491394,
      "logps/chosen": -422.9979553222656,
      "logps/rejected": -457.73223876953125,
      "loss": 0.2458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.330279141664505,
      "rewards/margins": 2.0437979698181152,
      "rewards/rejected": -2.374077081680298,
      "step": 9167
    },
    {
      "epoch": 1.1996892123987895,
      "grad_norm": 8.727029800415039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23567885160446167,
      "logits/rejected": -0.23108896613121033,
      "logps/chosen": -419.85565185546875,
      "logps/rejected": -518.3053588867188,
      "loss": 0.4645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7763772010803223,
      "rewards/margins": 1.112396240234375,
      "rewards/rejected": -2.8887734413146973,
      "step": 9168
    },
    {
      "epoch": 1.1998200703361412,
      "grad_norm": 8.507076263427734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06323553621768951,
      "logits/rejected": -0.17378751933574677,
      "logps/chosen": -482.523193359375,
      "logps/rejected": -392.07781982421875,
      "loss": 0.6331,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.138819694519043,
      "rewards/margins": 0.458394855260849,
      "rewards/rejected": -1.5972144603729248,
      "step": 9169
    },
    {
      "epoch": 1.199950928273493,
      "grad_norm": 14.550532341003418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012610673904418945,
      "logits/rejected": -0.08391629159450531,
      "logps/chosen": -449.35699462890625,
      "logps/rejected": -446.45068359375,
      "loss": 0.9982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.5159053802490234,
      "rewards/margins": 0.1264389604330063,
      "rewards/rejected": -2.6423444747924805,
      "step": 9170
    },
    {
      "epoch": 1.2000817862108448,
      "grad_norm": 4.017470836639404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21881243586540222,
      "logits/rejected": -0.18094654381275177,
      "logps/chosen": -659.1513061523438,
      "logps/rejected": -566.78857421875,
      "loss": 0.1944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1539980173110962,
      "rewards/margins": 1.6563658714294434,
      "rewards/rejected": -1.810364007949829,
      "step": 9171
    },
    {
      "epoch": 1.2002126441481966,
      "grad_norm": 5.277939796447754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13441067934036255,
      "logits/rejected": 0.07405450940132141,
      "logps/chosen": -375.4176025390625,
      "logps/rejected": -448.3916015625,
      "loss": 0.334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5620895624160767,
      "rewards/margins": 1.7898422479629517,
      "rewards/rejected": -2.351931571960449,
      "step": 9172
    },
    {
      "epoch": 1.2003435020855484,
      "grad_norm": 7.680543899536133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13224788010120392,
      "logits/rejected": 0.11138175427913666,
      "logps/chosen": -561.14501953125,
      "logps/rejected": -487.2905578613281,
      "loss": 0.3458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3096509873867035,
      "rewards/margins": 1.3067001104354858,
      "rewards/rejected": -1.6163512468338013,
      "step": 9173
    },
    {
      "epoch": 1.2004743600229002,
      "grad_norm": 7.434933662414551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30882129073143005,
      "logits/rejected": -0.568955659866333,
      "logps/chosen": -417.30035400390625,
      "logps/rejected": -444.34423828125,
      "loss": 0.4455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1755363941192627,
      "rewards/margins": 1.8485283851623535,
      "rewards/rejected": -3.024064540863037,
      "step": 9174
    },
    {
      "epoch": 1.200605217960252,
      "grad_norm": 11.613805770874023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1712017059326172,
      "logits/rejected": 0.08530266582965851,
      "logps/chosen": -442.17315673828125,
      "logps/rejected": -489.7478332519531,
      "loss": 0.515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.178401470184326,
      "rewards/margins": 1.6128062009811401,
      "rewards/rejected": -3.791208028793335,
      "step": 9175
    },
    {
      "epoch": 1.2007360758976038,
      "grad_norm": 9.016578674316406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08531695604324341,
      "logits/rejected": 0.00969728548079729,
      "logps/chosen": -391.9772033691406,
      "logps/rejected": -418.5413818359375,
      "loss": 0.4825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7418642044067383,
      "rewards/margins": 1.4414715766906738,
      "rewards/rejected": -2.183335781097412,
      "step": 9176
    },
    {
      "epoch": 1.2008669338349554,
      "grad_norm": 6.864232540130615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1846362203359604,
      "logits/rejected": -0.4340175688266754,
      "logps/chosen": -398.0328063964844,
      "logps/rejected": -373.77978515625,
      "loss": 0.4002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.748642086982727,
      "rewards/margins": 1.3054295778274536,
      "rewards/rejected": -3.0540719032287598,
      "step": 9177
    },
    {
      "epoch": 1.2009977917723071,
      "grad_norm": 5.849226474761963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1263389140367508,
      "logits/rejected": -0.06325831264257431,
      "logps/chosen": -316.772216796875,
      "logps/rejected": -432.4352111816406,
      "loss": 0.409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8783893585205078,
      "rewards/margins": 2.3946313858032227,
      "rewards/rejected": -3.2730209827423096,
      "step": 9178
    },
    {
      "epoch": 1.201128649709659,
      "grad_norm": 15.435785293579102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21878503262996674,
      "logits/rejected": -0.4054660499095917,
      "logps/chosen": -482.75311279296875,
      "logps/rejected": -386.7048034667969,
      "loss": 0.6275,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8723962306976318,
      "rewards/margins": 1.4465889930725098,
      "rewards/rejected": -3.3189852237701416,
      "step": 9179
    },
    {
      "epoch": 1.2012595076470107,
      "grad_norm": 11.545106887817383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2370639443397522,
      "logits/rejected": -0.12733429670333862,
      "logps/chosen": -328.4519348144531,
      "logps/rejected": -468.5172424316406,
      "loss": 0.7151,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2623991966247559,
      "rewards/margins": 0.4415389597415924,
      "rewards/rejected": -1.7039381265640259,
      "step": 9180
    },
    {
      "epoch": 1.2013903655843625,
      "grad_norm": 8.347423553466797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2058723419904709,
      "logits/rejected": 0.07437638193368912,
      "logps/chosen": -421.2145690917969,
      "logps/rejected": -447.75250244140625,
      "loss": 0.4565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1628048419952393,
      "rewards/margins": 1.2198693752288818,
      "rewards/rejected": -3.382674217224121,
      "step": 9181
    },
    {
      "epoch": 1.2015212235217143,
      "grad_norm": 5.893599987030029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.056055035442113876,
      "logits/rejected": -0.002434348687529564,
      "logps/chosen": -272.4399719238281,
      "logps/rejected": -298.359375,
      "loss": 0.3304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0349934101104736,
      "rewards/margins": 1.6416233777999878,
      "rewards/rejected": -2.676616907119751,
      "step": 9182
    },
    {
      "epoch": 1.2016520814590659,
      "grad_norm": 5.779366970062256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023081544786691666,
      "logits/rejected": -0.055485595017671585,
      "logps/chosen": -507.51641845703125,
      "logps/rejected": -428.1515197753906,
      "loss": 0.2655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24233628809452057,
      "rewards/margins": 1.5994606018066406,
      "rewards/rejected": -1.8417969942092896,
      "step": 9183
    },
    {
      "epoch": 1.2017829393964177,
      "grad_norm": 8.741890907287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5187389254570007,
      "logits/rejected": -0.4472653269767761,
      "logps/chosen": -416.83709716796875,
      "logps/rejected": -457.4294128417969,
      "loss": 0.6195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7394084930419922,
      "rewards/margins": 0.4284830391407013,
      "rewards/rejected": -2.167891502380371,
      "step": 9184
    },
    {
      "epoch": 1.2019137973337695,
      "grad_norm": 7.018562316894531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1609424203634262,
      "logits/rejected": -0.1853274554014206,
      "logps/chosen": -529.2833251953125,
      "logps/rejected": -468.34539794921875,
      "loss": 0.2755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6632024049758911,
      "rewards/margins": 1.797034502029419,
      "rewards/rejected": -3.4602370262145996,
      "step": 9185
    },
    {
      "epoch": 1.2020446552711213,
      "grad_norm": 9.156228065490723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10006523132324219,
      "logits/rejected": 0.013245508074760437,
      "logps/chosen": -528.9390869140625,
      "logps/rejected": -522.94091796875,
      "loss": 0.7384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1038120985031128,
      "rewards/margins": 1.2927744388580322,
      "rewards/rejected": -2.3965864181518555,
      "step": 9186
    },
    {
      "epoch": 1.202175513208473,
      "grad_norm": 8.476720809936523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21918557584285736,
      "logits/rejected": -0.2282457798719406,
      "logps/chosen": -485.79779052734375,
      "logps/rejected": -428.0894775390625,
      "loss": 0.5792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.736215353012085,
      "rewards/margins": 1.1900619268417358,
      "rewards/rejected": -3.9262771606445312,
      "step": 9187
    },
    {
      "epoch": 1.2023063711458248,
      "grad_norm": 4.582477569580078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3199444115161896,
      "logits/rejected": -0.20707812905311584,
      "logps/chosen": -416.80712890625,
      "logps/rejected": -456.5140686035156,
      "loss": 0.1472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5601980686187744,
      "rewards/margins": 2.36513614654541,
      "rewards/rejected": -2.9253339767456055,
      "step": 9188
    },
    {
      "epoch": 1.2024372290831766,
      "grad_norm": 9.449344635009766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17577245831489563,
      "logits/rejected": -0.25986623764038086,
      "logps/chosen": -288.2716064453125,
      "logps/rejected": -282.83404541015625,
      "loss": 0.6521,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.686344861984253,
      "rewards/margins": 0.6321938037872314,
      "rewards/rejected": -2.3185386657714844,
      "step": 9189
    },
    {
      "epoch": 1.2025680870205284,
      "grad_norm": 13.93593692779541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06306441128253937,
      "logits/rejected": -0.09513719379901886,
      "logps/chosen": -588.874267578125,
      "logps/rejected": -566.364013671875,
      "loss": 0.5938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1640673875808716,
      "rewards/margins": 1.5368716716766357,
      "rewards/rejected": -2.700939178466797,
      "step": 9190
    },
    {
      "epoch": 1.2026989449578802,
      "grad_norm": 5.902402877807617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6026232838630676,
      "logits/rejected": -0.2641229033470154,
      "logps/chosen": -415.6072082519531,
      "logps/rejected": -541.9349365234375,
      "loss": 0.3703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2632567882537842,
      "rewards/margins": 1.4506340026855469,
      "rewards/rejected": -2.71389102935791,
      "step": 9191
    },
    {
      "epoch": 1.2028298028952318,
      "grad_norm": 4.102019786834717,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1326414942741394,
      "logits/rejected": -0.007034756243228912,
      "logps/chosen": -718.6497802734375,
      "logps/rejected": -535.2327270507812,
      "loss": 0.1613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05968360975384712,
      "rewards/margins": 2.6310977935791016,
      "rewards/rejected": -2.690781593322754,
      "step": 9192
    },
    {
      "epoch": 1.2029606608325836,
      "grad_norm": 8.310050964355469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24934130907058716,
      "logits/rejected": -0.2092437446117401,
      "logps/chosen": -436.854248046875,
      "logps/rejected": -311.4510192871094,
      "loss": 0.3229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4883923530578613,
      "rewards/margins": 1.7645795345306396,
      "rewards/rejected": -3.25297212600708,
      "step": 9193
    },
    {
      "epoch": 1.2030915187699354,
      "grad_norm": 7.185649394989014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33572062849998474,
      "logits/rejected": -0.5292306542396545,
      "logps/chosen": -657.7734375,
      "logps/rejected": -558.053466796875,
      "loss": 0.2182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2017175555229187,
      "rewards/margins": 2.3585801124572754,
      "rewards/rejected": -2.560297727584839,
      "step": 9194
    },
    {
      "epoch": 1.2032223767072872,
      "grad_norm": 2.0699384212493896,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10662025213241577,
      "logits/rejected": -0.11231058835983276,
      "logps/chosen": -576.6549072265625,
      "logps/rejected": -391.58929443359375,
      "loss": 0.1125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11433148384094238,
      "rewards/margins": 2.6684532165527344,
      "rewards/rejected": -2.7827847003936768,
      "step": 9195
    },
    {
      "epoch": 1.203353234644639,
      "grad_norm": 9.214411735534668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11289692670106888,
      "logits/rejected": -0.1695351004600525,
      "logps/chosen": -478.4034118652344,
      "logps/rejected": -351.91497802734375,
      "loss": 0.816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8595167398452759,
      "rewards/margins": 0.3000390827655792,
      "rewards/rejected": -2.159555673599243,
      "step": 9196
    },
    {
      "epoch": 1.2034840925819907,
      "grad_norm": 7.27231502532959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3543340563774109,
      "logits/rejected": -0.2912905216217041,
      "logps/chosen": -416.75787353515625,
      "logps/rejected": -370.04791259765625,
      "loss": 0.3973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8039541244506836,
      "rewards/margins": 1.1409037113189697,
      "rewards/rejected": -1.9448579549789429,
      "step": 9197
    },
    {
      "epoch": 1.2036149505193425,
      "grad_norm": 8.510164260864258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21003887057304382,
      "logits/rejected": -0.20579257607460022,
      "logps/chosen": -536.69970703125,
      "logps/rejected": -492.8379211425781,
      "loss": 0.464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0398032665252686,
      "rewards/margins": 1.0870622396469116,
      "rewards/rejected": -2.1268656253814697,
      "step": 9198
    },
    {
      "epoch": 1.203745808456694,
      "grad_norm": 6.623191833496094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12735038995742798,
      "logits/rejected": -0.14930884540081024,
      "logps/chosen": -604.4495239257812,
      "logps/rejected": -516.1651000976562,
      "loss": 0.3612,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30277711153030396,
      "rewards/margins": 1.8957363367080688,
      "rewards/rejected": -2.1985132694244385,
      "step": 9199
    },
    {
      "epoch": 1.203876666394046,
      "grad_norm": 10.154623031616211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2028394639492035,
      "logits/rejected": -0.24499967694282532,
      "logps/chosen": -533.0379638671875,
      "logps/rejected": -376.5787658691406,
      "loss": 0.6555,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1910227537155151,
      "rewards/margins": 0.60959792137146,
      "rewards/rejected": -1.800620675086975,
      "step": 9200
    },
    {
      "epoch": 1.2040075243313977,
      "grad_norm": 4.340278148651123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00108322873711586,
      "logits/rejected": -0.1366526484489441,
      "logps/chosen": -404.51666259765625,
      "logps/rejected": -511.677490234375,
      "loss": 0.1534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6445194482803345,
      "rewards/margins": 3.0041189193725586,
      "rewards/rejected": -4.6486382484436035,
      "step": 9201
    },
    {
      "epoch": 1.2041383822687495,
      "grad_norm": 9.37198257446289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12800759077072144,
      "logits/rejected": 0.0006208643317222595,
      "logps/chosen": -543.5082397460938,
      "logps/rejected": -481.95831298828125,
      "loss": 0.5349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.394338995218277,
      "rewards/margins": 0.8024786114692688,
      "rewards/rejected": -1.1968176364898682,
      "step": 9202
    },
    {
      "epoch": 1.2042692402061013,
      "grad_norm": 2.926776885986328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.254497230052948,
      "logits/rejected": -0.15029285848140717,
      "logps/chosen": -596.5925903320312,
      "logps/rejected": -385.3032531738281,
      "loss": 0.1241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1889074295759201,
      "rewards/margins": 2.627173900604248,
      "rewards/rejected": -2.8160810470581055,
      "step": 9203
    },
    {
      "epoch": 1.204400098143453,
      "grad_norm": 11.588497161865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020097939297556877,
      "logits/rejected": -0.11658966541290283,
      "logps/chosen": -378.2405090332031,
      "logps/rejected": -384.5067138671875,
      "loss": 0.7862,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.9528554677963257,
      "rewards/margins": 0.584412693977356,
      "rewards/rejected": -2.5372681617736816,
      "step": 9204
    },
    {
      "epoch": 1.2045309560808048,
      "grad_norm": 8.984038352966309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.044647589325904846,
      "logits/rejected": -0.27539974451065063,
      "logps/chosen": -533.4567260742188,
      "logps/rejected": -477.4063415527344,
      "loss": 0.3802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7982301712036133,
      "rewards/margins": 1.471787929534912,
      "rewards/rejected": -3.2700181007385254,
      "step": 9205
    },
    {
      "epoch": 1.2046618140181566,
      "grad_norm": 7.304511070251465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3019292950630188,
      "logits/rejected": -0.10342633724212646,
      "logps/chosen": -532.6459350585938,
      "logps/rejected": -406.65594482421875,
      "loss": 0.3386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3620498776435852,
      "rewards/margins": 1.7216297388076782,
      "rewards/rejected": -2.083679676055908,
      "step": 9206
    },
    {
      "epoch": 1.2047926719555082,
      "grad_norm": 6.7311601638793945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16440147161483765,
      "logits/rejected": -0.33221399784088135,
      "logps/chosen": -432.2942810058594,
      "logps/rejected": -433.04296875,
      "loss": 0.3335,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0016690492630005,
      "rewards/margins": 1.9435780048370361,
      "rewards/rejected": -2.945246934890747,
      "step": 9207
    },
    {
      "epoch": 1.20492352989286,
      "grad_norm": 10.55970287322998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06427992880344391,
      "logits/rejected": -0.2956763803958893,
      "logps/chosen": -373.97686767578125,
      "logps/rejected": -356.2496032714844,
      "loss": 0.656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.156712293624878,
      "rewards/margins": 0.8071199059486389,
      "rewards/rejected": -1.9638322591781616,
      "step": 9208
    },
    {
      "epoch": 1.2050543878302118,
      "grad_norm": 8.021260261535645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024922681972384453,
      "logits/rejected": -0.011889835819602013,
      "logps/chosen": -673.1243286132812,
      "logps/rejected": -620.2885131835938,
      "loss": 0.3751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3908565640449524,
      "rewards/margins": 1.5781829357147217,
      "rewards/rejected": -1.9690395593643188,
      "step": 9209
    },
    {
      "epoch": 1.2051852457675636,
      "grad_norm": 8.262786865234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02145989239215851,
      "logits/rejected": -0.10775823891162872,
      "logps/chosen": -318.2671813964844,
      "logps/rejected": -311.32537841796875,
      "loss": 0.5951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2398589849472046,
      "rewards/margins": 0.6970691680908203,
      "rewards/rejected": -1.9369282722473145,
      "step": 9210
    },
    {
      "epoch": 1.2053161037049154,
      "grad_norm": 9.842546463012695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3217422664165497,
      "logits/rejected": 0.07666023820638657,
      "logps/chosen": -293.5871276855469,
      "logps/rejected": -329.1694030761719,
      "loss": 0.6094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3462891578674316,
      "rewards/margins": 0.519152045249939,
      "rewards/rejected": -2.865441083908081,
      "step": 9211
    },
    {
      "epoch": 1.2054469616422672,
      "grad_norm": 4.043263912200928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10092675685882568,
      "logits/rejected": -0.24419023096561432,
      "logps/chosen": -382.43408203125,
      "logps/rejected": -331.4309997558594,
      "loss": 0.1923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43146151304244995,
      "rewards/margins": 2.105029821395874,
      "rewards/rejected": -2.5364913940429688,
      "step": 9212
    },
    {
      "epoch": 1.205577819579619,
      "grad_norm": 7.248090744018555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0558805912733078,
      "logits/rejected": -0.041804395616054535,
      "logps/chosen": -389.4984130859375,
      "logps/rejected": -365.9249267578125,
      "loss": 0.2912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4038656949996948,
      "rewards/margins": 1.856722354888916,
      "rewards/rejected": -3.2605879306793213,
      "step": 9213
    },
    {
      "epoch": 1.2057086775169705,
      "grad_norm": 13.148943901062012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4106405973434448,
      "logits/rejected": -0.4696362614631653,
      "logps/chosen": -417.3155212402344,
      "logps/rejected": -307.3438415527344,
      "loss": 0.4779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6646987199783325,
      "rewards/margins": 1.4401302337646484,
      "rewards/rejected": -3.1048290729522705,
      "step": 9214
    },
    {
      "epoch": 1.2058395354543223,
      "grad_norm": 5.148421287536621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23353226482868195,
      "logits/rejected": 0.009647713974118233,
      "logps/chosen": -266.9120788574219,
      "logps/rejected": -426.85052490234375,
      "loss": 0.2457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3868623971939087,
      "rewards/margins": 2.036674737930298,
      "rewards/rejected": -2.423537254333496,
      "step": 9215
    },
    {
      "epoch": 1.2059703933916741,
      "grad_norm": 8.059337615966797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10462217032909393,
      "logits/rejected": 0.0017584264278411865,
      "logps/chosen": -667.762939453125,
      "logps/rejected": -764.3565063476562,
      "loss": 0.2679,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25684472918510437,
      "rewards/margins": 1.9821279048919678,
      "rewards/rejected": -2.2389724254608154,
      "step": 9216
    },
    {
      "epoch": 1.206101251329026,
      "grad_norm": 9.82600212097168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1869811862707138,
      "logits/rejected": -0.14197105169296265,
      "logps/chosen": -439.7177734375,
      "logps/rejected": -519.8643798828125,
      "loss": 0.4852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6132391095161438,
      "rewards/margins": 1.717494249343872,
      "rewards/rejected": -2.330733299255371,
      "step": 9217
    },
    {
      "epoch": 1.2062321092663777,
      "grad_norm": 8.751815795898438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06159524619579315,
      "logits/rejected": -0.14702460169792175,
      "logps/chosen": -462.1457214355469,
      "logps/rejected": -396.4168395996094,
      "loss": 0.4808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1820807456970215,
      "rewards/margins": 1.020440697669983,
      "rewards/rejected": -2.202521562576294,
      "step": 9218
    },
    {
      "epoch": 1.2063629672037295,
      "grad_norm": 4.096057415008545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27872973680496216,
      "logits/rejected": -0.2281932830810547,
      "logps/chosen": -315.5356140136719,
      "logps/rejected": -432.50537109375,
      "loss": 0.2179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40072423219680786,
      "rewards/margins": 2.1462206840515137,
      "rewards/rejected": -2.546945095062256,
      "step": 9219
    },
    {
      "epoch": 1.2064938251410813,
      "grad_norm": 10.715766906738281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6756581664085388,
      "logits/rejected": -0.3490088880062103,
      "logps/chosen": -510.545166015625,
      "logps/rejected": -325.4694519042969,
      "loss": 0.34,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9720577001571655,
      "rewards/margins": 1.7406935691833496,
      "rewards/rejected": -2.7127511501312256,
      "step": 9220
    },
    {
      "epoch": 1.206624683078433,
      "grad_norm": 5.602994441986084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028091153129935265,
      "logits/rejected": -0.07196465879678726,
      "logps/chosen": -446.45166015625,
      "logps/rejected": -450.4223937988281,
      "loss": 0.3204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.421303391456604,
      "rewards/margins": 1.806410312652588,
      "rewards/rejected": -2.2277138233184814,
      "step": 9221
    },
    {
      "epoch": 1.2067555410157849,
      "grad_norm": 9.401558876037598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19080586731433868,
      "logits/rejected": -0.3187044560909271,
      "logps/chosen": -342.08648681640625,
      "logps/rejected": -404.1304016113281,
      "loss": 0.2692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6846286654472351,
      "rewards/margins": 2.157553195953369,
      "rewards/rejected": -2.842181921005249,
      "step": 9222
    },
    {
      "epoch": 1.2068863989531364,
      "grad_norm": 9.505152702331543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2953684628009796,
      "logits/rejected": -0.22674305737018585,
      "logps/chosen": -373.3164367675781,
      "logps/rejected": -494.8174133300781,
      "loss": 0.5576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0556753873825073,
      "rewards/margins": 1.2236934900283813,
      "rewards/rejected": -2.2793688774108887,
      "step": 9223
    },
    {
      "epoch": 1.2070172568904882,
      "grad_norm": 8.966484069824219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24160811305046082,
      "logits/rejected": -0.33257076144218445,
      "logps/chosen": -373.7037048339844,
      "logps/rejected": -449.7232971191406,
      "loss": 0.4786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9071680307388306,
      "rewards/margins": 1.3170843124389648,
      "rewards/rejected": -3.224252223968506,
      "step": 9224
    },
    {
      "epoch": 1.20714811482784,
      "grad_norm": 5.221383571624756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04891036078333855,
      "logits/rejected": 0.008616358041763306,
      "logps/chosen": -403.21533203125,
      "logps/rejected": -358.7557678222656,
      "loss": 0.3249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1537729501724243,
      "rewards/margins": 1.4891045093536377,
      "rewards/rejected": -2.6428773403167725,
      "step": 9225
    },
    {
      "epoch": 1.2072789727651918,
      "grad_norm": 10.352851867675781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18152859807014465,
      "logits/rejected": 0.07125190645456314,
      "logps/chosen": -519.487548828125,
      "logps/rejected": -549.3946533203125,
      "loss": 0.4598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5408753156661987,
      "rewards/margins": 1.2468117475509644,
      "rewards/rejected": -1.787687063217163,
      "step": 9226
    },
    {
      "epoch": 1.2074098307025436,
      "grad_norm": 12.074286460876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09719306230545044,
      "logits/rejected": -0.3367519974708557,
      "logps/chosen": -379.2510070800781,
      "logps/rejected": -446.766845703125,
      "loss": 0.9131,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9049394130706787,
      "rewards/margins": 0.9083583354949951,
      "rewards/rejected": -2.8132975101470947,
      "step": 9227
    },
    {
      "epoch": 1.2075406886398954,
      "grad_norm": 4.9953293800354,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19353243708610535,
      "logits/rejected": -0.3071802854537964,
      "logps/chosen": -500.8167724609375,
      "logps/rejected": -437.0028381347656,
      "loss": 0.2379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1831082105636597,
      "rewards/margins": 2.344813585281372,
      "rewards/rejected": -3.5279219150543213,
      "step": 9228
    },
    {
      "epoch": 1.207671546577247,
      "grad_norm": 2.1908931732177734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2183794230222702,
      "logits/rejected": -0.5592819452285767,
      "logps/chosen": -455.5369873046875,
      "logps/rejected": -373.51812744140625,
      "loss": 0.1614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6923611164093018,
      "rewards/margins": 2.347524404525757,
      "rewards/rejected": -3.0398857593536377,
      "step": 9229
    },
    {
      "epoch": 1.2078024045145987,
      "grad_norm": 6.729604244232178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3416484296321869,
      "logits/rejected": -0.4319742023944855,
      "logps/chosen": -634.593505859375,
      "logps/rejected": -624.9376220703125,
      "loss": 0.3082,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49480366706848145,
      "rewards/margins": 1.6136330366134644,
      "rewards/rejected": -2.1084365844726562,
      "step": 9230
    },
    {
      "epoch": 1.2079332624519505,
      "grad_norm": 20.69940757751465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24083510041236877,
      "logits/rejected": -0.28954434394836426,
      "logps/chosen": -458.75970458984375,
      "logps/rejected": -351.41748046875,
      "loss": 0.6814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0913636684417725,
      "rewards/margins": 1.85781991481781,
      "rewards/rejected": -2.949183702468872,
      "step": 9231
    },
    {
      "epoch": 1.2080641203893023,
      "grad_norm": 4.676621437072754,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02473277971148491,
      "logits/rejected": -0.2680540680885315,
      "logps/chosen": -382.25390625,
      "logps/rejected": -371.59716796875,
      "loss": 0.2193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28462639451026917,
      "rewards/margins": 2.3783071041107178,
      "rewards/rejected": -2.662933349609375,
      "step": 9232
    },
    {
      "epoch": 1.2081949783266541,
      "grad_norm": 14.589303970336914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11894887685775757,
      "logits/rejected": -0.2848662734031677,
      "logps/chosen": -534.746337890625,
      "logps/rejected": -648.9573974609375,
      "loss": 0.7772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5750536918640137,
      "rewards/margins": 0.9944925308227539,
      "rewards/rejected": -2.5695459842681885,
      "step": 9233
    },
    {
      "epoch": 1.208325836264006,
      "grad_norm": 8.853392601013184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.142747163772583,
      "logits/rejected": -0.1678161919116974,
      "logps/chosen": -387.9256591796875,
      "logps/rejected": -415.02239990234375,
      "loss": 0.4504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9270299673080444,
      "rewards/margins": 1.561195969581604,
      "rewards/rejected": -2.4882256984710693,
      "step": 9234
    },
    {
      "epoch": 1.2084566942013577,
      "grad_norm": 8.747795104980469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19751837849617004,
      "logits/rejected": 0.09195829182863235,
      "logps/chosen": -459.5876770019531,
      "logps/rejected": -524.3447875976562,
      "loss": 0.4441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7843123078346252,
      "rewards/margins": 1.3611412048339844,
      "rewards/rejected": -2.145453691482544,
      "step": 9235
    },
    {
      "epoch": 1.2085875521387095,
      "grad_norm": 6.957765102386475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2399061769247055,
      "logits/rejected": -0.20805516839027405,
      "logps/chosen": -347.9012451171875,
      "logps/rejected": -390.2937316894531,
      "loss": 0.4253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7434812188148499,
      "rewards/margins": 1.3750739097595215,
      "rewards/rejected": -2.1185553073883057,
      "step": 9236
    },
    {
      "epoch": 1.2087184100760613,
      "grad_norm": 4.991465091705322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.226710706949234,
      "logits/rejected": -0.08645504713058472,
      "logps/chosen": -495.23541259765625,
      "logps/rejected": -525.8753662109375,
      "loss": 0.2209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3016887605190277,
      "rewards/margins": 2.314962863922119,
      "rewards/rejected": -2.6166515350341797,
      "step": 9237
    },
    {
      "epoch": 1.2088492680134129,
      "grad_norm": 10.44563102722168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026822464540600777,
      "logits/rejected": 0.012552425265312195,
      "logps/chosen": -486.5955810546875,
      "logps/rejected": -384.8249206542969,
      "loss": 0.7886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.36062029004096985,
      "rewards/margins": 0.19359436631202698,
      "rewards/rejected": -0.5542146563529968,
      "step": 9238
    },
    {
      "epoch": 1.2089801259507647,
      "grad_norm": 10.149128913879395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04039746895432472,
      "logits/rejected": -0.3510797917842865,
      "logps/chosen": -481.1007385253906,
      "logps/rejected": -730.4144287109375,
      "loss": 0.4461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6173601150512695,
      "rewards/margins": 2.1220009326934814,
      "rewards/rejected": -3.739360809326172,
      "step": 9239
    },
    {
      "epoch": 1.2091109838881164,
      "grad_norm": 11.434761047363281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2814040780067444,
      "logits/rejected": -0.21246308088302612,
      "logps/chosen": -508.34002685546875,
      "logps/rejected": -444.81622314453125,
      "loss": 0.5866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.522409200668335,
      "rewards/margins": 0.7233927249908447,
      "rewards/rejected": -2.2458016872406006,
      "step": 9240
    },
    {
      "epoch": 1.2092418418254682,
      "grad_norm": 7.255725860595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3267204165458679,
      "logits/rejected": -0.29206037521362305,
      "logps/chosen": -470.7969055175781,
      "logps/rejected": -515.014892578125,
      "loss": 0.3066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1758941411972046,
      "rewards/margins": 1.9386959075927734,
      "rewards/rejected": -3.1145899295806885,
      "step": 9241
    },
    {
      "epoch": 1.20937269976282,
      "grad_norm": 9.768235206604004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06278002262115479,
      "logits/rejected": 0.13330572843551636,
      "logps/chosen": -418.218994140625,
      "logps/rejected": -502.4580078125,
      "loss": 0.577,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8769848346710205,
      "rewards/margins": 1.2282885313034058,
      "rewards/rejected": -3.1052732467651367,
      "step": 9242
    },
    {
      "epoch": 1.2095035577001718,
      "grad_norm": 3.9085378646850586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3850741386413574,
      "logits/rejected": -0.5753399133682251,
      "logps/chosen": -412.736572265625,
      "logps/rejected": -522.2415161132812,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8515121936798096,
      "rewards/margins": 3.103039503097534,
      "rewards/rejected": -3.9545514583587646,
      "step": 9243
    },
    {
      "epoch": 1.2096344156375236,
      "grad_norm": 6.537219524383545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1183406338095665,
      "logits/rejected": -0.049894094467163086,
      "logps/chosen": -393.1685791015625,
      "logps/rejected": -441.47369384765625,
      "loss": 0.2632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7943226099014282,
      "rewards/margins": 1.8980066776275635,
      "rewards/rejected": -2.692329168319702,
      "step": 9244
    },
    {
      "epoch": 1.2097652735748752,
      "grad_norm": 4.712935924530029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2789158821105957,
      "logits/rejected": -0.41684794425964355,
      "logps/chosen": -407.7381591796875,
      "logps/rejected": -424.67462158203125,
      "loss": 0.2044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8734421133995056,
      "rewards/margins": 1.921457052230835,
      "rewards/rejected": -2.7948989868164062,
      "step": 9245
    },
    {
      "epoch": 1.209896131512227,
      "grad_norm": 8.088510513305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1458674520254135,
      "logits/rejected": -0.15474121272563934,
      "logps/chosen": -378.2238464355469,
      "logps/rejected": -323.1890869140625,
      "loss": 0.5381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.071035623550415,
      "rewards/margins": 0.60953688621521,
      "rewards/rejected": -1.6805726289749146,
      "step": 9246
    },
    {
      "epoch": 1.2100269894495788,
      "grad_norm": 6.621166706085205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07760966569185257,
      "logits/rejected": 0.09752075374126434,
      "logps/chosen": -297.2153625488281,
      "logps/rejected": -429.81097412109375,
      "loss": 0.2845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6910762190818787,
      "rewards/margins": 2.2762792110443115,
      "rewards/rejected": -2.967355489730835,
      "step": 9247
    },
    {
      "epoch": 1.2101578473869306,
      "grad_norm": 14.770817756652832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3709678053855896,
      "logits/rejected": -0.27839311957359314,
      "logps/chosen": -356.6302490234375,
      "logps/rejected": -325.94818115234375,
      "loss": 0.9141,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.9358155727386475,
      "rewards/margins": -0.04848453402519226,
      "rewards/rejected": -2.887331247329712,
      "step": 9248
    },
    {
      "epoch": 1.2102887053242823,
      "grad_norm": 14.626106262207031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.348694384098053,
      "logits/rejected": -0.36779528856277466,
      "logps/chosen": -385.1510925292969,
      "logps/rejected": -424.814697265625,
      "loss": 0.592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2696430683135986,
      "rewards/margins": 1.4710392951965332,
      "rewards/rejected": -2.740682363510132,
      "step": 9249
    },
    {
      "epoch": 1.2104195632616341,
      "grad_norm": 7.623178958892822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16660115122795105,
      "logits/rejected": -0.18140628933906555,
      "logps/chosen": -417.32489013671875,
      "logps/rejected": -427.472900390625,
      "loss": 0.4589,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2594300508499146,
      "rewards/margins": 1.0427789688110352,
      "rewards/rejected": -2.302208662033081,
      "step": 9250
    },
    {
      "epoch": 1.210550421198986,
      "grad_norm": 10.775896072387695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25692254304885864,
      "logits/rejected": -0.2749042809009552,
      "logps/chosen": -434.80548095703125,
      "logps/rejected": -458.61260986328125,
      "loss": 0.6723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.418581247329712,
      "rewards/margins": 0.24013160169124603,
      "rewards/rejected": -2.658712863922119,
      "step": 9251
    },
    {
      "epoch": 1.2106812791363377,
      "grad_norm": 12.446000099182129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02852948009967804,
      "logits/rejected": -0.17687620222568512,
      "logps/chosen": -741.4490966796875,
      "logps/rejected": -387.26617431640625,
      "loss": 0.5032,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.253208875656128,
      "rewards/margins": 1.4407825469970703,
      "rewards/rejected": -2.6939914226531982,
      "step": 9252
    },
    {
      "epoch": 1.2108121370736893,
      "grad_norm": 4.967015743255615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.062103234231472015,
      "logits/rejected": -0.07370978593826294,
      "logps/chosen": -364.7035827636719,
      "logps/rejected": -518.8275146484375,
      "loss": 0.2572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8158137202262878,
      "rewards/margins": 2.0507378578186035,
      "rewards/rejected": -2.866551399230957,
      "step": 9253
    },
    {
      "epoch": 1.210942995011041,
      "grad_norm": 8.593263626098633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09725616127252579,
      "logits/rejected": -0.24138018488883972,
      "logps/chosen": -531.513427734375,
      "logps/rejected": -639.5235595703125,
      "loss": 0.4998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7033700942993164,
      "rewards/margins": 1.147341012954712,
      "rewards/rejected": -2.850710868835449,
      "step": 9254
    },
    {
      "epoch": 1.2110738529483929,
      "grad_norm": 3.6564228534698486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040788933634757996,
      "logits/rejected": -0.19338512420654297,
      "logps/chosen": -600.4613037109375,
      "logps/rejected": -374.28765869140625,
      "loss": 0.3157,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13079698383808136,
      "rewards/margins": 2.0749449729919434,
      "rewards/rejected": -2.205742120742798,
      "step": 9255
    },
    {
      "epoch": 1.2112047108857447,
      "grad_norm": 7.474168300628662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2568185031414032,
      "logits/rejected": -0.26736024022102356,
      "logps/chosen": -536.5654296875,
      "logps/rejected": -638.378662109375,
      "loss": 0.3926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3014768362045288,
      "rewards/margins": 1.4767988920211792,
      "rewards/rejected": -2.778275728225708,
      "step": 9256
    },
    {
      "epoch": 1.2113355688230965,
      "grad_norm": 11.399465560913086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40416014194488525,
      "logits/rejected": -0.0005077347159385681,
      "logps/chosen": -371.31634521484375,
      "logps/rejected": -460.3212890625,
      "loss": 0.7198,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9494619369506836,
      "rewards/margins": 1.3634346723556519,
      "rewards/rejected": -2.312896966934204,
      "step": 9257
    },
    {
      "epoch": 1.2114664267604482,
      "grad_norm": 3.229074478149414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22645986080169678,
      "logits/rejected": -0.1646800935268402,
      "logps/chosen": -374.6903076171875,
      "logps/rejected": -486.5847473144531,
      "loss": 0.1195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.037081003189087,
      "rewards/margins": 3.3008103370666504,
      "rewards/rejected": -4.337891101837158,
      "step": 9258
    },
    {
      "epoch": 1.2115972846978,
      "grad_norm": 6.956727027893066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4005832374095917,
      "logits/rejected": -0.5890839695930481,
      "logps/chosen": -310.69427490234375,
      "logps/rejected": -418.0137939453125,
      "loss": 0.4005,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5434222221374512,
      "rewards/margins": 1.2659049034118652,
      "rewards/rejected": -2.8093271255493164,
      "step": 9259
    },
    {
      "epoch": 1.2117281426351516,
      "grad_norm": 9.697245597839355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025640955194830894,
      "logits/rejected": -0.17987893521785736,
      "logps/chosen": -422.84686279296875,
      "logps/rejected": -452.0231018066406,
      "loss": 0.5109,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9947935342788696,
      "rewards/margins": 0.6404703855514526,
      "rewards/rejected": -2.6352639198303223,
      "step": 9260
    },
    {
      "epoch": 1.2118590005725034,
      "grad_norm": 13.901728630065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41254928708076477,
      "logits/rejected": -0.29268383979797363,
      "logps/chosen": -496.2737731933594,
      "logps/rejected": -495.9727478027344,
      "loss": 0.8275,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.1826276779174805,
      "rewards/margins": 0.3679609000682831,
      "rewards/rejected": -2.550588607788086,
      "step": 9261
    },
    {
      "epoch": 1.2119898585098552,
      "grad_norm": 6.644183158874512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41328081488609314,
      "logits/rejected": -0.36848777532577515,
      "logps/chosen": -410.081787109375,
      "logps/rejected": -506.0791015625,
      "loss": 0.3233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.224490165710449,
      "rewards/margins": 1.206681728363037,
      "rewards/rejected": -3.4311718940734863,
      "step": 9262
    },
    {
      "epoch": 1.212120716447207,
      "grad_norm": 4.313108444213867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15172582864761353,
      "logits/rejected": -0.33589333295822144,
      "logps/chosen": -462.0454406738281,
      "logps/rejected": -401.64501953125,
      "loss": 0.1289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.796947181224823,
      "rewards/margins": 2.728851795196533,
      "rewards/rejected": -3.525798797607422,
      "step": 9263
    },
    {
      "epoch": 1.2122515743845588,
      "grad_norm": 4.341005802154541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3393841087818146,
      "logits/rejected": -0.27745160460472107,
      "logps/chosen": -368.8387756347656,
      "logps/rejected": -482.491455078125,
      "loss": 0.1611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6899359822273254,
      "rewards/margins": 2.4532229900360107,
      "rewards/rejected": -3.1431591510772705,
      "step": 9264
    },
    {
      "epoch": 1.2123824323219106,
      "grad_norm": 14.431289672851562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14763599634170532,
      "logits/rejected": -0.21017658710479736,
      "logps/chosen": -449.71893310546875,
      "logps/rejected": -410.21820068359375,
      "loss": 0.8534,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.852545976638794,
      "rewards/margins": 0.013969838619232178,
      "rewards/rejected": -1.8665156364440918,
      "step": 9265
    },
    {
      "epoch": 1.2125132902592624,
      "grad_norm": 10.6842041015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10187844187021255,
      "logits/rejected": -0.18993115425109863,
      "logps/chosen": -539.0077514648438,
      "logps/rejected": -462.5750427246094,
      "loss": 0.5326,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5174940824508667,
      "rewards/margins": 2.0248053073883057,
      "rewards/rejected": -3.542299270629883,
      "step": 9266
    },
    {
      "epoch": 1.2126441481966141,
      "grad_norm": 13.603145599365234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5072236657142639,
      "logits/rejected": -0.07471520453691483,
      "logps/chosen": -505.9045715332031,
      "logps/rejected": -556.1159057617188,
      "loss": 0.848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5010838508605957,
      "rewards/margins": 0.015373200178146362,
      "rewards/rejected": -1.5164570808410645,
      "step": 9267
    },
    {
      "epoch": 1.212775006133966,
      "grad_norm": 4.393695831298828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23471662402153015,
      "logits/rejected": -0.2747752368450165,
      "logps/chosen": -652.8091430664062,
      "logps/rejected": -405.76995849609375,
      "loss": 0.2546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11780967563390732,
      "rewards/margins": 2.188527822494507,
      "rewards/rejected": -2.07071852684021,
      "step": 9268
    },
    {
      "epoch": 1.2129058640713175,
      "grad_norm": 11.449735641479492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33135467767715454,
      "logits/rejected": -0.34688323736190796,
      "logps/chosen": -387.48126220703125,
      "logps/rejected": -388.1033630371094,
      "loss": 0.685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9065622091293335,
      "rewards/margins": 1.773612141609192,
      "rewards/rejected": -2.6801743507385254,
      "step": 9269
    },
    {
      "epoch": 1.2130367220086693,
      "grad_norm": 5.153769016265869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16652220487594604,
      "logits/rejected": -0.1992511749267578,
      "logps/chosen": -546.8621826171875,
      "logps/rejected": -571.283935546875,
      "loss": 0.231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4109740257263184,
      "rewards/margins": 1.8040735721588135,
      "rewards/rejected": -3.2150473594665527,
      "step": 9270
    },
    {
      "epoch": 1.213167579946021,
      "grad_norm": 6.7210283279418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04529301077127457,
      "logits/rejected": -0.35456258058547974,
      "logps/chosen": -428.27691650390625,
      "logps/rejected": -546.7379760742188,
      "loss": 0.2718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9508768320083618,
      "rewards/margins": 2.433844804763794,
      "rewards/rejected": -3.3847217559814453,
      "step": 9271
    },
    {
      "epoch": 1.2132984378833729,
      "grad_norm": 10.194433212280273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02577156201004982,
      "logits/rejected": -0.2775748074054718,
      "logps/chosen": -393.91180419921875,
      "logps/rejected": -560.296630859375,
      "loss": 0.6003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6916759014129639,
      "rewards/margins": 1.1671864986419678,
      "rewards/rejected": -2.8588621616363525,
      "step": 9272
    },
    {
      "epoch": 1.2134292958207247,
      "grad_norm": 4.87100887298584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27121007442474365,
      "logits/rejected": -0.2163373827934265,
      "logps/chosen": -469.84722900390625,
      "logps/rejected": -450.8369140625,
      "loss": 0.2285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0241563320159912,
      "rewards/margins": 2.3107144832611084,
      "rewards/rejected": -3.3348710536956787,
      "step": 9273
    },
    {
      "epoch": 1.2135601537580765,
      "grad_norm": 5.542622089385986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30708611011505127,
      "logits/rejected": -0.39429405331611633,
      "logps/chosen": -471.2467956542969,
      "logps/rejected": -427.18023681640625,
      "loss": 0.2022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9161255359649658,
      "rewards/margins": 2.4028162956237793,
      "rewards/rejected": -3.318941354751587,
      "step": 9274
    },
    {
      "epoch": 1.213691011695428,
      "grad_norm": 4.092328071594238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3090278208255768,
      "logits/rejected": -0.315480500459671,
      "logps/chosen": -506.3951110839844,
      "logps/rejected": -507.4649658203125,
      "loss": 0.176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5049293041229248,
      "rewards/margins": 2.0005664825439453,
      "rewards/rejected": -3.50549578666687,
      "step": 9275
    },
    {
      "epoch": 1.2138218696327798,
      "grad_norm": 4.025391101837158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2733246684074402,
      "logits/rejected": -0.593334972858429,
      "logps/chosen": -502.8703308105469,
      "logps/rejected": -445.0352783203125,
      "loss": 0.1698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3128347396850586,
      "rewards/margins": 2.1841447353363037,
      "rewards/rejected": -3.4969794750213623,
      "step": 9276
    },
    {
      "epoch": 1.2139527275701316,
      "grad_norm": 5.118885517120361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2541314661502838,
      "logits/rejected": -0.2837522625923157,
      "logps/chosen": -580.28466796875,
      "logps/rejected": -541.7235717773438,
      "loss": 0.2829,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4199724197387695,
      "rewards/margins": 1.6735248565673828,
      "rewards/rejected": -3.0934972763061523,
      "step": 9277
    },
    {
      "epoch": 1.2140835855074834,
      "grad_norm": 8.065160751342773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2363840937614441,
      "logits/rejected": -0.09632895886898041,
      "logps/chosen": -495.5186767578125,
      "logps/rejected": -378.512939453125,
      "loss": 0.5478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.465691328048706,
      "rewards/margins": 0.725725531578064,
      "rewards/rejected": -2.1914167404174805,
      "step": 9278
    },
    {
      "epoch": 1.2142144434448352,
      "grad_norm": 14.00684928894043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19792449474334717,
      "logits/rejected": -0.08263374865055084,
      "logps/chosen": -360.2597961425781,
      "logps/rejected": -375.320556640625,
      "loss": 0.7574,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.651601552963257,
      "rewards/margins": 0.42594924569129944,
      "rewards/rejected": -3.0775511264801025,
      "step": 9279
    },
    {
      "epoch": 1.214345301382187,
      "grad_norm": 3.198485851287842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20741435885429382,
      "logits/rejected": -0.3384207487106323,
      "logps/chosen": -389.7026062011719,
      "logps/rejected": -509.74462890625,
      "loss": 0.2431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6229116916656494,
      "rewards/margins": 2.336242198944092,
      "rewards/rejected": -3.959153652191162,
      "step": 9280
    },
    {
      "epoch": 1.2144761593195388,
      "grad_norm": 11.716005325317383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34360623359680176,
      "logits/rejected": -0.49913373589515686,
      "logps/chosen": -398.4474792480469,
      "logps/rejected": -383.41998291015625,
      "loss": 0.6721,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4264352321624756,
      "rewards/margins": 1.3111836910247803,
      "rewards/rejected": -2.737618923187256,
      "step": 9281
    },
    {
      "epoch": 1.2146070172568906,
      "grad_norm": 2.9452900886535645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1410049945116043,
      "logits/rejected": -0.21986152231693268,
      "logps/chosen": -682.3822021484375,
      "logps/rejected": -780.9088134765625,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3391155004501343,
      "rewards/margins": 3.1232850551605225,
      "rewards/rejected": -4.462400913238525,
      "step": 9282
    },
    {
      "epoch": 1.2147378751942424,
      "grad_norm": 8.421390533447266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15485888719558716,
      "logits/rejected": -0.41683608293533325,
      "logps/chosen": -428.7786865234375,
      "logps/rejected": -292.5877990722656,
      "loss": 0.345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7608551979064941,
      "rewards/margins": 1.5885916948318481,
      "rewards/rejected": -3.3494467735290527,
      "step": 9283
    },
    {
      "epoch": 1.214868733131594,
      "grad_norm": 10.921539306640625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04209743067622185,
      "logits/rejected": -0.375019907951355,
      "logps/chosen": -418.0328063964844,
      "logps/rejected": -410.6468811035156,
      "loss": 0.4649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6444213390350342,
      "rewards/margins": 1.5621330738067627,
      "rewards/rejected": -3.206554412841797,
      "step": 9284
    },
    {
      "epoch": 1.2149995910689457,
      "grad_norm": 9.434382438659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2706874907016754,
      "logits/rejected": -0.2613435983657837,
      "logps/chosen": -443.7386474609375,
      "logps/rejected": -534.8612060546875,
      "loss": 0.5595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8373568058013916,
      "rewards/margins": 1.3296236991882324,
      "rewards/rejected": -3.166980743408203,
      "step": 9285
    },
    {
      "epoch": 1.2151304490062975,
      "grad_norm": 8.415574073791504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18551698327064514,
      "logits/rejected": -0.12380945682525635,
      "logps/chosen": -372.3305969238281,
      "logps/rejected": -344.6978454589844,
      "loss": 0.517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8436172604560852,
      "rewards/margins": 2.3019754886627197,
      "rewards/rejected": -3.14559268951416,
      "step": 9286
    },
    {
      "epoch": 1.2152613069436493,
      "grad_norm": 11.099834442138672,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0995580181479454,
      "logits/rejected": -0.27617231011390686,
      "logps/chosen": -497.6636657714844,
      "logps/rejected": -452.34967041015625,
      "loss": 0.4624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1758646965026855,
      "rewards/margins": 1.460662841796875,
      "rewards/rejected": -2.6365275382995605,
      "step": 9287
    },
    {
      "epoch": 1.215392164881001,
      "grad_norm": 7.603940010070801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2288212776184082,
      "logits/rejected": -0.18281933665275574,
      "logps/chosen": -337.54052734375,
      "logps/rejected": -442.00439453125,
      "loss": 0.4051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0493276119232178,
      "rewards/margins": 2.083867073059082,
      "rewards/rejected": -4.133194446563721,
      "step": 9288
    },
    {
      "epoch": 1.215523022818353,
      "grad_norm": 8.531442642211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1359771490097046,
      "logits/rejected": -0.48899543285369873,
      "logps/chosen": -440.4366149902344,
      "logps/rejected": -463.9993896484375,
      "loss": 0.7298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.559313178062439,
      "rewards/margins": 0.7343675494194031,
      "rewards/rejected": -2.2936806678771973,
      "step": 9289
    },
    {
      "epoch": 1.2156538807557045,
      "grad_norm": 13.854405403137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2749216556549072,
      "logits/rejected": -0.4228372871875763,
      "logps/chosen": -541.2692260742188,
      "logps/rejected": -492.0135498046875,
      "loss": 0.4265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2429535388946533,
      "rewards/margins": 2.058840751647949,
      "rewards/rejected": -4.301794052124023,
      "step": 9290
    },
    {
      "epoch": 1.2157847386930563,
      "grad_norm": 8.342996597290039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07225148379802704,
      "logits/rejected": 0.0004563611000776291,
      "logps/chosen": -362.8362121582031,
      "logps/rejected": -477.717041015625,
      "loss": 0.4218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1612480878829956,
      "rewards/margins": 1.2082831859588623,
      "rewards/rejected": -2.3695313930511475,
      "step": 9291
    },
    {
      "epoch": 1.215915596630408,
      "grad_norm": 7.5422234535217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17780154943466187,
      "logits/rejected": -0.17394831776618958,
      "logps/chosen": -390.4815368652344,
      "logps/rejected": -421.7554626464844,
      "loss": 0.3727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9396341443061829,
      "rewards/margins": 1.6282358169555664,
      "rewards/rejected": -2.5678701400756836,
      "step": 9292
    },
    {
      "epoch": 1.2160464545677598,
      "grad_norm": 10.424140930175781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08057849109172821,
      "logits/rejected": 0.13772700726985931,
      "logps/chosen": -358.5408630371094,
      "logps/rejected": -416.1398620605469,
      "loss": 0.6466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7116596698760986,
      "rewards/margins": 1.2849960327148438,
      "rewards/rejected": -1.9966557025909424,
      "step": 9293
    },
    {
      "epoch": 1.2161773125051116,
      "grad_norm": 15.95025634765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26767870783805847,
      "logits/rejected": -0.07797817885875702,
      "logps/chosen": -491.468994140625,
      "logps/rejected": -403.2840881347656,
      "loss": 0.9631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.626463532447815,
      "rewards/margins": 0.23661312460899353,
      "rewards/rejected": -1.8630765676498413,
      "step": 9294
    },
    {
      "epoch": 1.2163081704424634,
      "grad_norm": 10.572104454040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39274224638938904,
      "logits/rejected": -0.2916887402534485,
      "logps/chosen": -469.308349609375,
      "logps/rejected": -459.9647521972656,
      "loss": 0.6393,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4074318408966064,
      "rewards/margins": 0.4543929696083069,
      "rewards/rejected": -1.861824870109558,
      "step": 9295
    },
    {
      "epoch": 1.2164390283798152,
      "grad_norm": 5.194918155670166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2548315227031708,
      "logits/rejected": -0.10898386687040329,
      "logps/chosen": -561.2046508789062,
      "logps/rejected": -491.0364990234375,
      "loss": 0.2692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4154342710971832,
      "rewards/margins": 2.0977749824523926,
      "rewards/rejected": -2.513209342956543,
      "step": 9296
    },
    {
      "epoch": 1.216569886317167,
      "grad_norm": 4.937537670135498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24734947085380554,
      "logits/rejected": -0.22067822515964508,
      "logps/chosen": -401.1670227050781,
      "logps/rejected": -400.1575927734375,
      "loss": 0.217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6049317717552185,
      "rewards/margins": 2.383342742919922,
      "rewards/rejected": -2.9882748126983643,
      "step": 9297
    },
    {
      "epoch": 1.2167007442545188,
      "grad_norm": 6.101619243621826,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10815412551164627,
      "logits/rejected": 0.014489710330963135,
      "logps/chosen": -394.1689758300781,
      "logps/rejected": -421.77154541015625,
      "loss": 0.2816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.039657711982727,
      "rewards/margins": 3.383739709854126,
      "rewards/rejected": -4.423397541046143,
      "step": 9298
    },
    {
      "epoch": 1.2168316021918704,
      "grad_norm": 12.14615535736084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0881289690732956,
      "logits/rejected": -0.1756080538034439,
      "logps/chosen": -438.202880859375,
      "logps/rejected": -481.9170227050781,
      "loss": 0.7385,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9884843826293945,
      "rewards/margins": 0.25692757964134216,
      "rewards/rejected": -2.2454118728637695,
      "step": 9299
    },
    {
      "epoch": 1.2169624601292222,
      "grad_norm": 10.452427864074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28212183713912964,
      "logits/rejected": -0.3451390862464905,
      "logps/chosen": -429.88800048828125,
      "logps/rejected": -519.0317993164062,
      "loss": 0.4315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4917821884155273,
      "rewards/margins": 1.8432996273040771,
      "rewards/rejected": -3.3350815773010254,
      "step": 9300
    },
    {
      "epoch": 1.217093318066574,
      "grad_norm": 8.341718673706055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22243070602416992,
      "logits/rejected": -0.16523361206054688,
      "logps/chosen": -281.8049621582031,
      "logps/rejected": -294.771240234375,
      "loss": 0.4151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5666875839233398,
      "rewards/margins": 1.1402161121368408,
      "rewards/rejected": -2.7069039344787598,
      "step": 9301
    },
    {
      "epoch": 1.2172241760039257,
      "grad_norm": 7.8635759353637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16786041855812073,
      "logits/rejected": -0.15271013975143433,
      "logps/chosen": -355.4239196777344,
      "logps/rejected": -468.2601623535156,
      "loss": 0.4113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3769075870513916,
      "rewards/margins": 1.0580413341522217,
      "rewards/rejected": -2.434948682785034,
      "step": 9302
    },
    {
      "epoch": 1.2173550339412775,
      "grad_norm": 8.394209861755371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015174075961112976,
      "logits/rejected": -0.1458626687526703,
      "logps/chosen": -554.3837280273438,
      "logps/rejected": -502.1986389160156,
      "loss": 0.3389,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.087729573249817,
      "rewards/margins": 1.4737746715545654,
      "rewards/rejected": -2.5615041255950928,
      "step": 9303
    },
    {
      "epoch": 1.2174858918786293,
      "grad_norm": 4.337523937225342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1566316783428192,
      "logits/rejected": -0.343679279088974,
      "logps/chosen": -376.71832275390625,
      "logps/rejected": -456.3070983886719,
      "loss": 0.2235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2598293423652649,
      "rewards/margins": 3.571803092956543,
      "rewards/rejected": -3.831632375717163,
      "step": 9304
    },
    {
      "epoch": 1.2176167498159811,
      "grad_norm": 12.440682411193848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3182808756828308,
      "logits/rejected": -0.3116088807582855,
      "logps/chosen": -555.1943969726562,
      "logps/rejected": -457.324462890625,
      "loss": 0.7,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0035079717636108,
      "rewards/margins": 0.5257064700126648,
      "rewards/rejected": -1.5292145013809204,
      "step": 9305
    },
    {
      "epoch": 1.2177476077533327,
      "grad_norm": 7.937257289886475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39479777216911316,
      "logits/rejected": -0.6590430736541748,
      "logps/chosen": -487.47283935546875,
      "logps/rejected": -511.7158508300781,
      "loss": 0.3595,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5447746515274048,
      "rewards/margins": 1.4631186723709106,
      "rewards/rejected": -3.0078930854797363,
      "step": 9306
    },
    {
      "epoch": 1.2178784656906845,
      "grad_norm": 14.753149032592773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48068973422050476,
      "logits/rejected": -0.3493884801864624,
      "logps/chosen": -447.9535827636719,
      "logps/rejected": -444.0551452636719,
      "loss": 1.3451,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0082037448883057,
      "rewards/margins": -0.4567461311817169,
      "rewards/rejected": -1.5514576435089111,
      "step": 9307
    },
    {
      "epoch": 1.2180093236280363,
      "grad_norm": 7.702273368835449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36637917160987854,
      "logits/rejected": -0.2905060946941376,
      "logps/chosen": -289.2825622558594,
      "logps/rejected": -413.9610290527344,
      "loss": 0.4196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8832216262817383,
      "rewards/margins": 2.1102874279022217,
      "rewards/rejected": -2.99350905418396,
      "step": 9308
    },
    {
      "epoch": 1.218140181565388,
      "grad_norm": 7.37738561630249,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03263501822948456,
      "logits/rejected": -0.30876046419143677,
      "logps/chosen": -479.7586669921875,
      "logps/rejected": -493.8197326660156,
      "loss": 0.3643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5064059495925903,
      "rewards/margins": 2.683013439178467,
      "rewards/rejected": -4.189419269561768,
      "step": 9309
    },
    {
      "epoch": 1.2182710395027398,
      "grad_norm": 11.18039608001709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4705619215965271,
      "logits/rejected": -0.5377694368362427,
      "logps/chosen": -301.6694641113281,
      "logps/rejected": -263.1634826660156,
      "loss": 0.6324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5437946319580078,
      "rewards/margins": 0.39242035150527954,
      "rewards/rejected": -1.936214804649353,
      "step": 9310
    },
    {
      "epoch": 1.2184018974400916,
      "grad_norm": 7.969450950622559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05377473682165146,
      "logits/rejected": -0.19084979593753815,
      "logps/chosen": -474.05096435546875,
      "logps/rejected": -488.4556884765625,
      "loss": 0.4942,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8572816848754883,
      "rewards/margins": 1.9162626266479492,
      "rewards/rejected": -3.7735438346862793,
      "step": 9311
    },
    {
      "epoch": 1.2185327553774434,
      "grad_norm": 10.890395164489746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38241755962371826,
      "logits/rejected": -0.43366968631744385,
      "logps/chosen": -413.6031494140625,
      "logps/rejected": -534.67041015625,
      "loss": 0.5545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.442669153213501,
      "rewards/margins": 1.008878469467163,
      "rewards/rejected": -3.451547622680664,
      "step": 9312
    },
    {
      "epoch": 1.2186636133147952,
      "grad_norm": 8.206975936889648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2067379653453827,
      "logits/rejected": -0.294636070728302,
      "logps/chosen": -473.858642578125,
      "logps/rejected": -399.09027099609375,
      "loss": 0.4294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3813464641571045,
      "rewards/margins": 1.2135260105133057,
      "rewards/rejected": -2.59487247467041,
      "step": 9313
    },
    {
      "epoch": 1.2187944712521468,
      "grad_norm": 9.486565589904785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26486027240753174,
      "logits/rejected": -0.20249910652637482,
      "logps/chosen": -501.525146484375,
      "logps/rejected": -590.36962890625,
      "loss": 0.5529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8005256652832031,
      "rewards/margins": 0.8506249785423279,
      "rewards/rejected": -2.651150703430176,
      "step": 9314
    },
    {
      "epoch": 1.2189253291894986,
      "grad_norm": 16.34390640258789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2041797935962677,
      "logits/rejected": -0.008026525378227234,
      "logps/chosen": -602.46875,
      "logps/rejected": -474.8904724121094,
      "loss": 0.8051,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6874618530273438,
      "rewards/margins": 0.7212809324264526,
      "rewards/rejected": -2.408742904663086,
      "step": 9315
    },
    {
      "epoch": 1.2190561871268504,
      "grad_norm": 8.818044662475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28762274980545044,
      "logits/rejected": -0.10048031061887741,
      "logps/chosen": -505.8238525390625,
      "logps/rejected": -486.1723937988281,
      "loss": 0.519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1094651222229004,
      "rewards/margins": 0.8575023412704468,
      "rewards/rejected": -2.9669675827026367,
      "step": 9316
    },
    {
      "epoch": 1.2191870450642022,
      "grad_norm": 8.616451263427734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10685697197914124,
      "logits/rejected": -0.4018218517303467,
      "logps/chosen": -466.7783203125,
      "logps/rejected": -356.7794494628906,
      "loss": 0.4598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.276139497756958,
      "rewards/margins": 1.0779623985290527,
      "rewards/rejected": -2.3541018962860107,
      "step": 9317
    },
    {
      "epoch": 1.219317903001554,
      "grad_norm": 5.5733771324157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09972697496414185,
      "logits/rejected": -0.23572991788387299,
      "logps/chosen": -334.4725646972656,
      "logps/rejected": -279.0113220214844,
      "loss": 0.3081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8127567172050476,
      "rewards/margins": 1.319533109664917,
      "rewards/rejected": -2.1322898864746094,
      "step": 9318
    },
    {
      "epoch": 1.2194487609389058,
      "grad_norm": 20.274944305419922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12717211246490479,
      "logits/rejected": -0.09084208309650421,
      "logps/chosen": -449.7691650390625,
      "logps/rejected": -403.8505554199219,
      "loss": 1.1232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.199737548828125,
      "rewards/margins": -0.1091044545173645,
      "rewards/rejected": -2.090632915496826,
      "step": 9319
    },
    {
      "epoch": 1.2195796188762575,
      "grad_norm": 4.041713237762451,
      "learning_rate": 5e-06,
      "logits/chosen": 0.36144760251045227,
      "logits/rejected": 0.1500227153301239,
      "logps/chosen": -524.6561889648438,
      "logps/rejected": -386.89385986328125,
      "loss": 0.2459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29845064878463745,
      "rewards/margins": 1.96875,
      "rewards/rejected": -1.6702992916107178,
      "step": 9320
    },
    {
      "epoch": 1.2197104768136091,
      "grad_norm": 10.093777656555176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26227664947509766,
      "logits/rejected": -0.41938266158103943,
      "logps/chosen": -674.4747924804688,
      "logps/rejected": -640.617919921875,
      "loss": 0.637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8679314255714417,
      "rewards/margins": 1.090477705001831,
      "rewards/rejected": -1.958409309387207,
      "step": 9321
    },
    {
      "epoch": 1.219841334750961,
      "grad_norm": 8.576026916503906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03932681679725647,
      "logits/rejected": 0.036924783140420914,
      "logps/chosen": -514.3141479492188,
      "logps/rejected": -566.0897827148438,
      "loss": 0.3774,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1638381481170654,
      "rewards/margins": 1.9555528163909912,
      "rewards/rejected": -3.1193909645080566,
      "step": 9322
    },
    {
      "epoch": 1.2199721926883127,
      "grad_norm": 5.790525913238525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11809822916984558,
      "logits/rejected": 0.07246115058660507,
      "logps/chosen": -452.2241516113281,
      "logps/rejected": -407.6611328125,
      "loss": 0.2801,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7852380871772766,
      "rewards/margins": 2.1380136013031006,
      "rewards/rejected": -2.9232513904571533,
      "step": 9323
    },
    {
      "epoch": 1.2201030506256645,
      "grad_norm": 14.146517753601074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25270766019821167,
      "logits/rejected": -0.45124560594558716,
      "logps/chosen": -566.605224609375,
      "logps/rejected": -595.9384765625,
      "loss": 0.969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7986301183700562,
      "rewards/margins": 0.24469433724880219,
      "rewards/rejected": -2.0433244705200195,
      "step": 9324
    },
    {
      "epoch": 1.2202339085630163,
      "grad_norm": 11.877659797668457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35132071375846863,
      "logits/rejected": -0.4213566184043884,
      "logps/chosen": -439.77490234375,
      "logps/rejected": -464.5174255371094,
      "loss": 0.5974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.997631311416626,
      "rewards/margins": 0.5683912038803101,
      "rewards/rejected": -2.5660223960876465,
      "step": 9325
    },
    {
      "epoch": 1.220364766500368,
      "grad_norm": 3.1746268272399902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19975696504116058,
      "logits/rejected": -0.2015155553817749,
      "logps/chosen": -424.68707275390625,
      "logps/rejected": -438.9585876464844,
      "loss": 0.2158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1612093150615692,
      "rewards/margins": 2.4259486198425293,
      "rewards/rejected": -2.587158203125,
      "step": 9326
    },
    {
      "epoch": 1.2204956244377199,
      "grad_norm": 10.487648963928223,
      "learning_rate": 5e-06,
      "logits/chosen": 0.050280626863241196,
      "logits/rejected": -0.10791775584220886,
      "logps/chosen": -532.4979858398438,
      "logps/rejected": -506.0515441894531,
      "loss": 0.5657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1107898950576782,
      "rewards/margins": 1.2588865756988525,
      "rewards/rejected": -2.3696765899658203,
      "step": 9327
    },
    {
      "epoch": 1.2206264823750717,
      "grad_norm": 11.714516639709473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16180621087551117,
      "logits/rejected": 0.0569419339299202,
      "logps/chosen": -566.8706665039062,
      "logps/rejected": -470.8958435058594,
      "loss": 0.7387,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1797624826431274,
      "rewards/margins": 0.3796171545982361,
      "rewards/rejected": -1.5593795776367188,
      "step": 9328
    },
    {
      "epoch": 1.2207573403124234,
      "grad_norm": 7.2651872634887695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03507634997367859,
      "logits/rejected": -0.055743299424648285,
      "logps/chosen": -509.8919372558594,
      "logps/rejected": -393.1788635253906,
      "loss": 0.5422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9148783087730408,
      "rewards/margins": 0.5933778285980225,
      "rewards/rejected": -1.5082560777664185,
      "step": 9329
    },
    {
      "epoch": 1.220888198249775,
      "grad_norm": 5.8974995613098145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34391361474990845,
      "logits/rejected": -0.3322311341762543,
      "logps/chosen": -284.7176818847656,
      "logps/rejected": -305.7897033691406,
      "loss": 0.3283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1628174781799316,
      "rewards/margins": 1.4403587579727173,
      "rewards/rejected": -2.6031763553619385,
      "step": 9330
    },
    {
      "epoch": 1.2210190561871268,
      "grad_norm": 9.492955207824707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06241818517446518,
      "logits/rejected": -0.257230281829834,
      "logps/chosen": -755.29248046875,
      "logps/rejected": -575.1189575195312,
      "loss": 0.4668,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4848424792289734,
      "rewards/margins": 1.5795609951019287,
      "rewards/rejected": -2.064403533935547,
      "step": 9331
    },
    {
      "epoch": 1.2211499141244786,
      "grad_norm": 6.936850547790527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31544890999794006,
      "logits/rejected": 0.03117295727133751,
      "logps/chosen": -374.9774169921875,
      "logps/rejected": -513.8626708984375,
      "loss": 0.3783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7463650107383728,
      "rewards/margins": 1.7535128593444824,
      "rewards/rejected": -2.499877691268921,
      "step": 9332
    },
    {
      "epoch": 1.2212807720618304,
      "grad_norm": 8.029314994812012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21619084477424622,
      "logits/rejected": 0.02705700509250164,
      "logps/chosen": -448.7884826660156,
      "logps/rejected": -337.0481872558594,
      "loss": 0.3867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9161646366119385,
      "rewards/margins": 1.2710859775543213,
      "rewards/rejected": -2.1872506141662598,
      "step": 9333
    },
    {
      "epoch": 1.2214116299991822,
      "grad_norm": 11.576460838317871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35017678141593933,
      "logits/rejected": -0.17766551673412323,
      "logps/chosen": -310.85504150390625,
      "logps/rejected": -425.0932922363281,
      "loss": 0.8517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2603247165679932,
      "rewards/margins": 0.3423343598842621,
      "rewards/rejected": -1.602658987045288,
      "step": 9334
    },
    {
      "epoch": 1.221542487936534,
      "grad_norm": 8.63291072845459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14432160556316376,
      "logits/rejected": -0.038046203553676605,
      "logps/chosen": -583.4432983398438,
      "logps/rejected": -581.9002075195312,
      "loss": 0.4147,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5054942965507507,
      "rewards/margins": 1.1320440769195557,
      "rewards/rejected": -1.6375384330749512,
      "step": 9335
    },
    {
      "epoch": 1.2216733458738855,
      "grad_norm": 9.122456550598145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047758787870407104,
      "logits/rejected": -0.026587145403027534,
      "logps/chosen": -646.0636596679688,
      "logps/rejected": -518.5679321289062,
      "loss": 0.3906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3651178777217865,
      "rewards/margins": 1.8769416809082031,
      "rewards/rejected": -2.2420594692230225,
      "step": 9336
    },
    {
      "epoch": 1.2218042038112373,
      "grad_norm": 14.420489311218262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20817920565605164,
      "logits/rejected": -0.22322100400924683,
      "logps/chosen": -546.9799194335938,
      "logps/rejected": -484.63128662109375,
      "loss": 0.9969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.523363471031189,
      "rewards/margins": -0.05217209458351135,
      "rewards/rejected": -1.47119140625,
      "step": 9337
    },
    {
      "epoch": 1.2219350617485891,
      "grad_norm": 7.86892557144165,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07732950896024704,
      "logits/rejected": -0.34207257628440857,
      "logps/chosen": -560.7335205078125,
      "logps/rejected": -490.0639343261719,
      "loss": 0.4397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7128864526748657,
      "rewards/margins": 1.3814747333526611,
      "rewards/rejected": -2.0943613052368164,
      "step": 9338
    },
    {
      "epoch": 1.222065919685941,
      "grad_norm": 12.038596153259277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35597047209739685,
      "logits/rejected": -0.21663981676101685,
      "logps/chosen": -484.9393310546875,
      "logps/rejected": -469.91961669921875,
      "loss": 1.0031,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9606794714927673,
      "rewards/margins": 0.02009524405002594,
      "rewards/rejected": -0.9807747006416321,
      "step": 9339
    },
    {
      "epoch": 1.2221967776232927,
      "grad_norm": 9.745494842529297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15418681502342224,
      "logits/rejected": 0.1183839812874794,
      "logps/chosen": -486.5675048828125,
      "logps/rejected": -439.2188415527344,
      "loss": 0.6563,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3910423517227173,
      "rewards/margins": 0.3134220242500305,
      "rewards/rejected": -0.704464316368103,
      "step": 9340
    },
    {
      "epoch": 1.2223276355606445,
      "grad_norm": 7.956028938293457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22685900330543518,
      "logits/rejected": -0.10008871555328369,
      "logps/chosen": -315.1089782714844,
      "logps/rejected": -445.8502197265625,
      "loss": 0.451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41935405135154724,
      "rewards/margins": 1.1921600103378296,
      "rewards/rejected": -1.6115142107009888,
      "step": 9341
    },
    {
      "epoch": 1.2224584934979963,
      "grad_norm": 7.670749187469482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31891512870788574,
      "logits/rejected": -0.25429755449295044,
      "logps/chosen": -301.1573791503906,
      "logps/rejected": -312.3084716796875,
      "loss": 0.4376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2050769329071045,
      "rewards/margins": 1.204936146736145,
      "rewards/rejected": -2.410013198852539,
      "step": 9342
    },
    {
      "epoch": 1.222589351435348,
      "grad_norm": 8.987814903259277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034647680819034576,
      "logits/rejected": -0.139431893825531,
      "logps/chosen": -513.0199584960938,
      "logps/rejected": -537.1055297851562,
      "loss": 0.7118,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0016063451766968,
      "rewards/margins": 0.4213452935218811,
      "rewards/rejected": -1.4229516983032227,
      "step": 9343
    },
    {
      "epoch": 1.2227202093726999,
      "grad_norm": 6.595403671264648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07939116656780243,
      "logits/rejected": -0.27453503012657166,
      "logps/chosen": -410.9244384765625,
      "logps/rejected": -453.582763671875,
      "loss": 0.3383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0761808454990387,
      "rewards/margins": 1.821688175201416,
      "rewards/rejected": -1.8978688716888428,
      "step": 9344
    },
    {
      "epoch": 1.2228510673100514,
      "grad_norm": 15.948521614074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10000830888748169,
      "logits/rejected": -0.09403494745492935,
      "logps/chosen": -509.8365478515625,
      "logps/rejected": -438.2447814941406,
      "loss": 0.9661,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7787818908691406,
      "rewards/margins": 0.20615597069263458,
      "rewards/rejected": -0.984937846660614,
      "step": 9345
    },
    {
      "epoch": 1.2229819252474032,
      "grad_norm": 10.016182899475098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11561846733093262,
      "logits/rejected": -0.17289669811725616,
      "logps/chosen": -417.87908935546875,
      "logps/rejected": -437.6487121582031,
      "loss": 0.5391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5106439590454102,
      "rewards/margins": 1.3969205617904663,
      "rewards/rejected": -1.907564401626587,
      "step": 9346
    },
    {
      "epoch": 1.223112783184755,
      "grad_norm": 8.872686386108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0938163474202156,
      "logits/rejected": -0.05568331480026245,
      "logps/chosen": -332.50946044921875,
      "logps/rejected": -414.7262268066406,
      "loss": 0.573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.017371654510498,
      "rewards/margins": 1.1123017072677612,
      "rewards/rejected": -2.129673480987549,
      "step": 9347
    },
    {
      "epoch": 1.2232436411221068,
      "grad_norm": 8.258466720581055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060392796993255615,
      "logits/rejected": -0.03997345641255379,
      "logps/chosen": -523.91650390625,
      "logps/rejected": -525.0180053710938,
      "loss": 0.5901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4565374553203583,
      "rewards/margins": 0.8890913128852844,
      "rewards/rejected": -1.3456288576126099,
      "step": 9348
    },
    {
      "epoch": 1.2233744990594586,
      "grad_norm": 6.409790992736816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08502230793237686,
      "logits/rejected": -0.017851702868938446,
      "logps/chosen": -418.7858581542969,
      "logps/rejected": -396.4270935058594,
      "loss": 0.3655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16243915259838104,
      "rewards/margins": 1.6061382293701172,
      "rewards/rejected": -1.7685774564743042,
      "step": 9349
    },
    {
      "epoch": 1.2235053569968104,
      "grad_norm": 13.624080657958984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08378497511148453,
      "logits/rejected": 0.23299568891525269,
      "logps/chosen": -544.7627563476562,
      "logps/rejected": -442.94244384765625,
      "loss": 0.7674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9485941529273987,
      "rewards/margins": 1.0075925588607788,
      "rewards/rejected": -1.9561867713928223,
      "step": 9350
    },
    {
      "epoch": 1.2236362149341622,
      "grad_norm": 4.923686981201172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018288753926753998,
      "logits/rejected": -0.28424784541130066,
      "logps/chosen": -310.9510192871094,
      "logps/rejected": -258.5245056152344,
      "loss": 0.2701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.586346447467804,
      "rewards/margins": 2.0959625244140625,
      "rewards/rejected": -2.682309150695801,
      "step": 9351
    },
    {
      "epoch": 1.2237670728715138,
      "grad_norm": 5.245835781097412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07582468539476395,
      "logits/rejected": 0.0919012576341629,
      "logps/chosen": -295.0031433105469,
      "logps/rejected": -448.6310729980469,
      "loss": 0.234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2855531573295593,
      "rewards/margins": 1.9693348407745361,
      "rewards/rejected": -2.2548880577087402,
      "step": 9352
    },
    {
      "epoch": 1.2238979308088656,
      "grad_norm": 4.212392330169678,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17986316978931427,
      "logits/rejected": 0.005655718967318535,
      "logps/chosen": -455.416259765625,
      "logps/rejected": -460.96875,
      "loss": 0.1661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2649469375610352,
      "rewards/margins": 1.8880038261413574,
      "rewards/rejected": -3.1529507637023926,
      "step": 9353
    },
    {
      "epoch": 1.2240287887462173,
      "grad_norm": 6.9344587326049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040997061878442764,
      "logits/rejected": -0.08629616349935532,
      "logps/chosen": -745.3271484375,
      "logps/rejected": -595.5763549804688,
      "loss": 0.3339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24301376938819885,
      "rewards/margins": 1.5366356372833252,
      "rewards/rejected": -1.7796493768692017,
      "step": 9354
    },
    {
      "epoch": 1.2241596466835691,
      "grad_norm": 10.230923652648926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3023614287376404,
      "logits/rejected": -0.16822706162929535,
      "logps/chosen": -451.39459228515625,
      "logps/rejected": -586.7352294921875,
      "loss": 0.6216,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4935276508331299,
      "rewards/margins": 1.236779808998108,
      "rewards/rejected": -1.7303074598312378,
      "step": 9355
    },
    {
      "epoch": 1.224290504620921,
      "grad_norm": 7.419800281524658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0897618904709816,
      "logits/rejected": -0.039702922105789185,
      "logps/chosen": -490.6336975097656,
      "logps/rejected": -494.58392333984375,
      "loss": 0.5338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17591150104999542,
      "rewards/margins": 0.507878303527832,
      "rewards/rejected": -0.3319668173789978,
      "step": 9356
    },
    {
      "epoch": 1.2244213625582727,
      "grad_norm": 10.120701789855957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48299604654312134,
      "logits/rejected": -0.2247476577758789,
      "logps/chosen": -439.8871765136719,
      "logps/rejected": -270.7334899902344,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5306851863861084,
      "rewards/margins": 0.20078188180923462,
      "rewards/rejected": -1.7314671277999878,
      "step": 9357
    },
    {
      "epoch": 1.2245522204956245,
      "grad_norm": 7.813331127166748,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17773622274398804,
      "logits/rejected": 0.02764412760734558,
      "logps/chosen": -409.057373046875,
      "logps/rejected": -512.58837890625,
      "loss": 0.4323,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6455864906311035,
      "rewards/margins": 1.3056070804595947,
      "rewards/rejected": -1.9511934518814087,
      "step": 9358
    },
    {
      "epoch": 1.2246830784329763,
      "grad_norm": 12.604530334472656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054438523948192596,
      "logits/rejected": -0.19070322811603546,
      "logps/chosen": -566.89013671875,
      "logps/rejected": -585.103271484375,
      "loss": 0.7367,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8721516132354736,
      "rewards/margins": 0.626034140586853,
      "rewards/rejected": -1.498185634613037,
      "step": 9359
    },
    {
      "epoch": 1.2248139363703279,
      "grad_norm": 6.411471366882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06547735631465912,
      "logits/rejected": -0.21420887112617493,
      "logps/chosen": -395.99261474609375,
      "logps/rejected": -362.8872985839844,
      "loss": 0.2572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45566558837890625,
      "rewards/margins": 1.38888680934906,
      "rewards/rejected": -1.8445522785186768,
      "step": 9360
    },
    {
      "epoch": 1.2249447943076797,
      "grad_norm": 5.73734188079834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23560050129890442,
      "logits/rejected": 0.1360737383365631,
      "logps/chosen": -488.7636413574219,
      "logps/rejected": -536.011474609375,
      "loss": 0.2422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24840638041496277,
      "rewards/margins": 2.500213146209717,
      "rewards/rejected": -2.25180721282959,
      "step": 9361
    },
    {
      "epoch": 1.2250756522450315,
      "grad_norm": 5.613500595092773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0002523288130760193,
      "logits/rejected": -0.18730327486991882,
      "logps/chosen": -416.9942932128906,
      "logps/rejected": -409.7990417480469,
      "loss": 0.4288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6941168308258057,
      "rewards/margins": 1.1716219186782837,
      "rewards/rejected": -1.8657387495040894,
      "step": 9362
    },
    {
      "epoch": 1.2252065101823832,
      "grad_norm": 6.557510852813721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1118859276175499,
      "logits/rejected": -0.0646647959947586,
      "logps/chosen": -462.4607238769531,
      "logps/rejected": -460.039306640625,
      "loss": 0.3227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3432791233062744,
      "rewards/margins": 1.2602415084838867,
      "rewards/rejected": -1.6035205125808716,
      "step": 9363
    },
    {
      "epoch": 1.225337368119735,
      "grad_norm": 7.550967216491699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06255881488323212,
      "logits/rejected": -0.2295314222574234,
      "logps/chosen": -405.99859619140625,
      "logps/rejected": -465.9681701660156,
      "loss": 0.4207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37399792671203613,
      "rewards/margins": 1.2527105808258057,
      "rewards/rejected": -1.6267085075378418,
      "step": 9364
    },
    {
      "epoch": 1.2254682260570868,
      "grad_norm": 9.744034767150879,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007966205477714539,
      "logits/rejected": 0.04033913090825081,
      "logps/chosen": -555.4826049804688,
      "logps/rejected": -529.8744506835938,
      "loss": 0.7443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0059925317764282,
      "rewards/margins": 0.23406870663166046,
      "rewards/rejected": -1.2400611639022827,
      "step": 9365
    },
    {
      "epoch": 1.2255990839944386,
      "grad_norm": 7.362171173095703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2777492105960846,
      "logits/rejected": -0.06932958215475082,
      "logps/chosen": -343.3834228515625,
      "logps/rejected": -435.0960388183594,
      "loss": 0.3692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3101089000701904,
      "rewards/margins": 1.434177279472351,
      "rewards/rejected": -2.744286060333252,
      "step": 9366
    },
    {
      "epoch": 1.2257299419317902,
      "grad_norm": 5.506855010986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39737963676452637,
      "logits/rejected": -0.47400420904159546,
      "logps/chosen": -461.4857482910156,
      "logps/rejected": -386.3177490234375,
      "loss": 0.3204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8609542846679688,
      "rewards/margins": 1.4289454221725464,
      "rewards/rejected": -2.2898998260498047,
      "step": 9367
    },
    {
      "epoch": 1.225860799869142,
      "grad_norm": 8.389453887939453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16380169987678528,
      "logits/rejected": -0.27251461148262024,
      "logps/chosen": -279.3985290527344,
      "logps/rejected": -373.06597900390625,
      "loss": 0.5147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.620474636554718,
      "rewards/margins": 1.0204944610595703,
      "rewards/rejected": -1.640969157218933,
      "step": 9368
    },
    {
      "epoch": 1.2259916578064938,
      "grad_norm": 11.936363220214844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5804139971733093,
      "logits/rejected": -0.6797321438789368,
      "logps/chosen": -318.49151611328125,
      "logps/rejected": -329.2209777832031,
      "loss": 0.5758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6843831539154053,
      "rewards/margins": 0.9713070392608643,
      "rewards/rejected": -2.6556904315948486,
      "step": 9369
    },
    {
      "epoch": 1.2261225157438456,
      "grad_norm": 9.258670806884766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027360916137695312,
      "logits/rejected": -0.00875866413116455,
      "logps/chosen": -437.8440856933594,
      "logps/rejected": -474.79620361328125,
      "loss": 0.7453,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8158529996871948,
      "rewards/margins": 0.4267624318599701,
      "rewards/rejected": -2.2426154613494873,
      "step": 9370
    },
    {
      "epoch": 1.2262533736811974,
      "grad_norm": 7.961537837982178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.447976291179657,
      "logits/rejected": -0.32221361994743347,
      "logps/chosen": -328.0101013183594,
      "logps/rejected": -312.8581237792969,
      "loss": 0.5278,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6027244925498962,
      "rewards/margins": 0.920906126499176,
      "rewards/rejected": -1.5236306190490723,
      "step": 9371
    },
    {
      "epoch": 1.2263842316185491,
      "grad_norm": 7.355379581451416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07357672601938248,
      "logits/rejected": -0.18116074800491333,
      "logps/chosen": -418.55419921875,
      "logps/rejected": -380.8045959472656,
      "loss": 0.4475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4709709882736206,
      "rewards/margins": 1.1074243783950806,
      "rewards/rejected": -1.5783953666687012,
      "step": 9372
    },
    {
      "epoch": 1.226515089555901,
      "grad_norm": 8.371302604675293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3437712490558624,
      "logits/rejected": -0.20321175456047058,
      "logps/chosen": -438.2470397949219,
      "logps/rejected": -650.765869140625,
      "loss": 0.421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9357922077178955,
      "rewards/margins": 1.3819612264633179,
      "rewards/rejected": -2.317753553390503,
      "step": 9373
    },
    {
      "epoch": 1.2266459474932527,
      "grad_norm": 10.939387321472168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005162090063095093,
      "logits/rejected": -0.048113130033016205,
      "logps/chosen": -407.7491455078125,
      "logps/rejected": -450.8848571777344,
      "loss": 0.5295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3240527808666229,
      "rewards/margins": 0.8661428689956665,
      "rewards/rejected": -1.1901957988739014,
      "step": 9374
    },
    {
      "epoch": 1.2267768054306045,
      "grad_norm": 8.91697883605957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1293250173330307,
      "logits/rejected": -0.14971688389778137,
      "logps/chosen": -528.4708862304688,
      "logps/rejected": -373.6753234863281,
      "loss": 0.3188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9157981872558594,
      "rewards/margins": 1.6397701501846313,
      "rewards/rejected": -2.555568218231201,
      "step": 9375
    },
    {
      "epoch": 1.226907663367956,
      "grad_norm": 10.063924789428711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06981289386749268,
      "logits/rejected": -0.28562214970588684,
      "logps/chosen": -412.63873291015625,
      "logps/rejected": -475.9878845214844,
      "loss": 0.6401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7015655636787415,
      "rewards/margins": 0.5951212644577026,
      "rewards/rejected": -1.2966867685317993,
      "step": 9376
    },
    {
      "epoch": 1.2270385213053079,
      "grad_norm": 7.3925580978393555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1787819117307663,
      "logits/rejected": -0.2249840795993805,
      "logps/chosen": -546.1795043945312,
      "logps/rejected": -432.421142578125,
      "loss": 0.48,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9557316303253174,
      "rewards/margins": 1.0568007230758667,
      "rewards/rejected": -2.0125322341918945,
      "step": 9377
    },
    {
      "epoch": 1.2271693792426597,
      "grad_norm": 9.790270805358887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11482446640729904,
      "logits/rejected": -0.12964750826358795,
      "logps/chosen": -448.1487731933594,
      "logps/rejected": -399.0469665527344,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0065860748291016,
      "rewards/margins": 0.44288668036460876,
      "rewards/rejected": -1.4494726657867432,
      "step": 9378
    },
    {
      "epoch": 1.2273002371800115,
      "grad_norm": 13.352972984313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07420822232961655,
      "logits/rejected": -0.42101240158081055,
      "logps/chosen": -532.2174682617188,
      "logps/rejected": -511.29034423828125,
      "loss": 0.7283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8909915685653687,
      "rewards/margins": 1.2202088832855225,
      "rewards/rejected": -2.1112005710601807,
      "step": 9379
    },
    {
      "epoch": 1.2274310951173633,
      "grad_norm": 9.233782768249512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2959359884262085,
      "logits/rejected": -0.33545759320259094,
      "logps/chosen": -414.8880310058594,
      "logps/rejected": -434.7784729003906,
      "loss": 0.5082,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1218065023422241,
      "rewards/margins": 1.3194316625595093,
      "rewards/rejected": -2.4412384033203125,
      "step": 9380
    },
    {
      "epoch": 1.227561953054715,
      "grad_norm": 5.953912258148193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18221022188663483,
      "logits/rejected": -0.18658268451690674,
      "logps/chosen": -633.0632934570312,
      "logps/rejected": -525.7991943359375,
      "loss": 0.2609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5722500085830688,
      "rewards/margins": 1.766649842262268,
      "rewards/rejected": -2.338899850845337,
      "step": 9381
    },
    {
      "epoch": 1.2276928109920666,
      "grad_norm": 7.097331523895264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1747489869594574,
      "logits/rejected": -0.20408865809440613,
      "logps/chosen": -378.1764221191406,
      "logps/rejected": -307.99822998046875,
      "loss": 0.5076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4208418130874634,
      "rewards/margins": 1.1561083793640137,
      "rewards/rejected": -1.5769500732421875,
      "step": 9382
    },
    {
      "epoch": 1.2278236689294184,
      "grad_norm": 9.867081642150879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03328869491815567,
      "logits/rejected": -0.0681360736489296,
      "logps/chosen": -371.619384765625,
      "logps/rejected": -433.4388427734375,
      "loss": 0.8289,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3253408670425415,
      "rewards/margins": 0.4230732023715973,
      "rewards/rejected": -1.7484140396118164,
      "step": 9383
    },
    {
      "epoch": 1.2279545268667702,
      "grad_norm": 10.921906471252441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2832563519477844,
      "logits/rejected": 7.208064198493958e-05,
      "logps/chosen": -467.3410339355469,
      "logps/rejected": -452.87451171875,
      "loss": 0.3961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33405646681785583,
      "rewards/margins": 1.3569365739822388,
      "rewards/rejected": -1.0228800773620605,
      "step": 9384
    },
    {
      "epoch": 1.228085384804122,
      "grad_norm": 8.167342185974121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2229461520910263,
      "logits/rejected": -0.09954365342855453,
      "logps/chosen": -413.9114685058594,
      "logps/rejected": -508.23504638671875,
      "loss": 0.4357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8676359057426453,
      "rewards/margins": 1.0382461547851562,
      "rewards/rejected": -1.9058821201324463,
      "step": 9385
    },
    {
      "epoch": 1.2282162427414738,
      "grad_norm": 4.861546039581299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.347831666469574,
      "logits/rejected": -0.26323845982551575,
      "logps/chosen": -450.7882995605469,
      "logps/rejected": -552.8370361328125,
      "loss": 0.1884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29248321056365967,
      "rewards/margins": 2.078702926635742,
      "rewards/rejected": -2.3711862564086914,
      "step": 9386
    },
    {
      "epoch": 1.2283471006788256,
      "grad_norm": 8.50562572479248,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005900822579860687,
      "logits/rejected": 0.04412706941366196,
      "logps/chosen": -663.052734375,
      "logps/rejected": -556.2692260742188,
      "loss": 0.5325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4399002194404602,
      "rewards/margins": 0.5015321969985962,
      "rewards/rejected": -0.9414324760437012,
      "step": 9387
    },
    {
      "epoch": 1.2284779586161774,
      "grad_norm": 8.970976829528809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11430692672729492,
      "logits/rejected": -0.0635688304901123,
      "logps/chosen": -568.4917602539062,
      "logps/rejected": -658.792236328125,
      "loss": 0.5608,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0176751613616943,
      "rewards/margins": 0.68121337890625,
      "rewards/rejected": -1.6988885402679443,
      "step": 9388
    },
    {
      "epoch": 1.2286088165535292,
      "grad_norm": 4.346017360687256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13579775393009186,
      "logits/rejected": -0.16317693889141083,
      "logps/chosen": -481.90545654296875,
      "logps/rejected": -481.4955749511719,
      "loss": 0.1615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16001950204372406,
      "rewards/margins": 2.1544253826141357,
      "rewards/rejected": -2.3144445419311523,
      "step": 9389
    },
    {
      "epoch": 1.228739674490881,
      "grad_norm": 8.976240158081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10900118947029114,
      "logits/rejected": -0.16393348574638367,
      "logps/chosen": -480.81072998046875,
      "logps/rejected": -492.71881103515625,
      "loss": 0.5161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15157794952392578,
      "rewards/margins": 0.9771891236305237,
      "rewards/rejected": -1.1287671327590942,
      "step": 9390
    },
    {
      "epoch": 1.2288705324282325,
      "grad_norm": 10.095271110534668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1925252079963684,
      "logits/rejected": -0.18969638645648956,
      "logps/chosen": -605.6557006835938,
      "logps/rejected": -510.0439453125,
      "loss": 0.6508,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29752424359321594,
      "rewards/margins": 0.7079185843467712,
      "rewards/rejected": -1.0054428577423096,
      "step": 9391
    },
    {
      "epoch": 1.2290013903655843,
      "grad_norm": 14.236950874328613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1552783101797104,
      "logits/rejected": -0.4115120470523834,
      "logps/chosen": -540.3068237304688,
      "logps/rejected": -624.9881591796875,
      "loss": 0.6183,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47492218017578125,
      "rewards/margins": 1.1569514274597168,
      "rewards/rejected": -1.631873607635498,
      "step": 9392
    },
    {
      "epoch": 1.229132248302936,
      "grad_norm": 6.943323612213135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01956656202673912,
      "logits/rejected": 0.06417439877986908,
      "logps/chosen": -482.1634826660156,
      "logps/rejected": -417.63299560546875,
      "loss": 0.3862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6409139037132263,
      "rewards/margins": 1.457840085029602,
      "rewards/rejected": -2.0987539291381836,
      "step": 9393
    },
    {
      "epoch": 1.229263106240288,
      "grad_norm": 8.007351875305176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01546422392129898,
      "logits/rejected": -0.0932818278670311,
      "logps/chosen": -547.3658447265625,
      "logps/rejected": -462.0521240234375,
      "loss": 0.3694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7244762182235718,
      "rewards/margins": 1.2076472043991089,
      "rewards/rejected": -1.9321234226226807,
      "step": 9394
    },
    {
      "epoch": 1.2293939641776397,
      "grad_norm": 6.821757793426514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7804906964302063,
      "logits/rejected": -0.7279649972915649,
      "logps/chosen": -509.2915954589844,
      "logps/rejected": -518.9879150390625,
      "loss": 0.3893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7104188203811646,
      "rewards/margins": 1.180485486984253,
      "rewards/rejected": -1.8909043073654175,
      "step": 9395
    },
    {
      "epoch": 1.2295248221149915,
      "grad_norm": 7.870083808898926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10012669861316681,
      "logits/rejected": -0.21359464526176453,
      "logps/chosen": -437.91534423828125,
      "logps/rejected": -344.9620056152344,
      "loss": 0.4173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3278743624687195,
      "rewards/margins": 1.327162742614746,
      "rewards/rejected": -1.6550372838974,
      "step": 9396
    },
    {
      "epoch": 1.2296556800523433,
      "grad_norm": 10.3671293258667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24030093848705292,
      "logits/rejected": -0.10777868330478668,
      "logps/chosen": -351.13360595703125,
      "logps/rejected": -352.5243835449219,
      "loss": 0.8589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1902825832366943,
      "rewards/margins": -0.17720390856266022,
      "rewards/rejected": -1.0130786895751953,
      "step": 9397
    },
    {
      "epoch": 1.2297865379896948,
      "grad_norm": 7.511312961578369,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5240656137466431,
      "logits/rejected": -0.4492858350276947,
      "logps/chosen": -313.4797058105469,
      "logps/rejected": -342.4793395996094,
      "loss": 0.4097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.750637412071228,
      "rewards/margins": 1.3314272165298462,
      "rewards/rejected": -2.082064628601074,
      "step": 9398
    },
    {
      "epoch": 1.2299173959270466,
      "grad_norm": 9.013105392456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3740570843219757,
      "logits/rejected": -0.5518816113471985,
      "logps/chosen": -384.7811584472656,
      "logps/rejected": -437.7760009765625,
      "loss": 0.5263,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9548515677452087,
      "rewards/margins": 1.2841224670410156,
      "rewards/rejected": -2.238974094390869,
      "step": 9399
    },
    {
      "epoch": 1.2300482538643984,
      "grad_norm": 9.720864295959473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02496635727584362,
      "logits/rejected": -0.248780757188797,
      "logps/chosen": -496.06976318359375,
      "logps/rejected": -672.022705078125,
      "loss": 0.6554,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7737573385238647,
      "rewards/margins": 1.1086419820785522,
      "rewards/rejected": -1.882399320602417,
      "step": 9400
    },
    {
      "epoch": 1.2301791118017502,
      "grad_norm": 3.2341036796569824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2056233435869217,
      "logits/rejected": 0.0380602590739727,
      "logps/chosen": -614.8916015625,
      "logps/rejected": -485.1273498535156,
      "loss": 0.1261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3778892457485199,
      "rewards/margins": 2.3135006427764893,
      "rewards/rejected": -1.935611367225647,
      "step": 9401
    },
    {
      "epoch": 1.230309969739102,
      "grad_norm": 5.222828388214111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23619228601455688,
      "logits/rejected": -0.3070380389690399,
      "logps/chosen": -520.3401489257812,
      "logps/rejected": -477.77911376953125,
      "loss": 0.2558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4641990661621094,
      "rewards/margins": 2.23776912689209,
      "rewards/rejected": -2.701968193054199,
      "step": 9402
    },
    {
      "epoch": 1.2304408276764538,
      "grad_norm": 5.190169811248779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009789921343326569,
      "logits/rejected": -0.3053686022758484,
      "logps/chosen": -560.423095703125,
      "logps/rejected": -448.17584228515625,
      "loss": 0.2435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31537124514579773,
      "rewards/margins": 1.8316738605499268,
      "rewards/rejected": -2.1470448970794678,
      "step": 9403
    },
    {
      "epoch": 1.2305716856138056,
      "grad_norm": 9.1744966506958,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02962198480963707,
      "logits/rejected": -0.1654309779405594,
      "logps/chosen": -297.084716796875,
      "logps/rejected": -385.720458984375,
      "loss": 0.4633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2991834878921509,
      "rewards/margins": 1.3160101175308228,
      "rewards/rejected": -2.6151938438415527,
      "step": 9404
    },
    {
      "epoch": 1.2307025435511574,
      "grad_norm": 6.3274102210998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2873198091983795,
      "logits/rejected": -0.160764679312706,
      "logps/chosen": -557.8687744140625,
      "logps/rejected": -516.4712524414062,
      "loss": 0.3113,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9299983978271484,
      "rewards/margins": 1.7847222089767456,
      "rewards/rejected": -2.7147204875946045,
      "step": 9405
    },
    {
      "epoch": 1.230833401488509,
      "grad_norm": 8.321202278137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31846433877944946,
      "logits/rejected": -0.3893797695636749,
      "logps/chosen": -359.65972900390625,
      "logps/rejected": -385.8121337890625,
      "loss": 0.4091,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4220577478408813,
      "rewards/margins": 1.2718298435211182,
      "rewards/rejected": -2.693887233734131,
      "step": 9406
    },
    {
      "epoch": 1.2309642594258607,
      "grad_norm": 4.266345024108887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09870480000972748,
      "logits/rejected": -0.38997241854667664,
      "logps/chosen": -514.8988037109375,
      "logps/rejected": -435.44232177734375,
      "loss": 0.2575,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5233964323997498,
      "rewards/margins": 2.0585837364196777,
      "rewards/rejected": -2.5819802284240723,
      "step": 9407
    },
    {
      "epoch": 1.2310951173632125,
      "grad_norm": 8.609928131103516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08482906222343445,
      "logits/rejected": -0.4175109565258026,
      "logps/chosen": -408.30889892578125,
      "logps/rejected": -390.17498779296875,
      "loss": 0.4605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3775277137756348,
      "rewards/margins": 1.1963602304458618,
      "rewards/rejected": -2.573887825012207,
      "step": 9408
    },
    {
      "epoch": 1.2312259753005643,
      "grad_norm": 10.749710083007812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36368197202682495,
      "logits/rejected": -0.30844518542289734,
      "logps/chosen": -622.1171875,
      "logps/rejected": -679.9458618164062,
      "loss": 0.5796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3491774797439575,
      "rewards/margins": 0.8760805726051331,
      "rewards/rejected": -2.2252581119537354,
      "step": 9409
    },
    {
      "epoch": 1.2313568332379161,
      "grad_norm": 9.589296340942383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08611766993999481,
      "logits/rejected": -0.10006941109895706,
      "logps/chosen": -325.74261474609375,
      "logps/rejected": -362.4527282714844,
      "loss": 0.4623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9427934885025024,
      "rewards/margins": 0.8976844549179077,
      "rewards/rejected": -1.8404779434204102,
      "step": 9410
    },
    {
      "epoch": 1.231487691175268,
      "grad_norm": 6.194545745849609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31801196932792664,
      "logits/rejected": 0.02378266677260399,
      "logps/chosen": -455.8708190917969,
      "logps/rejected": -483.9951477050781,
      "loss": 0.3541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8839256763458252,
      "rewards/margins": 1.6426607370376587,
      "rewards/rejected": -2.5265865325927734,
      "step": 9411
    },
    {
      "epoch": 1.2316185491126197,
      "grad_norm": 4.0189337730407715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11104610562324524,
      "logits/rejected": -0.17060218751430511,
      "logps/chosen": -420.87548828125,
      "logps/rejected": -414.1001281738281,
      "loss": 0.2285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4535645246505737,
      "rewards/margins": 1.8432822227478027,
      "rewards/rejected": -3.296846628189087,
      "step": 9412
    },
    {
      "epoch": 1.2317494070499713,
      "grad_norm": 9.110498428344727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12945275008678436,
      "logits/rejected": -0.13431404531002045,
      "logps/chosen": -459.554931640625,
      "logps/rejected": -416.5843200683594,
      "loss": 0.6726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.513967514038086,
      "rewards/margins": 0.7760022878646851,
      "rewards/rejected": -2.2899699211120605,
      "step": 9413
    },
    {
      "epoch": 1.231880264987323,
      "grad_norm": 6.214472770690918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16593748331069946,
      "logits/rejected": -0.1382385641336441,
      "logps/chosen": -507.3675231933594,
      "logps/rejected": -413.8205261230469,
      "loss": 0.4808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5352340340614319,
      "rewards/margins": 1.1940276622772217,
      "rewards/rejected": -1.7292617559432983,
      "step": 9414
    },
    {
      "epoch": 1.2320111229246749,
      "grad_norm": 10.67272663116455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14263352751731873,
      "logits/rejected": -0.1693730652332306,
      "logps/chosen": -309.5660705566406,
      "logps/rejected": -332.4771728515625,
      "loss": 0.9345,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.1026297807693481,
      "rewards/margins": -0.22133272886276245,
      "rewards/rejected": -0.8812969923019409,
      "step": 9415
    },
    {
      "epoch": 1.2321419808620266,
      "grad_norm": 5.094590663909912,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16918529570102692,
      "logits/rejected": -0.10136070847511292,
      "logps/chosen": -503.6034240722656,
      "logps/rejected": -499.36370849609375,
      "loss": 0.3223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1639021635055542,
      "rewards/margins": 1.2130206823349,
      "rewards/rejected": -2.376922845840454,
      "step": 9416
    },
    {
      "epoch": 1.2322728387993784,
      "grad_norm": 7.793751239776611,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09331720322370529,
      "logits/rejected": -0.00436173751950264,
      "logps/chosen": -502.16485595703125,
      "logps/rejected": -471.780029296875,
      "loss": 0.5108,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2192586660385132,
      "rewards/margins": 1.0374345779418945,
      "rewards/rejected": -2.2566933631896973,
      "step": 9417
    },
    {
      "epoch": 1.2324036967367302,
      "grad_norm": 7.721767425537109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10892902314662933,
      "logits/rejected": -0.28689539432525635,
      "logps/chosen": -427.00653076171875,
      "logps/rejected": -431.90869140625,
      "loss": 0.419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.529952883720398,
      "rewards/margins": 1.5796756744384766,
      "rewards/rejected": -3.109628438949585,
      "step": 9418
    },
    {
      "epoch": 1.232534554674082,
      "grad_norm": 10.10618782043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1252748817205429,
      "logits/rejected": -0.34187549352645874,
      "logps/chosen": -414.36669921875,
      "logps/rejected": -427.8904113769531,
      "loss": 0.6328,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.189037561416626,
      "rewards/margins": 0.4660683870315552,
      "rewards/rejected": -2.6551060676574707,
      "step": 9419
    },
    {
      "epoch": 1.2326654126114338,
      "grad_norm": 6.797458171844482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40000593662261963,
      "logits/rejected": -0.049352020025253296,
      "logps/chosen": -497.11956787109375,
      "logps/rejected": -541.68994140625,
      "loss": 0.3326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7053344249725342,
      "rewards/margins": 1.6353685855865479,
      "rewards/rejected": -2.340703248977661,
      "step": 9420
    },
    {
      "epoch": 1.2327962705487854,
      "grad_norm": 9.944934844970703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043450504541397095,
      "logits/rejected": -0.29909566044807434,
      "logps/chosen": -476.3086242675781,
      "logps/rejected": -495.638916015625,
      "loss": 0.599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9666496515274048,
      "rewards/margins": 0.9100226759910583,
      "rewards/rejected": -1.876672387123108,
      "step": 9421
    },
    {
      "epoch": 1.2329271284861372,
      "grad_norm": 8.619538307189941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1690209060907364,
      "logits/rejected": 0.1832127869129181,
      "logps/chosen": -414.98126220703125,
      "logps/rejected": -465.21258544921875,
      "loss": 0.4672,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.328296422958374,
      "rewards/margins": 1.5221434831619263,
      "rewards/rejected": -2.8504395484924316,
      "step": 9422
    },
    {
      "epoch": 1.233057986423489,
      "grad_norm": 5.363204002380371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33627787232398987,
      "logits/rejected": -0.44419562816619873,
      "logps/chosen": -364.0407409667969,
      "logps/rejected": -317.588623046875,
      "loss": 0.2997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3945425748825073,
      "rewards/margins": 1.5812948942184448,
      "rewards/rejected": -2.975837230682373,
      "step": 9423
    },
    {
      "epoch": 1.2331888443608408,
      "grad_norm": 10.635139465332031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.000651841051876545,
      "logits/rejected": 0.10144024342298508,
      "logps/chosen": -629.7799682617188,
      "logps/rejected": -431.4399108886719,
      "loss": 0.449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1069355010986328,
      "rewards/margins": 1.4204611778259277,
      "rewards/rejected": -2.5273969173431396,
      "step": 9424
    },
    {
      "epoch": 1.2333197022981925,
      "grad_norm": 8.185419082641602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32046231627464294,
      "logits/rejected": -0.015393318608403206,
      "logps/chosen": -381.0616149902344,
      "logps/rejected": -305.3620910644531,
      "loss": 0.3633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2735389471054077,
      "rewards/margins": 1.2368210554122925,
      "rewards/rejected": -2.5103600025177,
      "step": 9425
    },
    {
      "epoch": 1.2334505602355443,
      "grad_norm": 10.879467964172363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07191862165927887,
      "logits/rejected": -0.1077001690864563,
      "logps/chosen": -272.831787109375,
      "logps/rejected": -369.8737487792969,
      "loss": 0.5687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4316318035125732,
      "rewards/margins": 1.0729175806045532,
      "rewards/rejected": -2.504549503326416,
      "step": 9426
    },
    {
      "epoch": 1.2335814181728961,
      "grad_norm": 5.693964004516602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.044340264052152634,
      "logits/rejected": -0.08399748802185059,
      "logps/chosen": -556.682373046875,
      "logps/rejected": -411.5017395019531,
      "loss": 0.3387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.030519865453243256,
      "rewards/margins": 1.168135643005371,
      "rewards/rejected": -1.198655605316162,
      "step": 9427
    },
    {
      "epoch": 1.2337122761102477,
      "grad_norm": 15.605271339416504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36461830139160156,
      "logits/rejected": -0.13955406844615936,
      "logps/chosen": -509.9535827636719,
      "logps/rejected": -557.1050415039062,
      "loss": 0.6002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6673979759216309,
      "rewards/margins": 0.8950729966163635,
      "rewards/rejected": -2.5624711513519287,
      "step": 9428
    },
    {
      "epoch": 1.2338431340475995,
      "grad_norm": 6.453672409057617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07669676840305328,
      "logits/rejected": -0.1176631823182106,
      "logps/chosen": -536.8137817382812,
      "logps/rejected": -488.88897705078125,
      "loss": 0.2652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9055792093276978,
      "rewards/margins": 1.704582929611206,
      "rewards/rejected": -3.6101620197296143,
      "step": 9429
    },
    {
      "epoch": 1.2339739919849513,
      "grad_norm": 16.689132690429688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37818843126296997,
      "logits/rejected": -0.2809814214706421,
      "logps/chosen": -491.2016296386719,
      "logps/rejected": -421.45550537109375,
      "loss": 0.7494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6147125959396362,
      "rewards/margins": 0.8149504661560059,
      "rewards/rejected": -2.4296629428863525,
      "step": 9430
    },
    {
      "epoch": 1.234104849922303,
      "grad_norm": 8.496898651123047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23724400997161865,
      "logits/rejected": -0.07963219285011292,
      "logps/chosen": -394.0043640136719,
      "logps/rejected": -456.8587341308594,
      "loss": 0.3981,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.910966694355011,
      "rewards/margins": 1.7107523679733276,
      "rewards/rejected": -2.6217188835144043,
      "step": 9431
    },
    {
      "epoch": 1.2342357078596549,
      "grad_norm": 16.804698944091797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41180166602134705,
      "logits/rejected": -0.6748430132865906,
      "logps/chosen": -504.54718017578125,
      "logps/rejected": -401.42437744140625,
      "loss": 0.4355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.002984046936035,
      "rewards/margins": 1.7781258821487427,
      "rewards/rejected": -3.7811100482940674,
      "step": 9432
    },
    {
      "epoch": 1.2343665657970067,
      "grad_norm": 8.933642387390137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2253047525882721,
      "logits/rejected": -0.2727354168891907,
      "logps/chosen": -587.0969848632812,
      "logps/rejected": -523.9031372070312,
      "loss": 0.4812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.243966817855835,
      "rewards/margins": 0.944806694984436,
      "rewards/rejected": -2.1887733936309814,
      "step": 9433
    },
    {
      "epoch": 1.2344974237343584,
      "grad_norm": 5.63258171081543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3137590289115906,
      "logits/rejected": -0.3476179838180542,
      "logps/chosen": -426.562255859375,
      "logps/rejected": -584.8905029296875,
      "loss": 0.2473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5060511827468872,
      "rewards/margins": 1.6127737760543823,
      "rewards/rejected": -3.1188249588012695,
      "step": 9434
    },
    {
      "epoch": 1.2346282816717102,
      "grad_norm": 12.52885913848877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0015873387455940247,
      "logits/rejected": -0.2906946539878845,
      "logps/chosen": -343.4879150390625,
      "logps/rejected": -315.86175537109375,
      "loss": 0.5309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6104503870010376,
      "rewards/margins": 1.6346418857574463,
      "rewards/rejected": -3.2450921535491943,
      "step": 9435
    },
    {
      "epoch": 1.234759139609062,
      "grad_norm": 8.378432273864746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13645018637180328,
      "logits/rejected": -0.12371687591075897,
      "logps/chosen": -354.8967590332031,
      "logps/rejected": -358.33441162109375,
      "loss": 0.3859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3469648361206055,
      "rewards/margins": 1.2029242515563965,
      "rewards/rejected": -2.549889087677002,
      "step": 9436
    },
    {
      "epoch": 1.2348899975464136,
      "grad_norm": 11.54684066772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36457255482673645,
      "logits/rejected": -0.16290993988513947,
      "logps/chosen": -569.4854125976562,
      "logps/rejected": -526.4061889648438,
      "loss": 0.573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.643815040588379,
      "rewards/margins": 0.7035638093948364,
      "rewards/rejected": -2.347378730773926,
      "step": 9437
    },
    {
      "epoch": 1.2350208554837654,
      "grad_norm": 6.939074516296387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09450285136699677,
      "logits/rejected": 0.13857191801071167,
      "logps/chosen": -686.596435546875,
      "logps/rejected": -492.1337890625,
      "loss": 0.3951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.152446985244751,
      "rewards/margins": 1.397933006286621,
      "rewards/rejected": -2.550379991531372,
      "step": 9438
    },
    {
      "epoch": 1.2351517134211172,
      "grad_norm": 7.413882255554199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16632911562919617,
      "logits/rejected": -0.17501643300056458,
      "logps/chosen": -406.34918212890625,
      "logps/rejected": -487.8513488769531,
      "loss": 0.5169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9146518111228943,
      "rewards/margins": 0.561905026435852,
      "rewards/rejected": -1.4765567779541016,
      "step": 9439
    },
    {
      "epoch": 1.235282571358469,
      "grad_norm": 8.834485054016113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.147930309176445,
      "logits/rejected": -0.08666597306728363,
      "logps/chosen": -586.7922973632812,
      "logps/rejected": -512.0062255859375,
      "loss": 0.4489,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6914626955986023,
      "rewards/margins": 2.1550300121307373,
      "rewards/rejected": -2.8464927673339844,
      "step": 9440
    },
    {
      "epoch": 1.2354134292958208,
      "grad_norm": 10.449734687805176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20866014063358307,
      "logits/rejected": -0.15165437757968903,
      "logps/chosen": -357.15032958984375,
      "logps/rejected": -374.1838073730469,
      "loss": 0.6226,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.089505195617676,
      "rewards/margins": 1.1972564458847046,
      "rewards/rejected": -3.286761999130249,
      "step": 9441
    },
    {
      "epoch": 1.2355442872331726,
      "grad_norm": 13.951862335205078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4609638750553131,
      "logits/rejected": -0.6376942992210388,
      "logps/chosen": -381.2138366699219,
      "logps/rejected": -411.33154296875,
      "loss": 0.7651,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.998632788658142,
      "rewards/margins": 0.5723412036895752,
      "rewards/rejected": -2.570974111557007,
      "step": 9442
    },
    {
      "epoch": 1.2356751451705241,
      "grad_norm": 9.510520935058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5845746994018555,
      "logits/rejected": -0.5197409391403198,
      "logps/chosen": -461.8826904296875,
      "logps/rejected": -446.77471923828125,
      "loss": 0.5967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6690998077392578,
      "rewards/margins": 1.3353008031845093,
      "rewards/rejected": -3.0044004917144775,
      "step": 9443
    },
    {
      "epoch": 1.235806003107876,
      "grad_norm": 9.360010147094727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09911598265171051,
      "logits/rejected": -0.006610553711652756,
      "logps/chosen": -548.5484008789062,
      "logps/rejected": -418.4753112792969,
      "loss": 0.578,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.312164545059204,
      "rewards/margins": 0.6864065527915955,
      "rewards/rejected": -1.9985710382461548,
      "step": 9444
    },
    {
      "epoch": 1.2359368610452277,
      "grad_norm": 4.784178733825684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1746959686279297,
      "logits/rejected": 0.05264519155025482,
      "logps/chosen": -556.5808715820312,
      "logps/rejected": -472.3473205566406,
      "loss": 0.2359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49181514978408813,
      "rewards/margins": 1.7740156650543213,
      "rewards/rejected": -2.2658309936523438,
      "step": 9445
    },
    {
      "epoch": 1.2360677189825795,
      "grad_norm": 7.073066711425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24485215544700623,
      "logits/rejected": -0.49154022336006165,
      "logps/chosen": -482.5767822265625,
      "logps/rejected": -507.54315185546875,
      "loss": 0.4577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2457103729248047,
      "rewards/margins": 1.3080353736877441,
      "rewards/rejected": -3.553745746612549,
      "step": 9446
    },
    {
      "epoch": 1.2361985769199313,
      "grad_norm": 7.287212371826172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31599751114845276,
      "logits/rejected": -0.1191321387887001,
      "logps/chosen": -414.8222961425781,
      "logps/rejected": -462.536865234375,
      "loss": 0.4478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.441579818725586,
      "rewards/margins": 1.249549150466919,
      "rewards/rejected": -2.691129207611084,
      "step": 9447
    },
    {
      "epoch": 1.236329434857283,
      "grad_norm": 10.362192153930664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20181985199451447,
      "logits/rejected": -0.29769402742385864,
      "logps/chosen": -522.86572265625,
      "logps/rejected": -445.6000061035156,
      "loss": 0.4368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1370868682861328,
      "rewards/margins": 1.5692250728607178,
      "rewards/rejected": -2.7063119411468506,
      "step": 9448
    },
    {
      "epoch": 1.2364602927946349,
      "grad_norm": 7.522008895874023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23003147542476654,
      "logits/rejected": -0.10247637331485748,
      "logps/chosen": -540.6915893554688,
      "logps/rejected": -452.25775146484375,
      "loss": 0.3144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3985082507133484,
      "rewards/margins": 1.578125238418579,
      "rewards/rejected": -1.9766335487365723,
      "step": 9449
    },
    {
      "epoch": 1.2365911507319867,
      "grad_norm": 8.0288724899292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06830745190382004,
      "logits/rejected": -0.2702617645263672,
      "logps/chosen": -465.09063720703125,
      "logps/rejected": -505.08673095703125,
      "loss": 0.4605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.183171272277832,
      "rewards/margins": 1.6275972127914429,
      "rewards/rejected": -2.8107686042785645,
      "step": 9450
    },
    {
      "epoch": 1.2367220086693385,
      "grad_norm": 9.19602108001709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3136546015739441,
      "logits/rejected": -0.3371978998184204,
      "logps/chosen": -425.51800537109375,
      "logps/rejected": -377.5355224609375,
      "loss": 0.573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6886091232299805,
      "rewards/margins": 0.4081975817680359,
      "rewards/rejected": -2.0968070030212402,
      "step": 9451
    },
    {
      "epoch": 1.23685286660669,
      "grad_norm": 4.674335479736328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.053797394037246704,
      "logits/rejected": -0.21354560554027557,
      "logps/chosen": -504.13037109375,
      "logps/rejected": -464.91998291015625,
      "loss": 0.1823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8690170645713806,
      "rewards/margins": 2.647170305252075,
      "rewards/rejected": -3.5161874294281006,
      "step": 9452
    },
    {
      "epoch": 1.2369837245440418,
      "grad_norm": 9.306734085083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18027108907699585,
      "logits/rejected": -0.14832782745361328,
      "logps/chosen": -412.6355285644531,
      "logps/rejected": -388.03106689453125,
      "loss": 0.5389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2892241477966309,
      "rewards/margins": 0.5657732486724854,
      "rewards/rejected": -1.854997158050537,
      "step": 9453
    },
    {
      "epoch": 1.2371145824813936,
      "grad_norm": 8.711581230163574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38755562901496887,
      "logits/rejected": -0.34525197744369507,
      "logps/chosen": -617.159423828125,
      "logps/rejected": -532.2425537109375,
      "loss": 0.3963,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2697621583938599,
      "rewards/margins": 1.0138221979141235,
      "rewards/rejected": -2.2835843563079834,
      "step": 9454
    },
    {
      "epoch": 1.2372454404187454,
      "grad_norm": 10.889718055725098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05597112700343132,
      "logits/rejected": 0.06079027056694031,
      "logps/chosen": -379.7582702636719,
      "logps/rejected": -391.5085754394531,
      "loss": 0.7913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6637604236602783,
      "rewards/margins": 0.13564705848693848,
      "rewards/rejected": -1.7994073629379272,
      "step": 9455
    },
    {
      "epoch": 1.2373762983560972,
      "grad_norm": 5.5393900871276855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04183941334486008,
      "logits/rejected": -0.21104225516319275,
      "logps/chosen": -425.0572204589844,
      "logps/rejected": -451.9268493652344,
      "loss": 0.2956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0210071802139282,
      "rewards/margins": 1.6770555973052979,
      "rewards/rejected": -2.6980626583099365,
      "step": 9456
    },
    {
      "epoch": 1.237507156293449,
      "grad_norm": 5.3021063804626465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13833637535572052,
      "logits/rejected": -0.08545608818531036,
      "logps/chosen": -368.2890930175781,
      "logps/rejected": -395.1342468261719,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3630521297454834,
      "rewards/margins": 1.7984592914581299,
      "rewards/rejected": -3.1615114212036133,
      "step": 9457
    },
    {
      "epoch": 1.2376380142308008,
      "grad_norm": 7.893573760986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046371884644031525,
      "logits/rejected": 0.010592177510261536,
      "logps/chosen": -415.51409912109375,
      "logps/rejected": -550.3389282226562,
      "loss": 0.6027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5678510665893555,
      "rewards/margins": 0.9248600602149963,
      "rewards/rejected": -2.492711067199707,
      "step": 9458
    },
    {
      "epoch": 1.2377688721681523,
      "grad_norm": 7.796625137329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36420169472694397,
      "logits/rejected": -0.3858884572982788,
      "logps/chosen": -364.344970703125,
      "logps/rejected": -382.4487609863281,
      "loss": 0.3037,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7029385566711426,
      "rewards/margins": 1.8548729419708252,
      "rewards/rejected": -2.5578112602233887,
      "step": 9459
    },
    {
      "epoch": 1.2378997301055041,
      "grad_norm": 6.909698486328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08491422981023788,
      "logits/rejected": -0.1026117205619812,
      "logps/chosen": -508.275390625,
      "logps/rejected": -523.9046630859375,
      "loss": 0.4957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7171050310134888,
      "rewards/margins": 0.7250782251358032,
      "rewards/rejected": -2.442183256149292,
      "step": 9460
    },
    {
      "epoch": 1.238030588042856,
      "grad_norm": 8.408523559570312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19766496121883392,
      "logits/rejected": 0.10334765166044235,
      "logps/chosen": -439.1997985839844,
      "logps/rejected": -433.58160400390625,
      "loss": 0.5299,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0353610515594482,
      "rewards/margins": 0.9528625011444092,
      "rewards/rejected": -1.988223671913147,
      "step": 9461
    },
    {
      "epoch": 1.2381614459802077,
      "grad_norm": 6.2640533447265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14204932749271393,
      "logits/rejected": -0.23920847475528717,
      "logps/chosen": -538.523681640625,
      "logps/rejected": -420.8372802734375,
      "loss": 0.3115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.511568546295166,
      "rewards/margins": 1.969977855682373,
      "rewards/rejected": -3.481546401977539,
      "step": 9462
    },
    {
      "epoch": 1.2382923039175595,
      "grad_norm": 7.819300651550293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37659555673599243,
      "logits/rejected": -0.24409674108028412,
      "logps/chosen": -541.3817138671875,
      "logps/rejected": -504.22216796875,
      "loss": 0.4617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8952356576919556,
      "rewards/margins": 1.160325288772583,
      "rewards/rejected": -3.055561065673828,
      "step": 9463
    },
    {
      "epoch": 1.2384231618549113,
      "grad_norm": 3.9946348667144775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21706204116344452,
      "logits/rejected": -0.2839849293231964,
      "logps/chosen": -471.3544921875,
      "logps/rejected": -520.1976928710938,
      "loss": 0.1903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4393922686576843,
      "rewards/margins": 2.991281509399414,
      "rewards/rejected": -3.430673837661743,
      "step": 9464
    },
    {
      "epoch": 1.238554019792263,
      "grad_norm": 4.3258795738220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0006962611805647612,
      "logits/rejected": -0.07569454610347748,
      "logps/chosen": -373.4683837890625,
      "logps/rejected": -416.28826904296875,
      "loss": 0.2403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3379017114639282,
      "rewards/margins": 2.352891445159912,
      "rewards/rejected": -2.69079327583313,
      "step": 9465
    },
    {
      "epoch": 1.2386848777296149,
      "grad_norm": 6.7272868156433105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027605952695012093,
      "logits/rejected": -0.20094475150108337,
      "logps/chosen": -749.5782470703125,
      "logps/rejected": -440.79315185546875,
      "loss": 0.3734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0991657972335815,
      "rewards/margins": 1.0483195781707764,
      "rewards/rejected": -2.1474854946136475,
      "step": 9466
    },
    {
      "epoch": 1.2388157356669665,
      "grad_norm": 10.646425247192383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11199186742305756,
      "logits/rejected": -0.24628710746765137,
      "logps/chosen": -449.9892272949219,
      "logps/rejected": -351.2730712890625,
      "loss": 0.39,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1550477743148804,
      "rewards/margins": 1.8493292331695557,
      "rewards/rejected": -3.0043771266937256,
      "step": 9467
    },
    {
      "epoch": 1.2389465936043182,
      "grad_norm": 6.124941349029541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4740946292877197,
      "logits/rejected": -0.41049447655677795,
      "logps/chosen": -338.59930419921875,
      "logps/rejected": -390.07464599609375,
      "loss": 0.3885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6004085540771484,
      "rewards/margins": 1.1913782358169556,
      "rewards/rejected": -2.7917869091033936,
      "step": 9468
    },
    {
      "epoch": 1.23907745154167,
      "grad_norm": 6.4023613929748535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16151393949985504,
      "logits/rejected": -0.34262222051620483,
      "logps/chosen": -371.8391418457031,
      "logps/rejected": -356.0727844238281,
      "loss": 0.3235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9256477355957031,
      "rewards/margins": 1.3397245407104492,
      "rewards/rejected": -2.2653725147247314,
      "step": 9469
    },
    {
      "epoch": 1.2392083094790218,
      "grad_norm": 5.709780693054199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5496060252189636,
      "logits/rejected": -0.45941591262817383,
      "logps/chosen": -376.034912109375,
      "logps/rejected": -407.0428771972656,
      "loss": 0.2517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.171097993850708,
      "rewards/margins": 1.6803603172302246,
      "rewards/rejected": -3.8514580726623535,
      "step": 9470
    },
    {
      "epoch": 1.2393391674163736,
      "grad_norm": 7.938528060913086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17907992005348206,
      "logits/rejected": -0.025452367961406708,
      "logps/chosen": -435.1877136230469,
      "logps/rejected": -476.9940185546875,
      "loss": 0.3779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2950855493545532,
      "rewards/margins": 1.317693829536438,
      "rewards/rejected": -2.612779378890991,
      "step": 9471
    },
    {
      "epoch": 1.2394700253537254,
      "grad_norm": 10.43409252166748,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19186235964298248,
      "logits/rejected": 0.02635282836854458,
      "logps/chosen": -341.0235290527344,
      "logps/rejected": -464.8753356933594,
      "loss": 0.809,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.5361618995666504,
      "rewards/margins": 0.11259114742279053,
      "rewards/rejected": -1.648753046989441,
      "step": 9472
    },
    {
      "epoch": 1.2396008832910772,
      "grad_norm": 2.1083104610443115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08619524538516998,
      "logits/rejected": -0.19363664090633392,
      "logps/chosen": -494.3949890136719,
      "logps/rejected": -469.011474609375,
      "loss": 0.1503,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8028637766838074,
      "rewards/margins": 2.743739128112793,
      "rewards/rejected": -3.546602725982666,
      "step": 9473
    },
    {
      "epoch": 1.2397317412284288,
      "grad_norm": 9.231751441955566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5352375507354736,
      "logits/rejected": -0.3149833083152771,
      "logps/chosen": -377.1839599609375,
      "logps/rejected": -432.682861328125,
      "loss": 0.6487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1929395198822021,
      "rewards/margins": 1.077714204788208,
      "rewards/rejected": -2.27065372467041,
      "step": 9474
    },
    {
      "epoch": 1.2398625991657806,
      "grad_norm": 4.035912990570068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1148696094751358,
      "logits/rejected": -0.3115865886211395,
      "logps/chosen": -539.2352905273438,
      "logps/rejected": -488.3594665527344,
      "loss": 0.1185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28117525577545166,
      "rewards/margins": 2.595801591873169,
      "rewards/rejected": -2.876976728439331,
      "step": 9475
    },
    {
      "epoch": 1.2399934571031324,
      "grad_norm": 5.233267784118652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2929537296295166,
      "logits/rejected": -0.3466992676258087,
      "logps/chosen": -312.20806884765625,
      "logps/rejected": -408.18316650390625,
      "loss": 0.2861,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9293421506881714,
      "rewards/margins": 1.9159302711486816,
      "rewards/rejected": -2.8452725410461426,
      "step": 9476
    },
    {
      "epoch": 1.2401243150404841,
      "grad_norm": 7.640724182128906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020384803414344788,
      "logits/rejected": 0.014512434601783752,
      "logps/chosen": -427.2785339355469,
      "logps/rejected": -458.35711669921875,
      "loss": 0.4202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3554556369781494,
      "rewards/margins": 1.1694707870483398,
      "rewards/rejected": -2.5249264240264893,
      "step": 9477
    },
    {
      "epoch": 1.240255172977836,
      "grad_norm": 12.411545753479004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3280659019947052,
      "logits/rejected": -0.43202072381973267,
      "logps/chosen": -586.0282592773438,
      "logps/rejected": -433.991455078125,
      "loss": 0.5357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9406622648239136,
      "rewards/margins": 1.04348886013031,
      "rewards/rejected": -2.9841511249542236,
      "step": 9478
    },
    {
      "epoch": 1.2403860309151877,
      "grad_norm": 3.412177324295044,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00034406594932079315,
      "logits/rejected": -0.062397077679634094,
      "logps/chosen": -414.8197326660156,
      "logps/rejected": -422.65447998046875,
      "loss": 0.155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9187978506088257,
      "rewards/margins": 2.9145865440368652,
      "rewards/rejected": -3.8333840370178223,
      "step": 9479
    },
    {
      "epoch": 1.2405168888525395,
      "grad_norm": 5.692127227783203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.011388789862394333,
      "logits/rejected": -0.035174351185560226,
      "logps/chosen": -480.517333984375,
      "logps/rejected": -456.0439758300781,
      "loss": 0.3492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.199357032775879,
      "rewards/margins": 1.104512333869934,
      "rewards/rejected": -2.3038692474365234,
      "step": 9480
    },
    {
      "epoch": 1.2406477467898913,
      "grad_norm": 7.752847194671631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31755438446998596,
      "logits/rejected": -0.09124985337257385,
      "logps/chosen": -281.59552001953125,
      "logps/rejected": -383.56195068359375,
      "loss": 0.487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5766222476959229,
      "rewards/margins": 0.7891656160354614,
      "rewards/rejected": -2.3657877445220947,
      "step": 9481
    },
    {
      "epoch": 1.240778604727243,
      "grad_norm": 7.487089157104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5392411351203918,
      "logits/rejected": -0.6083385944366455,
      "logps/chosen": -418.68560791015625,
      "logps/rejected": -447.79833984375,
      "loss": 0.4257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.233607053756714,
      "rewards/margins": 1.7429734468460083,
      "rewards/rejected": -3.9765803813934326,
      "step": 9482
    },
    {
      "epoch": 1.2409094626645947,
      "grad_norm": 5.665940761566162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16660542786121368,
      "logits/rejected": -0.1077420637011528,
      "logps/chosen": -443.5119934082031,
      "logps/rejected": -443.4571533203125,
      "loss": 0.3065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6640230417251587,
      "rewards/margins": 1.7412371635437012,
      "rewards/rejected": -2.4052603244781494,
      "step": 9483
    },
    {
      "epoch": 1.2410403206019465,
      "grad_norm": 7.069566249847412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13569268584251404,
      "logits/rejected": -0.21196332573890686,
      "logps/chosen": -393.2008972167969,
      "logps/rejected": -470.2458190917969,
      "loss": 0.4132,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9794083833694458,
      "rewards/margins": 1.0190778970718384,
      "rewards/rejected": -2.998486280441284,
      "step": 9484
    },
    {
      "epoch": 1.2411711785392983,
      "grad_norm": 4.4049859046936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34851202368736267,
      "logits/rejected": -0.5844776034355164,
      "logps/chosen": -421.1975402832031,
      "logps/rejected": -376.4031066894531,
      "loss": 0.187,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4456286430358887,
      "rewards/margins": 2.096947431564331,
      "rewards/rejected": -3.5425760746002197,
      "step": 9485
    },
    {
      "epoch": 1.24130203647665,
      "grad_norm": 13.26701545715332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5455280542373657,
      "logits/rejected": -0.47671860456466675,
      "logps/chosen": -484.8486633300781,
      "logps/rejected": -385.7647399902344,
      "loss": 0.7091,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.663940191268921,
      "rewards/margins": 0.5741007328033447,
      "rewards/rejected": -2.2380409240722656,
      "step": 9486
    },
    {
      "epoch": 1.2414328944140018,
      "grad_norm": 8.041717529296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17022418975830078,
      "logits/rejected": -0.12936802208423615,
      "logps/chosen": -375.6178894042969,
      "logps/rejected": -447.38580322265625,
      "loss": 0.3774,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2250200510025024,
      "rewards/margins": 1.7875257730484009,
      "rewards/rejected": -3.0125458240509033,
      "step": 9487
    },
    {
      "epoch": 1.2415637523513536,
      "grad_norm": 17.357633590698242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28615906834602356,
      "logits/rejected": -0.14107881486415863,
      "logps/chosen": -513.5612182617188,
      "logps/rejected": -500.3729248046875,
      "loss": 1.2078,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.4219822883605957,
      "rewards/margins": -0.14205436408519745,
      "rewards/rejected": -2.279928207397461,
      "step": 9488
    },
    {
      "epoch": 1.2416946102887052,
      "grad_norm": 10.140437126159668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0034166499972343445,
      "logits/rejected": -0.15992194414138794,
      "logps/chosen": -530.22900390625,
      "logps/rejected": -456.2396240234375,
      "loss": 0.7404,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.835512399673462,
      "rewards/margins": 0.29086464643478394,
      "rewards/rejected": -2.1263771057128906,
      "step": 9489
    },
    {
      "epoch": 1.241825468226057,
      "grad_norm": 7.434084415435791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3492478132247925,
      "logits/rejected": -0.5028815269470215,
      "logps/chosen": -400.17816162109375,
      "logps/rejected": -421.10589599609375,
      "loss": 0.372,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7698725461959839,
      "rewards/margins": 2.21799373626709,
      "rewards/rejected": -3.9878664016723633,
      "step": 9490
    },
    {
      "epoch": 1.2419563261634088,
      "grad_norm": 4.5312886238098145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28019043803215027,
      "logits/rejected": -0.360727995634079,
      "logps/chosen": -600.4818725585938,
      "logps/rejected": -522.6868896484375,
      "loss": 0.2391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34027326107025146,
      "rewards/margins": 2.1166932582855225,
      "rewards/rejected": -2.4569664001464844,
      "step": 9491
    },
    {
      "epoch": 1.2420871841007606,
      "grad_norm": 11.613506317138672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17179465293884277,
      "logits/rejected": -0.3171020746231079,
      "logps/chosen": -549.0299072265625,
      "logps/rejected": -594.51318359375,
      "loss": 0.6268,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4888789653778076,
      "rewards/margins": 0.3966381549835205,
      "rewards/rejected": -2.885517120361328,
      "step": 9492
    },
    {
      "epoch": 1.2422180420381124,
      "grad_norm": 6.251895427703857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13960102200508118,
      "logits/rejected": -0.3932155966758728,
      "logps/chosen": -494.7142639160156,
      "logps/rejected": -562.3396606445312,
      "loss": 0.4422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4923231601715088,
      "rewards/margins": 1.0694962739944458,
      "rewards/rejected": -2.561819553375244,
      "step": 9493
    },
    {
      "epoch": 1.2423488999754642,
      "grad_norm": 9.498098373413086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0222022645175457,
      "logits/rejected": -0.0347348228096962,
      "logps/chosen": -372.0059814453125,
      "logps/rejected": -368.79132080078125,
      "loss": 0.5482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.44551682472229,
      "rewards/margins": 1.0076448917388916,
      "rewards/rejected": -2.4531614780426025,
      "step": 9494
    },
    {
      "epoch": 1.242479757912816,
      "grad_norm": 5.1419782638549805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28809988498687744,
      "logits/rejected": -0.3781808018684387,
      "logps/chosen": -537.5904541015625,
      "logps/rejected": -435.389892578125,
      "loss": 0.2153,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5888805389404297,
      "rewards/margins": 2.4410226345062256,
      "rewards/rejected": -4.029903411865234,
      "step": 9495
    },
    {
      "epoch": 1.2426106158501677,
      "grad_norm": 11.022085189819336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2950655519962311,
      "logits/rejected": -0.14484703540802002,
      "logps/chosen": -417.3459167480469,
      "logps/rejected": -431.3907165527344,
      "loss": 0.7278,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.21744966506958,
      "rewards/margins": 0.3669412434101105,
      "rewards/rejected": -2.584390878677368,
      "step": 9496
    },
    {
      "epoch": 1.2427414737875195,
      "grad_norm": 9.262394905090332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.192903071641922,
      "logits/rejected": -0.5444737076759338,
      "logps/chosen": -439.8143310546875,
      "logps/rejected": -476.78204345703125,
      "loss": 0.772,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1359388828277588,
      "rewards/margins": 0.44990381598472595,
      "rewards/rejected": -1.5858428478240967,
      "step": 9497
    },
    {
      "epoch": 1.242872331724871,
      "grad_norm": 6.694897174835205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3784092962741852,
      "logits/rejected": -0.41038578748703003,
      "logps/chosen": -582.4241943359375,
      "logps/rejected": -560.0653076171875,
      "loss": 0.2955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2394945621490479,
      "rewards/margins": 2.1232733726501465,
      "rewards/rejected": -3.3627681732177734,
      "step": 9498
    },
    {
      "epoch": 1.243003189662223,
      "grad_norm": 5.277898788452148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09285889565944672,
      "logits/rejected": -0.09554614126682281,
      "logps/chosen": -581.436767578125,
      "logps/rejected": -393.41217041015625,
      "loss": 0.2894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6790772676467896,
      "rewards/margins": 1.2107495069503784,
      "rewards/rejected": -2.889827013015747,
      "step": 9499
    },
    {
      "epoch": 1.2431340475995747,
      "grad_norm": 9.353184700012207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20564165711402893,
      "logits/rejected": -0.7233773469924927,
      "logps/chosen": -319.6168518066406,
      "logps/rejected": -363.41741943359375,
      "loss": 0.518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7713713645935059,
      "rewards/margins": 0.8579806089401245,
      "rewards/rejected": -2.629351854324341,
      "step": 9500
    },
    {
      "epoch": 1.2432649055369265,
      "grad_norm": 5.827040672302246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2944807708263397,
      "logits/rejected": -0.15165746212005615,
      "logps/chosen": -293.51190185546875,
      "logps/rejected": -382.3504943847656,
      "loss": 0.2539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1428927183151245,
      "rewards/margins": 1.3867182731628418,
      "rewards/rejected": -2.5296108722686768,
      "step": 9501
    },
    {
      "epoch": 1.2433957634742783,
      "grad_norm": 16.542388916015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2064398229122162,
      "logits/rejected": -0.4919581413269043,
      "logps/chosen": -526.185546875,
      "logps/rejected": -422.21600341796875,
      "loss": 0.7631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.5448734760284424,
      "rewards/margins": 0.6623624563217163,
      "rewards/rejected": -3.207235813140869,
      "step": 9502
    },
    {
      "epoch": 1.24352662141163,
      "grad_norm": 11.319907188415527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1208883747458458,
      "logits/rejected": -0.11264883726835251,
      "logps/chosen": -532.3856201171875,
      "logps/rejected": -431.96917724609375,
      "loss": 0.4545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4113770723342896,
      "rewards/margins": 1.2231680154800415,
      "rewards/rejected": -2.63454532623291,
      "step": 9503
    },
    {
      "epoch": 1.2436574793489819,
      "grad_norm": 12.097543716430664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05842534452676773,
      "logits/rejected": -0.25846606492996216,
      "logps/chosen": -489.1705322265625,
      "logps/rejected": -497.8926086425781,
      "loss": 0.6083,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.355515956878662,
      "rewards/margins": 1.0340723991394043,
      "rewards/rejected": -2.3895881175994873,
      "step": 9504
    },
    {
      "epoch": 1.2437883372863334,
      "grad_norm": 18.3419246673584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13291841745376587,
      "logits/rejected": -0.179274782538414,
      "logps/chosen": -361.1083984375,
      "logps/rejected": -371.32977294921875,
      "loss": 1.114,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.260129690170288,
      "rewards/margins": -0.503413736820221,
      "rewards/rejected": -1.756716012954712,
      "step": 9505
    },
    {
      "epoch": 1.2439191952236852,
      "grad_norm": 7.990944862365723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4583454132080078,
      "logits/rejected": -0.47246479988098145,
      "logps/chosen": -419.3248291015625,
      "logps/rejected": -437.51324462890625,
      "loss": 0.434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.956994652748108,
      "rewards/margins": 0.9129756689071655,
      "rewards/rejected": -2.8699703216552734,
      "step": 9506
    },
    {
      "epoch": 1.244050053161037,
      "grad_norm": 8.17294979095459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29029548168182373,
      "logits/rejected": -0.420024573802948,
      "logps/chosen": -447.3288269042969,
      "logps/rejected": -446.9485778808594,
      "loss": 0.3982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.547928810119629,
      "rewards/margins": 0.8933538198471069,
      "rewards/rejected": -2.4412825107574463,
      "step": 9507
    },
    {
      "epoch": 1.2441809110983888,
      "grad_norm": 7.429690361022949,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5009796023368835,
      "logits/rejected": -0.47533950209617615,
      "logps/chosen": -581.6371459960938,
      "logps/rejected": -497.98248291015625,
      "loss": 0.4427,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.163060426712036,
      "rewards/margins": 0.6515827178955078,
      "rewards/rejected": -2.814643144607544,
      "step": 9508
    },
    {
      "epoch": 1.2443117690357406,
      "grad_norm": 9.909486770629883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22930794954299927,
      "logits/rejected": -0.4458874464035034,
      "logps/chosen": -460.7581787109375,
      "logps/rejected": -440.5886535644531,
      "loss": 0.6459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2117919921875,
      "rewards/margins": 0.3857685327529907,
      "rewards/rejected": -1.5975604057312012,
      "step": 9509
    },
    {
      "epoch": 1.2444426269730924,
      "grad_norm": 7.556413650512695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3186747133731842,
      "logits/rejected": -0.2629082500934601,
      "logps/chosen": -393.86065673828125,
      "logps/rejected": -418.92010498046875,
      "loss": 0.4905,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9832864999771118,
      "rewards/margins": 1.1177505254745483,
      "rewards/rejected": -3.1010372638702393,
      "step": 9510
    },
    {
      "epoch": 1.2445734849104442,
      "grad_norm": 7.842717170715332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03558871150016785,
      "logits/rejected": -0.17334601283073425,
      "logps/chosen": -488.1015930175781,
      "logps/rejected": -417.23095703125,
      "loss": 0.3985,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9221917390823364,
      "rewards/margins": 1.4792927503585815,
      "rewards/rejected": -2.401484489440918,
      "step": 9511
    },
    {
      "epoch": 1.244704342847796,
      "grad_norm": 2.7364611625671387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06005315110087395,
      "logits/rejected": -0.39670825004577637,
      "logps/chosen": -560.0158081054688,
      "logps/rejected": -431.0201721191406,
      "loss": 0.1204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7380070090293884,
      "rewards/margins": 2.2868969440460205,
      "rewards/rejected": -3.0249037742614746,
      "step": 9512
    },
    {
      "epoch": 1.2448352007851475,
      "grad_norm": 6.727363586425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2666468024253845,
      "logits/rejected": -0.25006091594696045,
      "logps/chosen": -395.70648193359375,
      "logps/rejected": -422.19073486328125,
      "loss": 0.3299,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6995863914489746,
      "rewards/margins": 1.3620846271514893,
      "rewards/rejected": -3.0616707801818848,
      "step": 9513
    },
    {
      "epoch": 1.2449660587224993,
      "grad_norm": 7.718424320220947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1528596431016922,
      "logits/rejected": -0.055392004549503326,
      "logps/chosen": -502.4190673828125,
      "logps/rejected": -521.3299560546875,
      "loss": 0.2854,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.946936845779419,
      "rewards/margins": 1.797969102859497,
      "rewards/rejected": -2.744905948638916,
      "step": 9514
    },
    {
      "epoch": 1.2450969166598511,
      "grad_norm": 8.683539390563965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17159470915794373,
      "logits/rejected": -0.48455488681793213,
      "logps/chosen": -435.9565124511719,
      "logps/rejected": -453.45391845703125,
      "loss": 0.6323,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7378727197647095,
      "rewards/margins": 0.4598537087440491,
      "rewards/rejected": -2.1977264881134033,
      "step": 9515
    },
    {
      "epoch": 1.245227774597203,
      "grad_norm": 8.649728775024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30819690227508545,
      "logits/rejected": -0.5050143599510193,
      "logps/chosen": -593.5742797851562,
      "logps/rejected": -438.9161376953125,
      "loss": 0.49,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3367445468902588,
      "rewards/margins": 1.8616927862167358,
      "rewards/rejected": -3.198437452316284,
      "step": 9516
    },
    {
      "epoch": 1.2453586325345547,
      "grad_norm": 4.237759590148926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11382681131362915,
      "logits/rejected": -0.25941094756126404,
      "logps/chosen": -473.93902587890625,
      "logps/rejected": -450.68841552734375,
      "loss": 0.2383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12536926567554474,
      "rewards/margins": 2.1821203231811523,
      "rewards/rejected": -2.056751012802124,
      "step": 9517
    },
    {
      "epoch": 1.2454894904719065,
      "grad_norm": 7.741363525390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18358615040779114,
      "logits/rejected": -0.2266538441181183,
      "logps/chosen": -573.6993408203125,
      "logps/rejected": -544.130859375,
      "loss": 0.3109,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6272628903388977,
      "rewards/margins": 2.191462516784668,
      "rewards/rejected": -2.818725109100342,
      "step": 9518
    },
    {
      "epoch": 1.2456203484092583,
      "grad_norm": 4.744677543640137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4034320116043091,
      "logits/rejected": -0.48759984970092773,
      "logps/chosen": -512.4175415039062,
      "logps/rejected": -603.65576171875,
      "loss": 0.3017,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.102083683013916,
      "rewards/margins": 1.7817853689193726,
      "rewards/rejected": -2.883868932723999,
      "step": 9519
    },
    {
      "epoch": 1.2457512063466099,
      "grad_norm": 7.81146240234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3083539307117462,
      "logits/rejected": -0.45441269874572754,
      "logps/chosen": -413.28643798828125,
      "logps/rejected": -407.984375,
      "loss": 0.4466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0130958557128906,
      "rewards/margins": 1.2170615196228027,
      "rewards/rejected": -2.2301571369171143,
      "step": 9520
    },
    {
      "epoch": 1.2458820642839616,
      "grad_norm": 8.396409034729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41468486189842224,
      "logits/rejected": -0.029856868088245392,
      "logps/chosen": -375.8241271972656,
      "logps/rejected": -271.4293518066406,
      "loss": 0.4529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3229436874389648,
      "rewards/margins": 1.5485256910324097,
      "rewards/rejected": -2.871469259262085,
      "step": 9521
    },
    {
      "epoch": 1.2460129222213134,
      "grad_norm": 6.81760311126709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12316720187664032,
      "logits/rejected": -0.35463133454322815,
      "logps/chosen": -373.19647216796875,
      "logps/rejected": -457.7037353515625,
      "loss": 0.3043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2848819494247437,
      "rewards/margins": 1.8069833517074585,
      "rewards/rejected": -3.091865301132202,
      "step": 9522
    },
    {
      "epoch": 1.2461437801586652,
      "grad_norm": 5.999147415161133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15287944674491882,
      "logits/rejected": -0.03243274986743927,
      "logps/chosen": -587.6158447265625,
      "logps/rejected": -544.8106079101562,
      "loss": 0.3019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7362529635429382,
      "rewards/margins": 1.399352788925171,
      "rewards/rejected": -2.135606050491333,
      "step": 9523
    },
    {
      "epoch": 1.246274638096017,
      "grad_norm": 6.579927444458008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013680268079042435,
      "logits/rejected": -0.08880501985549927,
      "logps/chosen": -412.1002502441406,
      "logps/rejected": -375.4266357421875,
      "loss": 0.293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8496511578559875,
      "rewards/margins": 1.6684292554855347,
      "rewards/rejected": -2.518080234527588,
      "step": 9524
    },
    {
      "epoch": 1.2464054960333688,
      "grad_norm": 11.712739944458008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3288182020187378,
      "logits/rejected": -0.5026946663856506,
      "logps/chosen": -558.7420654296875,
      "logps/rejected": -426.1976623535156,
      "loss": 0.7533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1106348037719727,
      "rewards/margins": 0.4972836971282959,
      "rewards/rejected": -2.6079185009002686,
      "step": 9525
    },
    {
      "epoch": 1.2465363539707206,
      "grad_norm": 11.51718807220459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34048187732696533,
      "logits/rejected": -0.34959617257118225,
      "logps/chosen": -536.1701049804688,
      "logps/rejected": -412.1542053222656,
      "loss": 0.7436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0212128162384033,
      "rewards/margins": 0.3012468218803406,
      "rewards/rejected": -2.3224596977233887,
      "step": 9526
    },
    {
      "epoch": 1.2466672119080724,
      "grad_norm": 8.318244934082031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4930236041545868,
      "logits/rejected": -0.4379580616950989,
      "logps/chosen": -458.4234924316406,
      "logps/rejected": -418.8397521972656,
      "loss": 0.4837,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1060962677001953,
      "rewards/margins": 1.3956949710845947,
      "rewards/rejected": -2.50179123878479,
      "step": 9527
    },
    {
      "epoch": 1.2467980698454242,
      "grad_norm": 3.841170310974121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3463975191116333,
      "logits/rejected": 0.1790274679660797,
      "logps/chosen": -542.4202270507812,
      "logps/rejected": -559.8641967773438,
      "loss": 0.1334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6189610362052917,
      "rewards/margins": 2.5174663066864014,
      "rewards/rejected": -3.136427640914917,
      "step": 9528
    },
    {
      "epoch": 1.2469289277827758,
      "grad_norm": 6.131917476654053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4752978980541229,
      "logits/rejected": -0.5408497452735901,
      "logps/chosen": -347.5461730957031,
      "logps/rejected": -307.7113952636719,
      "loss": 0.5028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1219239234924316,
      "rewards/margins": 0.7705816030502319,
      "rewards/rejected": -1.8925055265426636,
      "step": 9529
    },
    {
      "epoch": 1.2470597857201275,
      "grad_norm": 10.773130416870117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34154531359672546,
      "logits/rejected": -0.4155736267566681,
      "logps/chosen": -651.3668212890625,
      "logps/rejected": -558.7452392578125,
      "loss": 0.4647,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.123482346534729,
      "rewards/margins": 1.569441556930542,
      "rewards/rejected": -2.6929240226745605,
      "step": 9530
    },
    {
      "epoch": 1.2471906436574793,
      "grad_norm": 7.353824615478516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15702803432941437,
      "logits/rejected": 0.07267733663320541,
      "logps/chosen": -670.099853515625,
      "logps/rejected": -609.9212036132812,
      "loss": 0.4231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37861865758895874,
      "rewards/margins": 1.2750048637390137,
      "rewards/rejected": -1.653623342514038,
      "step": 9531
    },
    {
      "epoch": 1.2473215015948311,
      "grad_norm": 5.666076183319092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007049389183521271,
      "logits/rejected": -0.2890477180480957,
      "logps/chosen": -439.26025390625,
      "logps/rejected": -385.01947021484375,
      "loss": 0.3557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5075504183769226,
      "rewards/margins": 1.3072407245635986,
      "rewards/rejected": -1.814790964126587,
      "step": 9532
    },
    {
      "epoch": 1.247452359532183,
      "grad_norm": 7.023403167724609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05312076583504677,
      "logits/rejected": -0.35539939999580383,
      "logps/chosen": -518.9606323242188,
      "logps/rejected": -507.02471923828125,
      "loss": 0.2774,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0662873983383179,
      "rewards/margins": 1.818819284439087,
      "rewards/rejected": -2.8851065635681152,
      "step": 9533
    },
    {
      "epoch": 1.2475832174695347,
      "grad_norm": 5.07468318939209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17841526865959167,
      "logits/rejected": -0.44781604409217834,
      "logps/chosen": -478.01068115234375,
      "logps/rejected": -351.9292297363281,
      "loss": 0.2292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4207144975662231,
      "rewards/margins": 2.4220399856567383,
      "rewards/rejected": -3.842754364013672,
      "step": 9534
    },
    {
      "epoch": 1.2477140754068863,
      "grad_norm": 11.18848705291748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12558701634407043,
      "logits/rejected": -0.10269627720117569,
      "logps/chosen": -416.23583984375,
      "logps/rejected": -404.03509521484375,
      "loss": 0.6072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0402638912200928,
      "rewards/margins": 1.1458371877670288,
      "rewards/rejected": -3.186101198196411,
      "step": 9535
    },
    {
      "epoch": 1.247844933344238,
      "grad_norm": 15.776153564453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05324331670999527,
      "logits/rejected": -0.1161753237247467,
      "logps/chosen": -431.79595947265625,
      "logps/rejected": -462.1044921875,
      "loss": 0.6757,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.51565420627594,
      "rewards/margins": 0.5635137557983398,
      "rewards/rejected": -2.0791680812835693,
      "step": 9536
    },
    {
      "epoch": 1.2479757912815899,
      "grad_norm": 7.972426414489746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2823440134525299,
      "logits/rejected": -0.36703988909721375,
      "logps/chosen": -527.938232421875,
      "logps/rejected": -416.3359375,
      "loss": 0.4798,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2080919742584229,
      "rewards/margins": 1.0168941020965576,
      "rewards/rejected": -2.2249858379364014,
      "step": 9537
    },
    {
      "epoch": 1.2481066492189417,
      "grad_norm": 7.262617111206055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052953965961933136,
      "logits/rejected": -0.02733100950717926,
      "logps/chosen": -433.51531982421875,
      "logps/rejected": -442.92724609375,
      "loss": 0.5356,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3545761108398438,
      "rewards/margins": 0.738817572593689,
      "rewards/rejected": -2.093393564224243,
      "step": 9538
    },
    {
      "epoch": 1.2482375071562934,
      "grad_norm": 27.041303634643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49820446968078613,
      "logits/rejected": -0.3073589503765106,
      "logps/chosen": -945.7802734375,
      "logps/rejected": -368.0307922363281,
      "loss": 1.1983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.2979683876037598,
      "rewards/margins": 0.02585279941558838,
      "rewards/rejected": -2.3238208293914795,
      "step": 9539
    },
    {
      "epoch": 1.2483683650936452,
      "grad_norm": 5.625452518463135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10204349458217621,
      "logits/rejected": -0.08001118898391724,
      "logps/chosen": -359.33917236328125,
      "logps/rejected": -345.5843200683594,
      "loss": 0.3207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8938377499580383,
      "rewards/margins": 1.337317943572998,
      "rewards/rejected": -2.2311556339263916,
      "step": 9540
    },
    {
      "epoch": 1.248499223030997,
      "grad_norm": 13.040794372558594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1870097517967224,
      "logits/rejected": -0.36710885167121887,
      "logps/chosen": -564.4242553710938,
      "logps/rejected": -542.6272583007812,
      "loss": 0.8258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1746962070465088,
      "rewards/margins": 0.7509305477142334,
      "rewards/rejected": -1.9256268739700317,
      "step": 9541
    },
    {
      "epoch": 1.2486300809683488,
      "grad_norm": 6.842928409576416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25332948565483093,
      "logits/rejected": -0.48284921050071716,
      "logps/chosen": -398.43115234375,
      "logps/rejected": -409.8040466308594,
      "loss": 0.294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6854257583618164,
      "rewards/margins": 1.2938419580459595,
      "rewards/rejected": -2.9792675971984863,
      "step": 9542
    },
    {
      "epoch": 1.2487609389057006,
      "grad_norm": 9.654258728027344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16218268871307373,
      "logits/rejected": -0.20900002121925354,
      "logps/chosen": -545.0615844726562,
      "logps/rejected": -388.9013366699219,
      "loss": 0.6601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8681786060333252,
      "rewards/margins": 0.9031803011894226,
      "rewards/rejected": -1.7713587284088135,
      "step": 9543
    },
    {
      "epoch": 1.2488917968430522,
      "grad_norm": 5.7985053062438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10678073018789291,
      "logits/rejected": -0.3416529595851898,
      "logps/chosen": -397.2020263671875,
      "logps/rejected": -406.40533447265625,
      "loss": 0.292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9079512357711792,
      "rewards/margins": 1.449153184890747,
      "rewards/rejected": -2.3571043014526367,
      "step": 9544
    },
    {
      "epoch": 1.249022654780404,
      "grad_norm": 7.028949737548828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01602356880903244,
      "logits/rejected": -0.05581596493721008,
      "logps/chosen": -585.792236328125,
      "logps/rejected": -541.330078125,
      "loss": 0.312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7749794125556946,
      "rewards/margins": 2.446563243865967,
      "rewards/rejected": -3.2215428352355957,
      "step": 9545
    },
    {
      "epoch": 1.2491535127177558,
      "grad_norm": 14.407553672790527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15915578603744507,
      "logits/rejected": -0.19240741431713104,
      "logps/chosen": -551.3157348632812,
      "logps/rejected": -457.7021789550781,
      "loss": 0.8429,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1707247495651245,
      "rewards/margins": 0.07846471667289734,
      "rewards/rejected": -1.2491894960403442,
      "step": 9546
    },
    {
      "epoch": 1.2492843706551076,
      "grad_norm": 7.296268939971924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1954762488603592,
      "logits/rejected": -0.32182228565216064,
      "logps/chosen": -401.6073913574219,
      "logps/rejected": -327.99591064453125,
      "loss": 0.3109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7731775641441345,
      "rewards/margins": 1.732823133468628,
      "rewards/rejected": -2.506000518798828,
      "step": 9547
    },
    {
      "epoch": 1.2494152285924593,
      "grad_norm": 9.745038986206055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5970398783683777,
      "logits/rejected": -0.4206467866897583,
      "logps/chosen": -334.06396484375,
      "logps/rejected": -454.6684875488281,
      "loss": 0.5387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9471599459648132,
      "rewards/margins": 1.005029559135437,
      "rewards/rejected": -1.9521894454956055,
      "step": 9548
    },
    {
      "epoch": 1.2495460865298111,
      "grad_norm": 5.419803142547607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29693251848220825,
      "logits/rejected": -0.22594425082206726,
      "logps/chosen": -402.8836364746094,
      "logps/rejected": -358.7388916015625,
      "loss": 0.2395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6778886914253235,
      "rewards/margins": 1.3868615627288818,
      "rewards/rejected": -2.0647501945495605,
      "step": 9549
    },
    {
      "epoch": 1.249676944467163,
      "grad_norm": 6.754729747772217,
      "learning_rate": 5e-06,
      "logits/chosen": -0.032652705907821655,
      "logits/rejected": 0.03340599685907364,
      "logps/chosen": -480.26971435546875,
      "logps/rejected": -478.6838684082031,
      "loss": 0.4115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4495672285556793,
      "rewards/margins": 1.1954618692398071,
      "rewards/rejected": -1.645029067993164,
      "step": 9550
    },
    {
      "epoch": 1.2498078024045145,
      "grad_norm": 8.202975273132324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05868725851178169,
      "logits/rejected": -0.23175184428691864,
      "logps/chosen": -601.534912109375,
      "logps/rejected": -504.61712646484375,
      "loss": 0.4023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3992712497711182,
      "rewards/margins": 1.5541114807128906,
      "rewards/rejected": -2.953382730484009,
      "step": 9551
    },
    {
      "epoch": 1.2499386603418663,
      "grad_norm": 5.440878868103027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10528473556041718,
      "logits/rejected": -0.04885274916887283,
      "logps/chosen": -470.20263671875,
      "logps/rejected": -414.80328369140625,
      "loss": 0.2889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8892300724983215,
      "rewards/margins": 1.5039854049682617,
      "rewards/rejected": -2.3932154178619385,
      "step": 9552
    },
    {
      "epoch": 1.250069518279218,
      "grad_norm": 3.3486242294311523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0006249938160181046,
      "logits/rejected": -0.07794453948736191,
      "logps/chosen": -742.5145874023438,
      "logps/rejected": -434.19488525390625,
      "loss": 0.1794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4350273311138153,
      "rewards/margins": 2.1286025047302246,
      "rewards/rejected": -2.5636298656463623,
      "step": 9553
    },
    {
      "epoch": 1.2502003762165699,
      "grad_norm": 11.793363571166992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5966666340827942,
      "logits/rejected": -0.6162713766098022,
      "logps/chosen": -523.63720703125,
      "logps/rejected": -654.70849609375,
      "loss": 0.8963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4490107297897339,
      "rewards/margins": 0.009835407137870789,
      "rewards/rejected": -1.458846092224121,
      "step": 9554
    },
    {
      "epoch": 1.2503312341539217,
      "grad_norm": 5.189924716949463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36009839177131653,
      "logits/rejected": -0.3362061381340027,
      "logps/chosen": -404.9180908203125,
      "logps/rejected": -418.82220458984375,
      "loss": 0.2447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8165642023086548,
      "rewards/margins": 1.877823829650879,
      "rewards/rejected": -2.6943881511688232,
      "step": 9555
    },
    {
      "epoch": 1.2504620920912735,
      "grad_norm": 11.647648811340332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3136722147464752,
      "logits/rejected": -0.3403325080871582,
      "logps/chosen": -432.3741760253906,
      "logps/rejected": -399.9394836425781,
      "loss": 0.7508,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.240460157394409,
      "rewards/margins": 0.43775802850723267,
      "rewards/rejected": -2.678218364715576,
      "step": 9556
    },
    {
      "epoch": 1.2505929500286252,
      "grad_norm": 9.426613807678223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3804779648780823,
      "logits/rejected": -0.3042336702346802,
      "logps/chosen": -555.5107421875,
      "logps/rejected": -556.3788452148438,
      "loss": 0.5611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3920271396636963,
      "rewards/margins": 0.7641093730926514,
      "rewards/rejected": -2.1561365127563477,
      "step": 9557
    },
    {
      "epoch": 1.250723807965977,
      "grad_norm": 5.855250835418701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2602728605270386,
      "logits/rejected": -0.4311975836753845,
      "logps/chosen": -374.76788330078125,
      "logps/rejected": -329.6136169433594,
      "loss": 0.4113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.323686957359314,
      "rewards/margins": 1.13393235206604,
      "rewards/rejected": -2.4576194286346436,
      "step": 9558
    },
    {
      "epoch": 1.2508546659033288,
      "grad_norm": 7.939109802246094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34488075971603394,
      "logits/rejected": -0.25346213579177856,
      "logps/chosen": -436.31768798828125,
      "logps/rejected": -453.5257873535156,
      "loss": 0.4465,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3161484003067017,
      "rewards/margins": 1.0226972103118896,
      "rewards/rejected": -2.3388454914093018,
      "step": 9559
    },
    {
      "epoch": 1.2509855238406804,
      "grad_norm": 4.712050437927246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15607108175754547,
      "logits/rejected": -0.08767801523208618,
      "logps/chosen": -757.647705078125,
      "logps/rejected": -551.6445922851562,
      "loss": 0.2487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5104109048843384,
      "rewards/margins": 1.7580816745758057,
      "rewards/rejected": -2.2684924602508545,
      "step": 9560
    },
    {
      "epoch": 1.2511163817780322,
      "grad_norm": 8.663729667663574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18559429049491882,
      "logits/rejected": 0.10519705712795258,
      "logps/chosen": -270.1170959472656,
      "logps/rejected": -395.3760070800781,
      "loss": 0.5197,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2247257232666016,
      "rewards/margins": 0.9094446897506714,
      "rewards/rejected": -2.1341702938079834,
      "step": 9561
    },
    {
      "epoch": 1.251247239715384,
      "grad_norm": 11.950851440429688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3281477093696594,
      "logits/rejected": -0.17495660483837128,
      "logps/chosen": -478.9438171386719,
      "logps/rejected": -489.2812805175781,
      "loss": 0.751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.284275531768799,
      "rewards/margins": 0.37497982382774353,
      "rewards/rejected": -2.659255266189575,
      "step": 9562
    },
    {
      "epoch": 1.2513780976527358,
      "grad_norm": 5.103269577026367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24508146941661835,
      "logits/rejected": 0.020920105278491974,
      "logps/chosen": -616.19140625,
      "logps/rejected": -608.1093139648438,
      "loss": 0.1798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1307017803192139,
      "rewards/margins": 2.300833225250244,
      "rewards/rejected": -3.431535005569458,
      "step": 9563
    },
    {
      "epoch": 1.2515089555900876,
      "grad_norm": 5.509862422943115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24041178822517395,
      "logits/rejected": -0.2253735363483429,
      "logps/chosen": -363.7770690917969,
      "logps/rejected": -400.5888671875,
      "loss": 0.2833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.596603274345398,
      "rewards/margins": 1.902820110321045,
      "rewards/rejected": -3.4994235038757324,
      "step": 9564
    },
    {
      "epoch": 1.2516398135274391,
      "grad_norm": 9.713644027709961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48239660263061523,
      "logits/rejected": -0.14137297868728638,
      "logps/chosen": -385.47967529296875,
      "logps/rejected": -457.0469055175781,
      "loss": 0.4422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.799177348613739,
      "rewards/margins": 1.3118510246276855,
      "rewards/rejected": -2.1110284328460693,
      "step": 9565
    },
    {
      "epoch": 1.251770671464791,
      "grad_norm": 5.841191291809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0016612631734460592,
      "logits/rejected": -0.2597402036190033,
      "logps/chosen": -409.0643005371094,
      "logps/rejected": -329.2077331542969,
      "loss": 0.291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3612558841705322,
      "rewards/margins": 1.2750978469848633,
      "rewards/rejected": -2.6363537311553955,
      "step": 9566
    },
    {
      "epoch": 1.2519015294021427,
      "grad_norm": 3.685401439666748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16103285551071167,
      "logits/rejected": -0.2274956852197647,
      "logps/chosen": -642.0517578125,
      "logps/rejected": -547.7449951171875,
      "loss": 0.1644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6466445922851562,
      "rewards/margins": 2.2651450634002686,
      "rewards/rejected": -2.9117894172668457,
      "step": 9567
    },
    {
      "epoch": 1.2520323873394945,
      "grad_norm": 7.807150840759277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21069592237472534,
      "logits/rejected": -0.16637369990348816,
      "logps/chosen": -323.8529052734375,
      "logps/rejected": -397.632568359375,
      "loss": 0.3929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7484371662139893,
      "rewards/margins": 1.0532543659210205,
      "rewards/rejected": -2.8016915321350098,
      "step": 9568
    },
    {
      "epoch": 1.2521632452768463,
      "grad_norm": 7.831175804138184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2121397852897644,
      "logits/rejected": -0.3404625356197357,
      "logps/chosen": -484.09100341796875,
      "logps/rejected": -434.90478515625,
      "loss": 0.4518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3713432550430298,
      "rewards/margins": 1.172533631324768,
      "rewards/rejected": -2.543877124786377,
      "step": 9569
    },
    {
      "epoch": 1.252294103214198,
      "grad_norm": 4.984424114227295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3308929204940796,
      "logits/rejected": -0.18458202481269836,
      "logps/chosen": -835.0624389648438,
      "logps/rejected": -721.4241943359375,
      "loss": 0.1874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6208308935165405,
      "rewards/margins": 2.405625343322754,
      "rewards/rejected": -3.026456117630005,
      "step": 9570
    },
    {
      "epoch": 1.2524249611515499,
      "grad_norm": 5.910849571228027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1759214699268341,
      "logits/rejected": -0.4306468367576599,
      "logps/chosen": -620.0245971679688,
      "logps/rejected": -669.7412109375,
      "loss": 0.2697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3665449619293213,
      "rewards/margins": 1.7912368774414062,
      "rewards/rejected": -3.1577818393707275,
      "step": 9571
    },
    {
      "epoch": 1.2525558190889017,
      "grad_norm": 10.626035690307617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5010715126991272,
      "logits/rejected": -0.41441893577575684,
      "logps/chosen": -262.7214660644531,
      "logps/rejected": -296.05596923828125,
      "loss": 0.6116,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5819237232208252,
      "rewards/margins": 0.6470082998275757,
      "rewards/rejected": -2.2289319038391113,
      "step": 9572
    },
    {
      "epoch": 1.2526866770262535,
      "grad_norm": 6.995431423187256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058763764798641205,
      "logits/rejected": 0.025732411071658134,
      "logps/chosen": -506.2790222167969,
      "logps/rejected": -464.93121337890625,
      "loss": 0.3949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7619476318359375,
      "rewards/margins": 1.137537956237793,
      "rewards/rejected": -2.8994855880737305,
      "step": 9573
    },
    {
      "epoch": 1.2528175349636053,
      "grad_norm": 7.549813747406006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26813966035842896,
      "logits/rejected": -0.23253726959228516,
      "logps/chosen": -406.3634338378906,
      "logps/rejected": -360.46588134765625,
      "loss": 0.4074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8919088244438171,
      "rewards/margins": 1.3463884592056274,
      "rewards/rejected": -2.2382972240448,
      "step": 9574
    },
    {
      "epoch": 1.2529483929009568,
      "grad_norm": 4.153542995452881,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0004869361873716116,
      "logits/rejected": 0.01529888715595007,
      "logps/chosen": -498.0433349609375,
      "logps/rejected": -591.978759765625,
      "loss": 0.1739,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8039942979812622,
      "rewards/margins": 1.8259303569793701,
      "rewards/rejected": -2.6299245357513428,
      "step": 9575
    },
    {
      "epoch": 1.2530792508383086,
      "grad_norm": 8.99722671508789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25140485167503357,
      "logits/rejected": -0.490642249584198,
      "logps/chosen": -429.96929931640625,
      "logps/rejected": -344.8540954589844,
      "loss": 0.5707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7435766458511353,
      "rewards/margins": 1.578676462173462,
      "rewards/rejected": -2.3222529888153076,
      "step": 9576
    },
    {
      "epoch": 1.2532101087756604,
      "grad_norm": 8.662678718566895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14114001393318176,
      "logits/rejected": -0.16168645024299622,
      "logps/chosen": -521.9990844726562,
      "logps/rejected": -447.90643310546875,
      "loss": 0.3688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.009762167930603,
      "rewards/margins": 1.4219284057617188,
      "rewards/rejected": -2.4316904544830322,
      "step": 9577
    },
    {
      "epoch": 1.2533409667130122,
      "grad_norm": 7.689244747161865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11882832646369934,
      "logits/rejected": 0.16390010714530945,
      "logps/chosen": -343.9344787597656,
      "logps/rejected": -368.2464294433594,
      "loss": 0.4829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7354134321212769,
      "rewards/margins": 1.0211001634597778,
      "rewards/rejected": -2.7565135955810547,
      "step": 9578
    },
    {
      "epoch": 1.253471824650364,
      "grad_norm": 7.257585525512695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2687767446041107,
      "logits/rejected": -0.3955012559890747,
      "logps/chosen": -382.88604736328125,
      "logps/rejected": -297.3728942871094,
      "loss": 0.4614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3302531242370605,
      "rewards/margins": 0.7207155227661133,
      "rewards/rejected": -2.050968647003174,
      "step": 9579
    },
    {
      "epoch": 1.2536026825877158,
      "grad_norm": 10.779926300048828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19773095846176147,
      "logits/rejected": -0.0941477119922638,
      "logps/chosen": -437.32061767578125,
      "logps/rejected": -339.111572265625,
      "loss": 0.6477,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8233257532119751,
      "rewards/margins": 1.3924851417541504,
      "rewards/rejected": -2.215810775756836,
      "step": 9580
    },
    {
      "epoch": 1.2537335405250674,
      "grad_norm": 13.164804458618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04390067607164383,
      "logits/rejected": -0.41904741525650024,
      "logps/chosen": -509.3535461425781,
      "logps/rejected": -404.5018005371094,
      "loss": 0.7381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7617745399475098,
      "rewards/margins": 1.1911420822143555,
      "rewards/rejected": -2.9529166221618652,
      "step": 9581
    },
    {
      "epoch": 1.2538643984624191,
      "grad_norm": 9.434348106384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4505463242530823,
      "logits/rejected": -0.42624908685684204,
      "logps/chosen": -440.04296875,
      "logps/rejected": -425.3654479980469,
      "loss": 0.5232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.586148977279663,
      "rewards/margins": 0.5061670541763306,
      "rewards/rejected": -2.092315912246704,
      "step": 9582
    },
    {
      "epoch": 1.253995256399771,
      "grad_norm": 5.3227219581604,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3221703767776489,
      "logits/rejected": -0.48582175374031067,
      "logps/chosen": -488.638671875,
      "logps/rejected": -552.7330932617188,
      "loss": 0.2284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8211004734039307,
      "rewards/margins": 1.9197336435317993,
      "rewards/rejected": -2.7408339977264404,
      "step": 9583
    },
    {
      "epoch": 1.2541261143371227,
      "grad_norm": 7.17015266418457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012391619384288788,
      "logits/rejected": -0.060516953468322754,
      "logps/chosen": -338.005126953125,
      "logps/rejected": -336.8398132324219,
      "loss": 0.4367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5746332406997681,
      "rewards/margins": 1.2901493310928345,
      "rewards/rejected": -1.8647825717926025,
      "step": 9584
    },
    {
      "epoch": 1.2542569722744745,
      "grad_norm": 13.14327335357666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023883264511823654,
      "logits/rejected": -0.023201167583465576,
      "logps/chosen": -583.3800659179688,
      "logps/rejected": -664.8788452148438,
      "loss": 0.8978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6981241703033447,
      "rewards/margins": 0.20666563510894775,
      "rewards/rejected": -1.904789686203003,
      "step": 9585
    },
    {
      "epoch": 1.2543878302118263,
      "grad_norm": 6.305208206176758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1894322633743286,
      "logits/rejected": -0.08933568000793457,
      "logps/chosen": -395.41552734375,
      "logps/rejected": -364.33380126953125,
      "loss": 0.3247,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1438466310501099,
      "rewards/margins": 2.066555976867676,
      "rewards/rejected": -3.2104029655456543,
      "step": 9586
    },
    {
      "epoch": 1.254518688149178,
      "grad_norm": 5.386983871459961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17882417142391205,
      "logits/rejected": -0.1980028599500656,
      "logps/chosen": -431.83355712890625,
      "logps/rejected": -338.5559387207031,
      "loss": 0.3087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0465055704116821,
      "rewards/margins": 1.9030189514160156,
      "rewards/rejected": -2.949524402618408,
      "step": 9587
    },
    {
      "epoch": 1.25464954608653,
      "grad_norm": 10.155543327331543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07362758368253708,
      "logits/rejected": -0.18331551551818848,
      "logps/chosen": -547.7333374023438,
      "logps/rejected": -494.0010070800781,
      "loss": 0.5546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1625713109970093,
      "rewards/margins": 0.6091480255126953,
      "rewards/rejected": -1.7717194557189941,
      "step": 9588
    },
    {
      "epoch": 1.2547804040238817,
      "grad_norm": 8.024033546447754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3030581772327423,
      "logits/rejected": -0.294501930475235,
      "logps/chosen": -310.4822692871094,
      "logps/rejected": -395.02105712890625,
      "loss": 0.5272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.718732237815857,
      "rewards/margins": 0.5643981695175171,
      "rewards/rejected": -2.283130645751953,
      "step": 9589
    },
    {
      "epoch": 1.2549112619612333,
      "grad_norm": 10.244112968444824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020062632858753204,
      "logits/rejected": 0.07161864638328552,
      "logps/chosen": -339.1686706542969,
      "logps/rejected": -389.359130859375,
      "loss": 0.4945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2220431566238403,
      "rewards/margins": 0.9877620935440063,
      "rewards/rejected": -2.2098050117492676,
      "step": 9590
    },
    {
      "epoch": 1.255042119898585,
      "grad_norm": 15.800544738769531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4381055235862732,
      "logits/rejected": -0.5887820720672607,
      "logps/chosen": -729.7621459960938,
      "logps/rejected": -453.6243591308594,
      "loss": 0.8016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.2066268920898438,
      "rewards/margins": 0.4906874895095825,
      "rewards/rejected": -2.6973142623901367,
      "step": 9591
    },
    {
      "epoch": 1.2551729778359368,
      "grad_norm": 7.214695930480957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13486430048942566,
      "logits/rejected": 0.1573682278394699,
      "logps/chosen": -402.1310729980469,
      "logps/rejected": -534.059326171875,
      "loss": 0.4189,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0167334079742432,
      "rewards/margins": 0.9000774621963501,
      "rewards/rejected": -1.9168109893798828,
      "step": 9592
    },
    {
      "epoch": 1.2553038357732886,
      "grad_norm": 5.615384578704834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0163886621594429,
      "logits/rejected": -0.0187847837805748,
      "logps/chosen": -608.18701171875,
      "logps/rejected": -511.49163818359375,
      "loss": 0.346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.243573784828186,
      "rewards/margins": 1.648705005645752,
      "rewards/rejected": -2.8922789096832275,
      "step": 9593
    },
    {
      "epoch": 1.2554346937106404,
      "grad_norm": 15.341931343078613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3878001272678375,
      "logits/rejected": -0.3892311453819275,
      "logps/chosen": -367.34716796875,
      "logps/rejected": -300.8056640625,
      "loss": 0.9649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2866437435150146,
      "rewards/margins": 0.3426837921142578,
      "rewards/rejected": -1.6293275356292725,
      "step": 9594
    },
    {
      "epoch": 1.2555655516479922,
      "grad_norm": 11.488489151000977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3602450489997864,
      "logits/rejected": -0.3318425118923187,
      "logps/chosen": -431.3036804199219,
      "logps/rejected": -433.8385314941406,
      "loss": 0.6016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0343210697174072,
      "rewards/margins": 0.622301459312439,
      "rewards/rejected": -1.6566225290298462,
      "step": 9595
    },
    {
      "epoch": 1.2556964095853438,
      "grad_norm": 6.542734622955322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38974449038505554,
      "logits/rejected": -0.24032704532146454,
      "logps/chosen": -355.5810852050781,
      "logps/rejected": -400.7090148925781,
      "loss": 0.4883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8626297116279602,
      "rewards/margins": 0.7444435358047485,
      "rewards/rejected": -1.607073187828064,
      "step": 9596
    },
    {
      "epoch": 1.2558272675226956,
      "grad_norm": 5.052806854248047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1677303910255432,
      "logits/rejected": -0.11584601551294327,
      "logps/chosen": -431.1642761230469,
      "logps/rejected": -417.29840087890625,
      "loss": 0.2643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.092532753944397,
      "rewards/margins": 1.9562463760375977,
      "rewards/rejected": -3.048779010772705,
      "step": 9597
    },
    {
      "epoch": 1.2559581254600474,
      "grad_norm": 6.715884685516357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16746753454208374,
      "logits/rejected": -0.22926178574562073,
      "logps/chosen": -589.2798461914062,
      "logps/rejected": -562.6105346679688,
      "loss": 0.269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39850085973739624,
      "rewards/margins": 1.6896816492080688,
      "rewards/rejected": -2.0881826877593994,
      "step": 9598
    },
    {
      "epoch": 1.2560889833973992,
      "grad_norm": 7.9646687507629395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016475936397910118,
      "logits/rejected": 0.02874498814344406,
      "logps/chosen": -332.97998046875,
      "logps/rejected": -481.07196044921875,
      "loss": 0.403,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5806423425674438,
      "rewards/margins": 1.4530937671661377,
      "rewards/rejected": -2.033735990524292,
      "step": 9599
    },
    {
      "epoch": 1.256219841334751,
      "grad_norm": 2.907557725906372,
      "learning_rate": 5e-06,
      "logits/chosen": 0.279327392578125,
      "logits/rejected": -0.1947232484817505,
      "logps/chosen": -548.1427001953125,
      "logps/rejected": -427.31170654296875,
      "loss": 0.1574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8079284429550171,
      "rewards/margins": 3.3742167949676514,
      "rewards/rejected": -4.182145118713379,
      "step": 9600
    },
    {
      "epoch": 1.2563506992721027,
      "grad_norm": 10.827370643615723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25515511631965637,
      "logits/rejected": -0.2973199486732483,
      "logps/chosen": -308.5157165527344,
      "logps/rejected": -427.63946533203125,
      "loss": 0.4739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.339371681213379,
      "rewards/margins": 1.7317759990692139,
      "rewards/rejected": -3.0711476802825928,
      "step": 9601
    },
    {
      "epoch": 1.2564815572094545,
      "grad_norm": 6.782340049743652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1967991441488266,
      "logits/rejected": -0.10020682215690613,
      "logps/chosen": -454.4163818359375,
      "logps/rejected": -475.8127746582031,
      "loss": 0.3388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.214341163635254,
      "rewards/margins": 1.6595308780670166,
      "rewards/rejected": -2.8738720417022705,
      "step": 9602
    },
    {
      "epoch": 1.2566124151468063,
      "grad_norm": 6.6433234214782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4626713693141937,
      "logits/rejected": -0.4679543972015381,
      "logps/chosen": -524.1251220703125,
      "logps/rejected": -427.8165283203125,
      "loss": 0.5383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3427894115447998,
      "rewards/margins": 1.0523285865783691,
      "rewards/rejected": -2.395117998123169,
      "step": 9603
    },
    {
      "epoch": 1.2567432730841581,
      "grad_norm": 7.532505989074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.263010710477829,
      "logits/rejected": -0.2933509945869446,
      "logps/chosen": -475.8639221191406,
      "logps/rejected": -462.279052734375,
      "loss": 0.4387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.680594801902771,
      "rewards/margins": 1.4206680059432983,
      "rewards/rejected": -2.1012625694274902,
      "step": 9604
    },
    {
      "epoch": 1.25687413102151,
      "grad_norm": 8.542314529418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22508825361728668,
      "logits/rejected": 0.21949996054172516,
      "logps/chosen": -572.4927978515625,
      "logps/rejected": -634.1726684570312,
      "loss": 0.3792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7978477478027344,
      "rewards/margins": 2.222607374191284,
      "rewards/rejected": -3.0204551219940186,
      "step": 9605
    },
    {
      "epoch": 1.2570049889588615,
      "grad_norm": 9.935980796813965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11692140996456146,
      "logits/rejected": -0.1984311044216156,
      "logps/chosen": -504.04461669921875,
      "logps/rejected": -392.0550231933594,
      "loss": 0.481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8100042343139648,
      "rewards/margins": 0.9038709402084351,
      "rewards/rejected": -1.7138750553131104,
      "step": 9606
    },
    {
      "epoch": 1.2571358468962133,
      "grad_norm": 8.41815185546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27791425585746765,
      "logits/rejected": -0.3758194148540497,
      "logps/chosen": -477.2088928222656,
      "logps/rejected": -483.75653076171875,
      "loss": 0.5299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5907493829727173,
      "rewards/margins": 1.6144871711730957,
      "rewards/rejected": -2.2052364349365234,
      "step": 9607
    },
    {
      "epoch": 1.257266704833565,
      "grad_norm": 13.44428825378418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16520246863365173,
      "logits/rejected": 0.09315156191587448,
      "logps/chosen": -500.3472900390625,
      "logps/rejected": -384.5887756347656,
      "loss": 0.6621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0621020793914795,
      "rewards/margins": 0.905699610710144,
      "rewards/rejected": -2.967801809310913,
      "step": 9608
    },
    {
      "epoch": 1.2573975627709169,
      "grad_norm": 3.1981375217437744,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05345941707491875,
      "logits/rejected": -0.19668225944042206,
      "logps/chosen": -336.5529479980469,
      "logps/rejected": -416.55316162109375,
      "loss": 0.1388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8983200788497925,
      "rewards/margins": 2.1193642616271973,
      "rewards/rejected": -4.017684459686279,
      "step": 9609
    },
    {
      "epoch": 1.2575284207082686,
      "grad_norm": 8.664133071899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25607115030288696,
      "logits/rejected": -0.19476184248924255,
      "logps/chosen": -293.6782531738281,
      "logps/rejected": -334.4567565917969,
      "loss": 0.5447,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.881587028503418,
      "rewards/margins": 0.5800603628158569,
      "rewards/rejected": -2.4616475105285645,
      "step": 9610
    },
    {
      "epoch": 1.2576592786456202,
      "grad_norm": 4.741615295410156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12570559978485107,
      "logits/rejected": -0.12994393706321716,
      "logps/chosen": -623.5321655273438,
      "logps/rejected": -596.130126953125,
      "loss": 0.1953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0714786052703857,
      "rewards/margins": 2.796942710876465,
      "rewards/rejected": -3.8684213161468506,
      "step": 9611
    },
    {
      "epoch": 1.257790136582972,
      "grad_norm": 9.301323890686035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14445728063583374,
      "logits/rejected": -0.23800010979175568,
      "logps/chosen": -317.8527526855469,
      "logps/rejected": -406.3023376464844,
      "loss": 0.5603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7966346740722656,
      "rewards/margins": 2.096503973007202,
      "rewards/rejected": -2.8931386470794678,
      "step": 9612
    },
    {
      "epoch": 1.2579209945203238,
      "grad_norm": 6.6462907791137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3233278691768646,
      "logits/rejected": -0.5554545521736145,
      "logps/chosen": -664.52783203125,
      "logps/rejected": -538.276123046875,
      "loss": 0.3413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6399700045585632,
      "rewards/margins": 1.8253910541534424,
      "rewards/rejected": -2.4653611183166504,
      "step": 9613
    },
    {
      "epoch": 1.2580518524576756,
      "grad_norm": 8.122427940368652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12043541669845581,
      "logits/rejected": -0.2885206937789917,
      "logps/chosen": -421.4653015136719,
      "logps/rejected": -390.837158203125,
      "loss": 0.4915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0914274454116821,
      "rewards/margins": 1.1504247188568115,
      "rewards/rejected": -2.241852283477783,
      "step": 9614
    },
    {
      "epoch": 1.2581827103950274,
      "grad_norm": 9.873196601867676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08606886118650436,
      "logits/rejected": -0.08491989970207214,
      "logps/chosen": -715.7383422851562,
      "logps/rejected": -518.5581665039062,
      "loss": 0.4855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3294336795806885,
      "rewards/margins": 1.6269301176071167,
      "rewards/rejected": -2.9563636779785156,
      "step": 9615
    },
    {
      "epoch": 1.2583135683323792,
      "grad_norm": 9.385892868041992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04488244652748108,
      "logits/rejected": -0.03142879158258438,
      "logps/chosen": -571.4794311523438,
      "logps/rejected": -492.356689453125,
      "loss": 0.6346,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0642807483673096,
      "rewards/margins": 1.405902624130249,
      "rewards/rejected": -2.4701831340789795,
      "step": 9616
    },
    {
      "epoch": 1.258444426269731,
      "grad_norm": 10.814837455749512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025759786367416382,
      "logits/rejected": -0.21270674467086792,
      "logps/chosen": -536.5062255859375,
      "logps/rejected": -520.85302734375,
      "loss": 0.587,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7211753129959106,
      "rewards/margins": 1.1054716110229492,
      "rewards/rejected": -2.826646566390991,
      "step": 9617
    },
    {
      "epoch": 1.2585752842070828,
      "grad_norm": 12.75661849975586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33899104595184326,
      "logits/rejected": 0.3924221992492676,
      "logps/chosen": -465.54736328125,
      "logps/rejected": -605.5388793945312,
      "loss": 0.9237,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7288258075714111,
      "rewards/margins": -0.025122418999671936,
      "rewards/rejected": -1.7037031650543213,
      "step": 9618
    },
    {
      "epoch": 1.2587061421444345,
      "grad_norm": 8.591553688049316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07669418305158615,
      "logits/rejected": -0.18089309334754944,
      "logps/chosen": -540.0592651367188,
      "logps/rejected": -438.00830078125,
      "loss": 0.4437,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3853321075439453,
      "rewards/margins": 1.3821942806243896,
      "rewards/rejected": -1.767526388168335,
      "step": 9619
    },
    {
      "epoch": 1.2588370000817863,
      "grad_norm": 7.088245868682861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15868093073368073,
      "logits/rejected": -0.3451272249221802,
      "logps/chosen": -472.7569274902344,
      "logps/rejected": -535.0117797851562,
      "loss": 0.4974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06790181249380112,
      "rewards/margins": 1.463971734046936,
      "rewards/rejected": -1.396069884300232,
      "step": 9620
    },
    {
      "epoch": 1.258967858019138,
      "grad_norm": 6.081439971923828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053160253912210464,
      "logits/rejected": -0.16970713436603546,
      "logps/chosen": -365.0656433105469,
      "logps/rejected": -494.77789306640625,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23291434347629547,
      "rewards/margins": 2.180023431777954,
      "rewards/rejected": -1.9471092224121094,
      "step": 9621
    },
    {
      "epoch": 1.2590987159564897,
      "grad_norm": 5.168177127838135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2393365502357483,
      "logits/rejected": -0.45368099212646484,
      "logps/chosen": -722.039306640625,
      "logps/rejected": -517.5781860351562,
      "loss": 0.2064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5634665489196777,
      "rewards/margins": 1.7243748903274536,
      "rewards/rejected": -3.287841320037842,
      "step": 9622
    },
    {
      "epoch": 1.2592295738938415,
      "grad_norm": 5.505588531494141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20352934300899506,
      "logits/rejected": -0.2917158603668213,
      "logps/chosen": -621.8782348632812,
      "logps/rejected": -453.27850341796875,
      "loss": 0.2857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4080846905708313,
      "rewards/margins": 1.761016607284546,
      "rewards/rejected": -2.1691014766693115,
      "step": 9623
    },
    {
      "epoch": 1.2593604318311933,
      "grad_norm": 8.622259140014648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08800484240055084,
      "logits/rejected": -0.09140163660049438,
      "logps/chosen": -465.8603210449219,
      "logps/rejected": -460.4305725097656,
      "loss": 0.5769,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.431721568107605,
      "rewards/margins": 0.5656865239143372,
      "rewards/rejected": -1.9974077939987183,
      "step": 9624
    },
    {
      "epoch": 1.259491289768545,
      "grad_norm": 10.087347984313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36701491475105286,
      "logits/rejected": -0.14428746700286865,
      "logps/chosen": -586.2797241210938,
      "logps/rejected": -468.0259094238281,
      "loss": 0.615,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4219311475753784,
      "rewards/margins": 0.9830303192138672,
      "rewards/rejected": -2.404961347579956,
      "step": 9625
    },
    {
      "epoch": 1.2596221477058969,
      "grad_norm": 8.750520706176758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2959572672843933,
      "logits/rejected": -0.2890763580799103,
      "logps/chosen": -445.2250061035156,
      "logps/rejected": -509.1504211425781,
      "loss": 0.6214,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0933135747909546,
      "rewards/margins": 0.45143091678619385,
      "rewards/rejected": -1.5447444915771484,
      "step": 9626
    },
    {
      "epoch": 1.2597530056432484,
      "grad_norm": 6.870359897613525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2749707102775574,
      "logits/rejected": -0.3873017728328705,
      "logps/chosen": -478.49334716796875,
      "logps/rejected": -362.6898498535156,
      "loss": 0.3591,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4720289707183838,
      "rewards/margins": 1.3431336879730225,
      "rewards/rejected": -1.8151627779006958,
      "step": 9627
    },
    {
      "epoch": 1.2598838635806002,
      "grad_norm": 29.2929630279541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10312387347221375,
      "logits/rejected": -0.3190508484840393,
      "logps/chosen": -585.7454833984375,
      "logps/rejected": -738.9872436523438,
      "loss": 0.4145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2891643047332764,
      "rewards/margins": 1.5740253925323486,
      "rewards/rejected": -2.863189697265625,
      "step": 9628
    },
    {
      "epoch": 1.260014721517952,
      "grad_norm": 5.209278583526611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12586481869220734,
      "logits/rejected": -0.27329638600349426,
      "logps/chosen": -515.138916015625,
      "logps/rejected": -514.4691162109375,
      "loss": 0.2222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2697095274925232,
      "rewards/margins": 1.93548583984375,
      "rewards/rejected": -2.205195426940918,
      "step": 9629
    },
    {
      "epoch": 1.2601455794553038,
      "grad_norm": 11.47563362121582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18324565887451172,
      "logits/rejected": 0.08817344903945923,
      "logps/chosen": -506.904052734375,
      "logps/rejected": -508.5157470703125,
      "loss": 0.8856,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5192689895629883,
      "rewards/margins": 0.4782741069793701,
      "rewards/rejected": -1.9975430965423584,
      "step": 9630
    },
    {
      "epoch": 1.2602764373926556,
      "grad_norm": 5.831366539001465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28372347354888916,
      "logits/rejected": -0.38162145018577576,
      "logps/chosen": -404.6556396484375,
      "logps/rejected": -491.9927978515625,
      "loss": 0.2619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0704329013824463,
      "rewards/margins": 1.4674158096313477,
      "rewards/rejected": -2.537848711013794,
      "step": 9631
    },
    {
      "epoch": 1.2604072953300074,
      "grad_norm": 8.476357460021973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.434150755405426,
      "logits/rejected": -0.35634738206863403,
      "logps/chosen": -340.57257080078125,
      "logps/rejected": -413.4968566894531,
      "loss": 0.5758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.825134038925171,
      "rewards/margins": 0.7810379862785339,
      "rewards/rejected": -2.6061720848083496,
      "step": 9632
    },
    {
      "epoch": 1.2605381532673592,
      "grad_norm": 8.479753494262695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057662226259708405,
      "logits/rejected": -0.047738756984472275,
      "logps/chosen": -496.0592041015625,
      "logps/rejected": -639.42822265625,
      "loss": 0.4577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9028961062431335,
      "rewards/margins": 1.7355693578720093,
      "rewards/rejected": -2.638465404510498,
      "step": 9633
    },
    {
      "epoch": 1.260669011204711,
      "grad_norm": 8.254096984863281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12927395105361938,
      "logits/rejected": -0.04598788544535637,
      "logps/chosen": -489.5562744140625,
      "logps/rejected": -362.0162353515625,
      "loss": 0.2466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5994695425033569,
      "rewards/margins": 2.331922769546509,
      "rewards/rejected": -2.9313924312591553,
      "step": 9634
    },
    {
      "epoch": 1.2607998691420628,
      "grad_norm": 4.952192306518555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2226119339466095,
      "logits/rejected": -0.4632307291030884,
      "logps/chosen": -366.0694274902344,
      "logps/rejected": -376.6965026855469,
      "loss": 0.3116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3449904918670654,
      "rewards/margins": 1.4755783081054688,
      "rewards/rejected": -2.8205690383911133,
      "step": 9635
    },
    {
      "epoch": 1.2609307270794143,
      "grad_norm": 8.105803489685059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3268546164035797,
      "logits/rejected": -0.36584237217903137,
      "logps/chosen": -346.2098083496094,
      "logps/rejected": -373.17266845703125,
      "loss": 0.3459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2867481708526611,
      "rewards/margins": 1.415909767150879,
      "rewards/rejected": -2.70265793800354,
      "step": 9636
    },
    {
      "epoch": 1.2610615850167661,
      "grad_norm": 6.40825080871582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2694007456302643,
      "logits/rejected": -0.5194730758666992,
      "logps/chosen": -585.9508056640625,
      "logps/rejected": -547.9960327148438,
      "loss": 0.2426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33565789461135864,
      "rewards/margins": 2.55111026763916,
      "rewards/rejected": -2.886768102645874,
      "step": 9637
    },
    {
      "epoch": 1.261192442954118,
      "grad_norm": 6.853188991546631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03503686189651489,
      "logits/rejected": -0.23411327600479126,
      "logps/chosen": -612.166748046875,
      "logps/rejected": -560.7293701171875,
      "loss": 0.3037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.25465127825737,
      "rewards/margins": 2.215715169906616,
      "rewards/rejected": -1.961064100265503,
      "step": 9638
    },
    {
      "epoch": 1.2613233008914697,
      "grad_norm": 13.671992301940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2935662567615509,
      "logits/rejected": -0.30429065227508545,
      "logps/chosen": -558.1451416015625,
      "logps/rejected": -536.0711059570312,
      "loss": 0.9301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9890641570091248,
      "rewards/margins": -0.16783463954925537,
      "rewards/rejected": -0.8212295770645142,
      "step": 9639
    },
    {
      "epoch": 1.2614541588288215,
      "grad_norm": 7.364981651306152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12195300310850143,
      "logits/rejected": -0.1038612648844719,
      "logps/chosen": -488.41424560546875,
      "logps/rejected": -398.85150146484375,
      "loss": 0.4306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22220343351364136,
      "rewards/margins": 1.2576782703399658,
      "rewards/rejected": -1.0354747772216797,
      "step": 9640
    },
    {
      "epoch": 1.2615850167661733,
      "grad_norm": 4.3168792724609375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15359118580818176,
      "logits/rejected": -0.24075673520565033,
      "logps/chosen": -598.52001953125,
      "logps/rejected": -433.51239013671875,
      "loss": 0.2668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26240214705467224,
      "rewards/margins": 1.547870397567749,
      "rewards/rejected": -1.8102725744247437,
      "step": 9641
    },
    {
      "epoch": 1.2617158747035249,
      "grad_norm": 13.422432899475098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0011134296655654907,
      "logits/rejected": -0.2265302836894989,
      "logps/chosen": -420.33282470703125,
      "logps/rejected": -459.547119140625,
      "loss": 0.7134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9859123229980469,
      "rewards/margins": 1.6726789474487305,
      "rewards/rejected": -2.6585912704467773,
      "step": 9642
    },
    {
      "epoch": 1.2618467326408767,
      "grad_norm": 9.948814392089844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2312355935573578,
      "logits/rejected": -0.1670280247926712,
      "logps/chosen": -384.4691467285156,
      "logps/rejected": -411.6430358886719,
      "loss": 0.4331,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4847272634506226,
      "rewards/margins": 1.7784686088562012,
      "rewards/rejected": -3.263195753097534,
      "step": 9643
    },
    {
      "epoch": 1.2619775905782284,
      "grad_norm": 6.10704231262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3711742162704468,
      "logits/rejected": -0.37890902161598206,
      "logps/chosen": -518.0430297851562,
      "logps/rejected": -532.3992919921875,
      "loss": 0.2765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5709273815155029,
      "rewards/margins": 2.221391201019287,
      "rewards/rejected": -2.792318820953369,
      "step": 9644
    },
    {
      "epoch": 1.2621084485155802,
      "grad_norm": 7.5767927169799805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.223357692360878,
      "logits/rejected": -0.35618171095848083,
      "logps/chosen": -489.8880615234375,
      "logps/rejected": -460.4008483886719,
      "loss": 0.5524,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.108994483947754,
      "rewards/margins": 1.3153759241104126,
      "rewards/rejected": -2.424370288848877,
      "step": 9645
    },
    {
      "epoch": 1.262239306452932,
      "grad_norm": 8.107976913452148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04793466627597809,
      "logits/rejected": 0.051231250166893005,
      "logps/chosen": -628.4623413085938,
      "logps/rejected": -584.6646118164062,
      "loss": 0.3933,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38431280851364136,
      "rewards/margins": 1.1169755458831787,
      "rewards/rejected": -1.5012884140014648,
      "step": 9646
    },
    {
      "epoch": 1.2623701643902838,
      "grad_norm": 7.192448616027832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22676536440849304,
      "logits/rejected": -0.14068220555782318,
      "logps/chosen": -413.4823303222656,
      "logps/rejected": -400.820068359375,
      "loss": 0.4823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9669654965400696,
      "rewards/margins": 1.332251787185669,
      "rewards/rejected": -2.2992169857025146,
      "step": 9647
    },
    {
      "epoch": 1.2625010223276356,
      "grad_norm": 11.672965049743652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.041636284440755844,
      "logits/rejected": 0.08849082887172699,
      "logps/chosen": -404.92254638671875,
      "logps/rejected": -520.0873413085938,
      "loss": 0.8724,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.40408194065094,
      "rewards/margins": 1.2510285377502441,
      "rewards/rejected": -2.6551101207733154,
      "step": 9648
    },
    {
      "epoch": 1.2626318802649874,
      "grad_norm": 8.506784439086914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06507855653762817,
      "logits/rejected": 0.09557220339775085,
      "logps/chosen": -547.97607421875,
      "logps/rejected": -578.1312866210938,
      "loss": 0.349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17260071635246277,
      "rewards/margins": 2.0087080001831055,
      "rewards/rejected": -2.1813089847564697,
      "step": 9649
    },
    {
      "epoch": 1.2627627382023392,
      "grad_norm": 6.435142517089844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22914570569992065,
      "logits/rejected": -0.18745514750480652,
      "logps/chosen": -558.6461181640625,
      "logps/rejected": -476.5542297363281,
      "loss": 0.2853,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5685287714004517,
      "rewards/margins": 1.9528855085372925,
      "rewards/rejected": -2.521414279937744,
      "step": 9650
    },
    {
      "epoch": 1.2628935961396908,
      "grad_norm": 5.27431583404541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.331185519695282,
      "logits/rejected": -0.3777307868003845,
      "logps/chosen": -325.04547119140625,
      "logps/rejected": -466.5467224121094,
      "loss": 0.2242,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.548405647277832,
      "rewards/margins": 2.1080636978149414,
      "rewards/rejected": -2.6564693450927734,
      "step": 9651
    },
    {
      "epoch": 1.2630244540770426,
      "grad_norm": 6.705560207366943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23487935960292816,
      "logits/rejected": -0.2700748145580292,
      "logps/chosen": -538.92626953125,
      "logps/rejected": -625.9087524414062,
      "loss": 0.2393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6147592067718506,
      "rewards/margins": 1.878006935119629,
      "rewards/rejected": -3.4927661418914795,
      "step": 9652
    },
    {
      "epoch": 1.2631553120143943,
      "grad_norm": 10.051373481750488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16367700695991516,
      "logits/rejected": -0.3064817488193512,
      "logps/chosen": -485.5740661621094,
      "logps/rejected": -272.03704833984375,
      "loss": 0.6613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3496330976486206,
      "rewards/margins": 0.6066408157348633,
      "rewards/rejected": -1.9562737941741943,
      "step": 9653
    },
    {
      "epoch": 1.2632861699517461,
      "grad_norm": 6.800705909729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2741392254829407,
      "logits/rejected": -0.3301851153373718,
      "logps/chosen": -369.3952331542969,
      "logps/rejected": -469.8240966796875,
      "loss": 0.5659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9888164401054382,
      "rewards/margins": 2.2342095375061035,
      "rewards/rejected": -3.2230262756347656,
      "step": 9654
    },
    {
      "epoch": 1.263417027889098,
      "grad_norm": 8.236018180847168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34663328528404236,
      "logits/rejected": -0.30176064372062683,
      "logps/chosen": -453.166015625,
      "logps/rejected": -467.69683837890625,
      "loss": 0.5479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6058323383331299,
      "rewards/margins": 0.661590039730072,
      "rewards/rejected": -2.2674221992492676,
      "step": 9655
    },
    {
      "epoch": 1.2635478858264497,
      "grad_norm": 8.830384254455566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17749449610710144,
      "logits/rejected": -0.21771955490112305,
      "logps/chosen": -455.95501708984375,
      "logps/rejected": -487.42779541015625,
      "loss": 0.3629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1418404579162598,
      "rewards/margins": 1.8423078060150146,
      "rewards/rejected": -2.9841482639312744,
      "step": 9656
    },
    {
      "epoch": 1.2636787437638013,
      "grad_norm": 7.868279457092285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09849794209003448,
      "logits/rejected": -0.271011084318161,
      "logps/chosen": -485.2318115234375,
      "logps/rejected": -510.35601806640625,
      "loss": 0.2921,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6696704626083374,
      "rewards/margins": 2.15078067779541,
      "rewards/rejected": -2.820451259613037,
      "step": 9657
    },
    {
      "epoch": 1.263809601701153,
      "grad_norm": 8.719236373901367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3447016477584839,
      "logits/rejected": -0.28886985778808594,
      "logps/chosen": -448.9832763671875,
      "logps/rejected": -442.31976318359375,
      "loss": 0.3984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9556041359901428,
      "rewards/margins": 1.6301395893096924,
      "rewards/rejected": -2.5857439041137695,
      "step": 9658
    },
    {
      "epoch": 1.2639404596385049,
      "grad_norm": 9.010919570922852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005330421030521393,
      "logits/rejected": -0.11704619228839874,
      "logps/chosen": -441.8838195800781,
      "logps/rejected": -426.8343200683594,
      "loss": 0.3988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42695775628089905,
      "rewards/margins": 1.8202050924301147,
      "rewards/rejected": -2.2471628189086914,
      "step": 9659
    },
    {
      "epoch": 1.2640713175758567,
      "grad_norm": 8.372035026550293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08213010430335999,
      "logits/rejected": -0.26130127906799316,
      "logps/chosen": -395.4612731933594,
      "logps/rejected": -355.00341796875,
      "loss": 0.3779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5496322512626648,
      "rewards/margins": 1.7456036806106567,
      "rewards/rejected": -2.2952358722686768,
      "step": 9660
    },
    {
      "epoch": 1.2642021755132085,
      "grad_norm": 11.903099060058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1368170529603958,
      "logits/rejected": -0.18906013667583466,
      "logps/chosen": -477.5413513183594,
      "logps/rejected": -353.5971374511719,
      "loss": 0.8623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1278185844421387,
      "rewards/margins": 0.7714889049530029,
      "rewards/rejected": -1.8993074893951416,
      "step": 9661
    },
    {
      "epoch": 1.2643330334505603,
      "grad_norm": 15.441926002502441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08351123332977295,
      "logits/rejected": -0.1006280705332756,
      "logps/chosen": -371.5220947265625,
      "logps/rejected": -428.6629333496094,
      "loss": 0.6422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9153037071228027,
      "rewards/margins": 0.8554856777191162,
      "rewards/rejected": -2.770789623260498,
      "step": 9662
    },
    {
      "epoch": 1.264463891387912,
      "grad_norm": 5.090081691741943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.298103004693985,
      "logits/rejected": -0.6396811604499817,
      "logps/chosen": -350.68212890625,
      "logps/rejected": -675.09228515625,
      "loss": 0.3307,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.564753532409668,
      "rewards/margins": 2.0096824169158936,
      "rewards/rejected": -3.5744361877441406,
      "step": 9663
    },
    {
      "epoch": 1.2645947493252638,
      "grad_norm": 6.560748100280762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31520622968673706,
      "logits/rejected": -0.13274848461151123,
      "logps/chosen": -384.5540771484375,
      "logps/rejected": -464.9754638671875,
      "loss": 0.3349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3926187753677368,
      "rewards/margins": 1.7577048540115356,
      "rewards/rejected": -3.1503236293792725,
      "step": 9664
    },
    {
      "epoch": 1.2647256072626156,
      "grad_norm": 8.011338233947754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15995199978351593,
      "logits/rejected": -0.16701018810272217,
      "logps/chosen": -260.3122253417969,
      "logps/rejected": -417.9238586425781,
      "loss": 0.5114,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7494686841964722,
      "rewards/margins": 1.320736050605774,
      "rewards/rejected": -2.070204734802246,
      "step": 9665
    },
    {
      "epoch": 1.2648564651999674,
      "grad_norm": 5.609310150146484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013275470584630966,
      "logits/rejected": -0.2065797597169876,
      "logps/chosen": -485.83697509765625,
      "logps/rejected": -639.3502197265625,
      "loss": 0.2192,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37853968143463135,
      "rewards/margins": 2.7956738471984863,
      "rewards/rejected": -3.1742136478424072,
      "step": 9666
    },
    {
      "epoch": 1.264987323137319,
      "grad_norm": 7.487654685974121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02973092347383499,
      "logits/rejected": -0.2063407152891159,
      "logps/chosen": -365.82757568359375,
      "logps/rejected": -369.93218994140625,
      "loss": 0.5223,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1858627796173096,
      "rewards/margins": 1.5661990642547607,
      "rewards/rejected": -2.7520618438720703,
      "step": 9667
    },
    {
      "epoch": 1.2651181810746708,
      "grad_norm": 9.252498626708984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09126753360033035,
      "logits/rejected": 0.10077834129333496,
      "logps/chosen": -413.20782470703125,
      "logps/rejected": -414.8562927246094,
      "loss": 0.6206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8903934955596924,
      "rewards/margins": 0.54336017370224,
      "rewards/rejected": -1.4337536096572876,
      "step": 9668
    },
    {
      "epoch": 1.2652490390120226,
      "grad_norm": 5.395244121551514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6272034645080566,
      "logits/rejected": -0.6253694295883179,
      "logps/chosen": -328.43634033203125,
      "logps/rejected": -374.66522216796875,
      "loss": 0.3091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29836273193359375,
      "rewards/margins": 1.510226845741272,
      "rewards/rejected": -1.8085895776748657,
      "step": 9669
    },
    {
      "epoch": 1.2653798969493744,
      "grad_norm": 9.698787689208984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31383955478668213,
      "logits/rejected": -0.3634667694568634,
      "logps/chosen": -449.1786804199219,
      "logps/rejected": -426.4979553222656,
      "loss": 0.5088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.310758113861084,
      "rewards/margins": 0.6043720841407776,
      "rewards/rejected": -1.9151301383972168,
      "step": 9670
    },
    {
      "epoch": 1.2655107548867262,
      "grad_norm": 13.80517578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40857768058776855,
      "logits/rejected": -0.4334511160850525,
      "logps/chosen": -518.4293212890625,
      "logps/rejected": -402.06964111328125,
      "loss": 0.4885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7331126928329468,
      "rewards/margins": 1.092005968093872,
      "rewards/rejected": -1.8251186609268188,
      "step": 9671
    },
    {
      "epoch": 1.2656416128240777,
      "grad_norm": 12.048583984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05048145353794098,
      "logits/rejected": 0.01985558494925499,
      "logps/chosen": -455.0863037109375,
      "logps/rejected": -409.91192626953125,
      "loss": 0.7693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6984142065048218,
      "rewards/margins": 1.1477771997451782,
      "rewards/rejected": -1.8461915254592896,
      "step": 9672
    },
    {
      "epoch": 1.2657724707614295,
      "grad_norm": 6.686677932739258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07611805200576782,
      "logits/rejected": -0.2550727427005768,
      "logps/chosen": -498.5525207519531,
      "logps/rejected": -500.3686218261719,
      "loss": 0.3958,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7036058306694031,
      "rewards/margins": 1.3807419538497925,
      "rewards/rejected": -2.08434796333313,
      "step": 9673
    },
    {
      "epoch": 1.2659033286987813,
      "grad_norm": 7.861941814422607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11347389966249466,
      "logits/rejected": -0.20323264598846436,
      "logps/chosen": -368.1553649902344,
      "logps/rejected": -435.3763427734375,
      "loss": 0.3588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0749785900115967,
      "rewards/margins": 1.5470207929611206,
      "rewards/rejected": -2.621999502182007,
      "step": 9674
    },
    {
      "epoch": 1.266034186636133,
      "grad_norm": 5.983136177062988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19582796096801758,
      "logits/rejected": -0.12557466328144073,
      "logps/chosen": -429.9153747558594,
      "logps/rejected": -443.6374816894531,
      "loss": 0.2939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4005701541900635,
      "rewards/margins": 1.3281803131103516,
      "rewards/rejected": -2.728750228881836,
      "step": 9675
    },
    {
      "epoch": 1.2661650445734849,
      "grad_norm": 6.322027683258057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3352647125720978,
      "logits/rejected": -0.5182626247406006,
      "logps/chosen": -561.3060302734375,
      "logps/rejected": -422.04595947265625,
      "loss": 0.3021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2150338888168335,
      "rewards/margins": 1.67555832862854,
      "rewards/rejected": -2.890592336654663,
      "step": 9676
    },
    {
      "epoch": 1.2662959025108367,
      "grad_norm": 6.02452278137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14572539925575256,
      "logits/rejected": -0.14132773876190186,
      "logps/chosen": -400.3873596191406,
      "logps/rejected": -467.91265869140625,
      "loss": 0.2963,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0474004745483398,
      "rewards/margins": 1.6234409809112549,
      "rewards/rejected": -2.6708414554595947,
      "step": 9677
    },
    {
      "epoch": 1.2664267604481885,
      "grad_norm": 13.803290367126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30290350317955017,
      "logits/rejected": 0.06452716141939163,
      "logps/chosen": -422.2991943359375,
      "logps/rejected": -519.4420776367188,
      "loss": 1.1381,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2798845767974854,
      "rewards/margins": 0.1069309413433075,
      "rewards/rejected": -1.3868155479431152,
      "step": 9678
    },
    {
      "epoch": 1.2665576183855403,
      "grad_norm": 9.15798568725586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11698971688747406,
      "logits/rejected": -0.1376062035560608,
      "logps/chosen": -605.0482788085938,
      "logps/rejected": -498.97802734375,
      "loss": 0.5281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3961239457130432,
      "rewards/margins": 1.009505271911621,
      "rewards/rejected": -1.4056291580200195,
      "step": 9679
    },
    {
      "epoch": 1.266688476322892,
      "grad_norm": 5.4147820472717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05189263075590134,
      "logits/rejected": -0.25472739338874817,
      "logps/chosen": -495.36590576171875,
      "logps/rejected": -599.4457397460938,
      "loss": 0.2228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.585617184638977,
      "rewards/margins": 2.8637664318084717,
      "rewards/rejected": -3.4493837356567383,
      "step": 9680
    },
    {
      "epoch": 1.2668193342602438,
      "grad_norm": 7.2908034324646,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32118022441864014,
      "logits/rejected": -0.3106120824813843,
      "logps/chosen": -584.09423828125,
      "logps/rejected": -633.4991455078125,
      "loss": 0.4069,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.923477292060852,
      "rewards/margins": 1.4647661447525024,
      "rewards/rejected": -2.3882436752319336,
      "step": 9681
    },
    {
      "epoch": 1.2669501921975954,
      "grad_norm": 5.5256876945495605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0937994047999382,
      "logits/rejected": -0.20014704763889313,
      "logps/chosen": -620.8818359375,
      "logps/rejected": -466.447509765625,
      "loss": 0.1948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12985683977603912,
      "rewards/margins": 2.017784357070923,
      "rewards/rejected": -2.147641181945801,
      "step": 9682
    },
    {
      "epoch": 1.2670810501349472,
      "grad_norm": 7.309751987457275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3755291700363159,
      "logits/rejected": -0.5227488279342651,
      "logps/chosen": -437.8016357421875,
      "logps/rejected": -376.7982177734375,
      "loss": 0.4388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.164681077003479,
      "rewards/margins": 0.9582768678665161,
      "rewards/rejected": -2.122958183288574,
      "step": 9683
    },
    {
      "epoch": 1.267211908072299,
      "grad_norm": 6.548277378082275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2513681948184967,
      "logits/rejected": -0.2715620696544647,
      "logps/chosen": -555.4324340820312,
      "logps/rejected": -581.6159057617188,
      "loss": 0.3362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2161418199539185,
      "rewards/margins": 1.1964048147201538,
      "rewards/rejected": -2.4125466346740723,
      "step": 9684
    },
    {
      "epoch": 1.2673427660096508,
      "grad_norm": 7.350760459899902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02020246535539627,
      "logits/rejected": -0.004753321409225464,
      "logps/chosen": -376.5088195800781,
      "logps/rejected": -546.9349975585938,
      "loss": 0.3468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8695635795593262,
      "rewards/margins": 1.481522560119629,
      "rewards/rejected": -3.351086139678955,
      "step": 9685
    },
    {
      "epoch": 1.2674736239470026,
      "grad_norm": 10.535261154174805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2660648226737976,
      "logits/rejected": -0.42513561248779297,
      "logps/chosen": -466.4696350097656,
      "logps/rejected": -358.58013916015625,
      "loss": 0.7421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6882717609405518,
      "rewards/margins": 1.021871566772461,
      "rewards/rejected": -2.710143566131592,
      "step": 9686
    },
    {
      "epoch": 1.2676044818843544,
      "grad_norm": 11.821714401245117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11246635764837265,
      "logits/rejected": -0.08954886347055435,
      "logps/chosen": -516.6468505859375,
      "logps/rejected": -564.1781005859375,
      "loss": 0.8323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7733896970748901,
      "rewards/margins": 0.3953533172607422,
      "rewards/rejected": -1.1687428951263428,
      "step": 9687
    },
    {
      "epoch": 1.267735339821706,
      "grad_norm": 11.425165176391602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1893332600593567,
      "logits/rejected": -0.16039076447486877,
      "logps/chosen": -736.3187255859375,
      "logps/rejected": -600.1668701171875,
      "loss": 0.7119,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.561230480670929,
      "rewards/margins": 0.9813762903213501,
      "rewards/rejected": -1.5426067113876343,
      "step": 9688
    },
    {
      "epoch": 1.2678661977590577,
      "grad_norm": 9.05955982208252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20766383409500122,
      "logits/rejected": -0.3196496367454529,
      "logps/chosen": -345.28704833984375,
      "logps/rejected": -384.48992919921875,
      "loss": 0.2918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7771515846252441,
      "rewards/margins": 1.5114705562591553,
      "rewards/rejected": -3.2886219024658203,
      "step": 9689
    },
    {
      "epoch": 1.2679970556964095,
      "grad_norm": 6.999074459075928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20538142323493958,
      "logits/rejected": 0.1824476420879364,
      "logps/chosen": -360.9992980957031,
      "logps/rejected": -400.1152648925781,
      "loss": 0.3739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8848793506622314,
      "rewards/margins": 1.4999535083770752,
      "rewards/rejected": -2.3848326206207275,
      "step": 9690
    },
    {
      "epoch": 1.2681279136337613,
      "grad_norm": 7.9766130447387695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.551084041595459,
      "logits/rejected": -0.20689266920089722,
      "logps/chosen": -799.0762329101562,
      "logps/rejected": -619.2362670898438,
      "loss": 0.4824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8693853616714478,
      "rewards/margins": 1.034925103187561,
      "rewards/rejected": -1.9043104648590088,
      "step": 9691
    },
    {
      "epoch": 1.268258771571113,
      "grad_norm": 7.650460720062256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2452130913734436,
      "logits/rejected": -0.1458202749490738,
      "logps/chosen": -418.1817321777344,
      "logps/rejected": -356.5101318359375,
      "loss": 0.3231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7145862579345703,
      "rewards/margins": 1.3834447860717773,
      "rewards/rejected": -2.0980312824249268,
      "step": 9692
    },
    {
      "epoch": 1.268389629508465,
      "grad_norm": 9.356002807617188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4365233778953552,
      "logits/rejected": -0.3290078341960907,
      "logps/chosen": -623.9139404296875,
      "logps/rejected": -583.208251953125,
      "loss": 0.5436,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6355010867118835,
      "rewards/margins": 1.3162705898284912,
      "rewards/rejected": -1.9517717361450195,
      "step": 9693
    },
    {
      "epoch": 1.2685204874458167,
      "grad_norm": 7.2980170249938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10072897374629974,
      "logits/rejected": -0.14465856552124023,
      "logps/chosen": -440.742919921875,
      "logps/rejected": -368.4460754394531,
      "loss": 0.4703,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7317970395088196,
      "rewards/margins": 1.4879586696624756,
      "rewards/rejected": -2.2197554111480713,
      "step": 9694
    },
    {
      "epoch": 1.2686513453831685,
      "grad_norm": 9.613310813903809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12530046701431274,
      "logits/rejected": -0.17537231743335724,
      "logps/chosen": -471.2889709472656,
      "logps/rejected": -367.3196716308594,
      "loss": 0.5006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3720053434371948,
      "rewards/margins": 0.8867572546005249,
      "rewards/rejected": -2.2587625980377197,
      "step": 9695
    },
    {
      "epoch": 1.2687822033205203,
      "grad_norm": 6.652088165283203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6814782023429871,
      "logits/rejected": -0.4553321301937103,
      "logps/chosen": -341.455078125,
      "logps/rejected": -306.24114990234375,
      "loss": 0.4717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0893417596817017,
      "rewards/margins": 1.541861653327942,
      "rewards/rejected": -2.6312031745910645,
      "step": 9696
    },
    {
      "epoch": 1.2689130612578718,
      "grad_norm": 6.413169860839844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20700371265411377,
      "logits/rejected": -0.13031843304634094,
      "logps/chosen": -422.4952392578125,
      "logps/rejected": -405.40673828125,
      "loss": 0.3362,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7860316038131714,
      "rewards/margins": 1.6333370208740234,
      "rewards/rejected": -2.4193685054779053,
      "step": 9697
    },
    {
      "epoch": 1.2690439191952236,
      "grad_norm": 5.516409873962402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057770274579524994,
      "logits/rejected": -0.28438764810562134,
      "logps/chosen": -371.2821350097656,
      "logps/rejected": -335.7841491699219,
      "loss": 0.2314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2437070608139038,
      "rewards/margins": 3.0033159255981445,
      "rewards/rejected": -4.247023105621338,
      "step": 9698
    },
    {
      "epoch": 1.2691747771325754,
      "grad_norm": 7.996853351593018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4827437996864319,
      "logits/rejected": -0.46144336462020874,
      "logps/chosen": -379.2810363769531,
      "logps/rejected": -354.0699462890625,
      "loss": 0.5665,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3037816882133484,
      "rewards/margins": 0.42001402378082275,
      "rewards/rejected": -0.7237957119941711,
      "step": 9699
    },
    {
      "epoch": 1.2693056350699272,
      "grad_norm": 6.213010311126709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34487083554267883,
      "logits/rejected": -0.3012153208255768,
      "logps/chosen": -475.1339111328125,
      "logps/rejected": -529.0817260742188,
      "loss": 0.3252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0725466012954712,
      "rewards/margins": 1.391875982284546,
      "rewards/rejected": -2.4644227027893066,
      "step": 9700
    },
    {
      "epoch": 1.269436493007279,
      "grad_norm": 4.106308460235596,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1726764589548111,
      "logits/rejected": -0.050298187881708145,
      "logps/chosen": -546.6347045898438,
      "logps/rejected": -477.1143493652344,
      "loss": 0.1821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42498281598091125,
      "rewards/margins": 2.374263286590576,
      "rewards/rejected": -2.799246072769165,
      "step": 9701
    },
    {
      "epoch": 1.2695673509446308,
      "grad_norm": 8.2027587890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2161286622285843,
      "logits/rejected": -0.3681800365447998,
      "logps/chosen": -670.017822265625,
      "logps/rejected": -445.4859924316406,
      "loss": 0.4287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3072633743286133,
      "rewards/margins": 1.3729416131973267,
      "rewards/rejected": -2.6802048683166504,
      "step": 9702
    },
    {
      "epoch": 1.2696982088819824,
      "grad_norm": 6.005122184753418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04364652559161186,
      "logits/rejected": -0.21900907158851624,
      "logps/chosen": -608.3897705078125,
      "logps/rejected": -470.38165283203125,
      "loss": 0.3155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36187246441841125,
      "rewards/margins": 1.5577921867370605,
      "rewards/rejected": -1.9196648597717285,
      "step": 9703
    },
    {
      "epoch": 1.2698290668193342,
      "grad_norm": 9.006265640258789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048420943319797516,
      "logits/rejected": -0.0331856831908226,
      "logps/chosen": -551.3670654296875,
      "logps/rejected": -616.2401733398438,
      "loss": 0.4783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5079097747802734,
      "rewards/margins": 1.381678581237793,
      "rewards/rejected": -1.8895883560180664,
      "step": 9704
    },
    {
      "epoch": 1.269959924756686,
      "grad_norm": 6.262720108032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16662392020225525,
      "logits/rejected": -0.2963981330394745,
      "logps/chosen": -552.6016235351562,
      "logps/rejected": -405.7470703125,
      "loss": 0.3498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25013887882232666,
      "rewards/margins": 1.1070165634155273,
      "rewards/rejected": -1.357155442237854,
      "step": 9705
    },
    {
      "epoch": 1.2700907826940377,
      "grad_norm": 11.276484489440918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14195477962493896,
      "logits/rejected": -0.48067089915275574,
      "logps/chosen": -579.6172485351562,
      "logps/rejected": -459.26751708984375,
      "loss": 0.5126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0005173683166504,
      "rewards/margins": 0.891438901424408,
      "rewards/rejected": -1.8919563293457031,
      "step": 9706
    },
    {
      "epoch": 1.2702216406313895,
      "grad_norm": 7.535489559173584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10199572145938873,
      "logits/rejected": -0.07526637613773346,
      "logps/chosen": -408.12322998046875,
      "logps/rejected": -512.7227783203125,
      "loss": 0.3184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3300280570983887,
      "rewards/margins": 1.944960594177246,
      "rewards/rejected": -3.2749886512756348,
      "step": 9707
    },
    {
      "epoch": 1.2703524985687413,
      "grad_norm": 8.063396453857422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17770759761333466,
      "logits/rejected": -0.1479053497314453,
      "logps/chosen": -354.9868469238281,
      "logps/rejected": -414.3668212890625,
      "loss": 0.5196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31702327728271484,
      "rewards/margins": 1.1048506498336792,
      "rewards/rejected": -1.4218738079071045,
      "step": 9708
    },
    {
      "epoch": 1.2704833565060931,
      "grad_norm": 8.201786041259766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044383540749549866,
      "logits/rejected": -0.24232298135757446,
      "logps/chosen": -434.9768981933594,
      "logps/rejected": -458.9371337890625,
      "loss": 0.3343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4700371026992798,
      "rewards/margins": 1.4572103023529053,
      "rewards/rejected": -2.9272472858428955,
      "step": 9709
    },
    {
      "epoch": 1.270614214443445,
      "grad_norm": 4.943160533905029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28334933519363403,
      "logits/rejected": -0.2961711883544922,
      "logps/chosen": -398.20703125,
      "logps/rejected": -437.09429931640625,
      "loss": 0.3257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7569466233253479,
      "rewards/margins": 1.6046029329299927,
      "rewards/rejected": -2.3615493774414062,
      "step": 9710
    },
    {
      "epoch": 1.2707450723807967,
      "grad_norm": 10.315533638000488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18529082834720612,
      "logits/rejected": -0.2930295467376709,
      "logps/chosen": -482.71197509765625,
      "logps/rejected": -546.5797729492188,
      "loss": 0.5655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7611353397369385,
      "rewards/margins": 0.6970915198326111,
      "rewards/rejected": -2.4582269191741943,
      "step": 9711
    },
    {
      "epoch": 1.2708759303181485,
      "grad_norm": 9.984598159790039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09030494093894958,
      "logits/rejected": -0.3170676529407501,
      "logps/chosen": -534.5918579101562,
      "logps/rejected": -485.6197509765625,
      "loss": 0.6366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6702670454978943,
      "rewards/margins": 1.265669822692871,
      "rewards/rejected": -1.9359370470046997,
      "step": 9712
    },
    {
      "epoch": 1.2710067882555,
      "grad_norm": 7.81519079208374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0681854635477066,
      "logits/rejected": 0.05025237426161766,
      "logps/chosen": -499.26092529296875,
      "logps/rejected": -639.4702758789062,
      "loss": 0.3979,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25443193316459656,
      "rewards/margins": 1.365630030632019,
      "rewards/rejected": -1.6200618743896484,
      "step": 9713
    },
    {
      "epoch": 1.2711376461928519,
      "grad_norm": 9.030404090881348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15834328532218933,
      "logits/rejected": -0.0771128386259079,
      "logps/chosen": -492.2135925292969,
      "logps/rejected": -570.7257080078125,
      "loss": 0.4806,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6856266260147095,
      "rewards/margins": 1.5616888999938965,
      "rewards/rejected": -2.2473156452178955,
      "step": 9714
    },
    {
      "epoch": 1.2712685041302036,
      "grad_norm": 13.018723487854004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3462052345275879,
      "logits/rejected": -0.1479099988937378,
      "logps/chosen": -396.2944030761719,
      "logps/rejected": -408.333984375,
      "loss": 0.588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9919588565826416,
      "rewards/margins": 0.9122198224067688,
      "rewards/rejected": -1.9041786193847656,
      "step": 9715
    },
    {
      "epoch": 1.2713993620675554,
      "grad_norm": 5.762678146362305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25185829401016235,
      "logits/rejected": -0.15479052066802979,
      "logps/chosen": -387.87969970703125,
      "logps/rejected": -444.9258117675781,
      "loss": 0.3034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0299123525619507,
      "rewards/margins": 1.6144182682037354,
      "rewards/rejected": -2.6443305015563965,
      "step": 9716
    },
    {
      "epoch": 1.2715302200049072,
      "grad_norm": 8.838753700256348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16488559544086456,
      "logits/rejected": -0.20196186006069183,
      "logps/chosen": -331.0502014160156,
      "logps/rejected": -434.3577880859375,
      "loss": 0.441,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3413664102554321,
      "rewards/margins": 1.1621891260147095,
      "rewards/rejected": -2.5035552978515625,
      "step": 9717
    },
    {
      "epoch": 1.2716610779422588,
      "grad_norm": 8.631247520446777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2412509173154831,
      "logits/rejected": -0.2889349162578583,
      "logps/chosen": -535.1617431640625,
      "logps/rejected": -504.94500732421875,
      "loss": 0.4147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0962004661560059,
      "rewards/margins": 0.8702707290649414,
      "rewards/rejected": -1.9664710760116577,
      "step": 9718
    },
    {
      "epoch": 1.2717919358796106,
      "grad_norm": 6.14814567565918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17623506486415863,
      "logits/rejected": -0.43666231632232666,
      "logps/chosen": -357.45947265625,
      "logps/rejected": -431.930908203125,
      "loss": 0.3367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3594440817832947,
      "rewards/margins": 1.4200176000595093,
      "rewards/rejected": -1.7794616222381592,
      "step": 9719
    },
    {
      "epoch": 1.2719227938169624,
      "grad_norm": 11.16802978515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05714114382863045,
      "logits/rejected": -0.13961732387542725,
      "logps/chosen": -388.1188659667969,
      "logps/rejected": -422.1629333496094,
      "loss": 0.6992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3035778999328613,
      "rewards/margins": 0.906067967414856,
      "rewards/rejected": -2.2096457481384277,
      "step": 9720
    },
    {
      "epoch": 1.2720536517543142,
      "grad_norm": 6.500832557678223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1844651997089386,
      "logits/rejected": -0.36316874623298645,
      "logps/chosen": -374.47216796875,
      "logps/rejected": -355.1711120605469,
      "loss": 0.3773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5558700561523438,
      "rewards/margins": 1.144971251487732,
      "rewards/rejected": -1.7008413076400757,
      "step": 9721
    },
    {
      "epoch": 1.272184509691666,
      "grad_norm": 3.02335524559021,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06749938428401947,
      "logits/rejected": -0.46046483516693115,
      "logps/chosen": -437.1551818847656,
      "logps/rejected": -435.62396240234375,
      "loss": 0.126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.32068464159965515,
      "rewards/margins": 2.906503200531006,
      "rewards/rejected": -3.2271881103515625,
      "step": 9722
    },
    {
      "epoch": 1.2723153676290178,
      "grad_norm": 9.900684356689453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33554673194885254,
      "logits/rejected": -0.40304964780807495,
      "logps/chosen": -434.7503356933594,
      "logps/rejected": -397.9130554199219,
      "loss": 0.5752,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6871985793113708,
      "rewards/margins": 0.8227646350860596,
      "rewards/rejected": -1.5099631547927856,
      "step": 9723
    },
    {
      "epoch": 1.2724462255663695,
      "grad_norm": 9.375664710998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1763898730278015,
      "logits/rejected": -0.19807545840740204,
      "logps/chosen": -571.8594970703125,
      "logps/rejected": -668.409423828125,
      "loss": 0.54,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21878740191459656,
      "rewards/margins": 0.9486712217330933,
      "rewards/rejected": -1.1674585342407227,
      "step": 9724
    },
    {
      "epoch": 1.2725770835037213,
      "grad_norm": 11.142071723937988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26479747891426086,
      "logits/rejected": -0.17555132508277893,
      "logps/chosen": -695.895751953125,
      "logps/rejected": -581.2130737304688,
      "loss": 0.7159,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4717894792556763,
      "rewards/margins": 0.39984071254730225,
      "rewards/rejected": -1.8716301918029785,
      "step": 9725
    },
    {
      "epoch": 1.2727079414410731,
      "grad_norm": 8.635187149047852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01664210483431816,
      "logits/rejected": 0.15118259191513062,
      "logps/chosen": -396.4737548828125,
      "logps/rejected": -511.685546875,
      "loss": 0.4761,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28163814544677734,
      "rewards/margins": 0.9559935927391052,
      "rewards/rejected": -1.2376317977905273,
      "step": 9726
    },
    {
      "epoch": 1.272838799378425,
      "grad_norm": 7.360415458679199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20686985552310944,
      "logits/rejected": -0.2777389883995056,
      "logps/chosen": -466.8998718261719,
      "logps/rejected": -517.5269165039062,
      "loss": 0.3465,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7634536623954773,
      "rewards/margins": 1.630661964416504,
      "rewards/rejected": -2.394115686416626,
      "step": 9727
    },
    {
      "epoch": 1.2729696573157765,
      "grad_norm": 2.669135332107544,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2882272005081177,
      "logits/rejected": -0.3050675392150879,
      "logps/chosen": -501.8369445800781,
      "logps/rejected": -570.1791381835938,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9282294511795044,
      "rewards/margins": 3.086853265762329,
      "rewards/rejected": -4.015082836151123,
      "step": 9728
    },
    {
      "epoch": 1.2731005152531283,
      "grad_norm": 6.9055047035217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04134261608123779,
      "logits/rejected": -0.2146279215812683,
      "logps/chosen": -400.2551574707031,
      "logps/rejected": -515.4408569335938,
      "loss": 0.2794,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8089755773544312,
      "rewards/margins": 1.8498430252075195,
      "rewards/rejected": -2.6588187217712402,
      "step": 9729
    },
    {
      "epoch": 1.27323137319048,
      "grad_norm": 8.741765975952148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3208421766757965,
      "logits/rejected": -0.4196230471134186,
      "logps/chosen": -442.9488525390625,
      "logps/rejected": -459.15582275390625,
      "loss": 0.4301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.489449143409729,
      "rewards/margins": 1.570260763168335,
      "rewards/rejected": -3.0597100257873535,
      "step": 9730
    },
    {
      "epoch": 1.2733622311278319,
      "grad_norm": 6.992184638977051,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06786959618330002,
      "logits/rejected": -0.2220170646905899,
      "logps/chosen": -584.4210815429688,
      "logps/rejected": -573.4810791015625,
      "loss": 0.4024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9852921962738037,
      "rewards/margins": 1.2060283422470093,
      "rewards/rejected": -2.1913204193115234,
      "step": 9731
    },
    {
      "epoch": 1.2734930890651837,
      "grad_norm": 10.355079650878906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21539583802223206,
      "logits/rejected": -0.48670080304145813,
      "logps/chosen": -409.012451171875,
      "logps/rejected": -315.31292724609375,
      "loss": 0.6307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0417089462280273,
      "rewards/margins": 0.3666147291660309,
      "rewards/rejected": -1.4083236455917358,
      "step": 9732
    },
    {
      "epoch": 1.2736239470025355,
      "grad_norm": 3.5384583473205566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1946258246898651,
      "logits/rejected": -0.2759881317615509,
      "logps/chosen": -462.125244140625,
      "logps/rejected": -530.1319580078125,
      "loss": 0.1451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6256071925163269,
      "rewards/margins": 2.290419578552246,
      "rewards/rejected": -2.9160265922546387,
      "step": 9733
    },
    {
      "epoch": 1.273754804939887,
      "grad_norm": 7.518283843994141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4215174615383148,
      "logits/rejected": -0.33034348487854004,
      "logps/chosen": -359.58349609375,
      "logps/rejected": -368.0671081542969,
      "loss": 0.4696,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.039151668548584,
      "rewards/margins": 1.0055489540100098,
      "rewards/rejected": -2.0447006225585938,
      "step": 9734
    },
    {
      "epoch": 1.2738856628772388,
      "grad_norm": 5.514207363128662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25029510259628296,
      "logits/rejected": -0.11438940465450287,
      "logps/chosen": -605.9973754882812,
      "logps/rejected": -486.77276611328125,
      "loss": 0.2077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.749498188495636,
      "rewards/margins": 2.058138847351074,
      "rewards/rejected": -2.8076372146606445,
      "step": 9735
    },
    {
      "epoch": 1.2740165208145906,
      "grad_norm": 5.266119003295898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06573352962732315,
      "logits/rejected": 0.0015884824097156525,
      "logps/chosen": -453.3028259277344,
      "logps/rejected": -507.3760070800781,
      "loss": 0.2436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7093960046768188,
      "rewards/margins": 1.5376056432724,
      "rewards/rejected": -2.2470016479492188,
      "step": 9736
    },
    {
      "epoch": 1.2741473787519424,
      "grad_norm": 5.797269821166992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.248045414686203,
      "logits/rejected": -0.23762506246566772,
      "logps/chosen": -360.54345703125,
      "logps/rejected": -398.9419860839844,
      "loss": 0.2733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7534196376800537,
      "rewards/margins": 1.2742030620574951,
      "rewards/rejected": -2.027622699737549,
      "step": 9737
    },
    {
      "epoch": 1.2742782366892942,
      "grad_norm": 11.41348934173584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2735663652420044,
      "logits/rejected": -0.27233633399009705,
      "logps/chosen": -462.6826477050781,
      "logps/rejected": -537.0279541015625,
      "loss": 0.6833,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8336925506591797,
      "rewards/margins": 0.8486659526824951,
      "rewards/rejected": -1.6823585033416748,
      "step": 9738
    },
    {
      "epoch": 1.274409094626646,
      "grad_norm": 6.742866039276123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3188994526863098,
      "logits/rejected": -0.44905295968055725,
      "logps/chosen": -656.3502807617188,
      "logps/rejected": -470.62469482421875,
      "loss": 0.225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2313696146011353,
      "rewards/margins": 2.3856825828552246,
      "rewards/rejected": -3.6170520782470703,
      "step": 9739
    },
    {
      "epoch": 1.2745399525639978,
      "grad_norm": 9.64463996887207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4651680886745453,
      "logits/rejected": -0.5566431879997253,
      "logps/chosen": -556.394775390625,
      "logps/rejected": -405.7303161621094,
      "loss": 0.5075,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2539079189300537,
      "rewards/margins": 1.0160974264144897,
      "rewards/rejected": -2.270005226135254,
      "step": 9740
    },
    {
      "epoch": 1.2746708105013496,
      "grad_norm": 6.571481227874756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20776766538619995,
      "logits/rejected": -0.14749455451965332,
      "logps/chosen": -347.5210266113281,
      "logps/rejected": -360.56475830078125,
      "loss": 0.4194,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34824371337890625,
      "rewards/margins": 1.1099233627319336,
      "rewards/rejected": -1.4581671953201294,
      "step": 9741
    },
    {
      "epoch": 1.2748016684387014,
      "grad_norm": 13.812795639038086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2923673689365387,
      "logits/rejected": -0.3805251121520996,
      "logps/chosen": -521.9073486328125,
      "logps/rejected": -636.9484252929688,
      "loss": 0.835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9849300384521484,
      "rewards/margins": 0.804417610168457,
      "rewards/rejected": -2.7893478870391846,
      "step": 9742
    },
    {
      "epoch": 1.274932526376053,
      "grad_norm": 5.986687183380127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7039687037467957,
      "logits/rejected": -0.6409135460853577,
      "logps/chosen": -321.775634765625,
      "logps/rejected": -397.5726623535156,
      "loss": 0.2452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2229464054107666,
      "rewards/margins": 2.4274487495422363,
      "rewards/rejected": -3.650395393371582,
      "step": 9743
    },
    {
      "epoch": 1.2750633843134047,
      "grad_norm": 5.806796073913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33603304624557495,
      "logits/rejected": -0.30661314725875854,
      "logps/chosen": -342.671875,
      "logps/rejected": -330.113525390625,
      "loss": 0.3628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4352491497993469,
      "rewards/margins": 1.2007511854171753,
      "rewards/rejected": -1.6360002756118774,
      "step": 9744
    },
    {
      "epoch": 1.2751942422507565,
      "grad_norm": 5.058134078979492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2221437394618988,
      "logits/rejected": -0.36722517013549805,
      "logps/chosen": -387.7615966796875,
      "logps/rejected": -366.41668701171875,
      "loss": 0.1586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7999564409255981,
      "rewards/margins": 2.266354560852051,
      "rewards/rejected": -4.066310882568359,
      "step": 9745
    },
    {
      "epoch": 1.2753251001881083,
      "grad_norm": 4.212443828582764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11617720127105713,
      "logits/rejected": -0.2999454140663147,
      "logps/chosen": -505.67864990234375,
      "logps/rejected": -491.259765625,
      "loss": 0.1697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5464504957199097,
      "rewards/margins": 2.0763652324676514,
      "rewards/rejected": -3.6228156089782715,
      "step": 9746
    },
    {
      "epoch": 1.27545595812546,
      "grad_norm": 7.939564228057861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1810389906167984,
      "logits/rejected": -0.3676931858062744,
      "logps/chosen": -729.6100463867188,
      "logps/rejected": -495.59710693359375,
      "loss": 0.3248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7493587732315063,
      "rewards/margins": 1.682746171951294,
      "rewards/rejected": -3.43210506439209,
      "step": 9747
    },
    {
      "epoch": 1.2755868160628119,
      "grad_norm": 7.67009162902832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22349342703819275,
      "logits/rejected": -0.2688037157058716,
      "logps/chosen": -359.83489990234375,
      "logps/rejected": -367.7338562011719,
      "loss": 0.4174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3247991800308228,
      "rewards/margins": 1.473725438117981,
      "rewards/rejected": -2.7985246181488037,
      "step": 9748
    },
    {
      "epoch": 1.2757176740001634,
      "grad_norm": 9.100982666015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10406890511512756,
      "logits/rejected": -0.005019063130021095,
      "logps/chosen": -516.2990112304688,
      "logps/rejected": -470.0550842285156,
      "loss": 0.5388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3332008123397827,
      "rewards/margins": 0.8000346422195435,
      "rewards/rejected": -2.133235216140747,
      "step": 9749
    },
    {
      "epoch": 1.2758485319375152,
      "grad_norm": 9.186263084411621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11515350639820099,
      "logits/rejected": -0.010060733184218407,
      "logps/chosen": -381.82159423828125,
      "logps/rejected": -414.30322265625,
      "loss": 0.3927,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.379732370376587,
      "rewards/margins": 2.4847183227539062,
      "rewards/rejected": -3.864450693130493,
      "step": 9750
    },
    {
      "epoch": 1.275979389874867,
      "grad_norm": 12.302297592163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26167720556259155,
      "logits/rejected": -0.252338171005249,
      "logps/chosen": -425.415283203125,
      "logps/rejected": -518.4998779296875,
      "loss": 0.6317,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5006351470947266,
      "rewards/margins": 0.8776917457580566,
      "rewards/rejected": -2.378326892852783,
      "step": 9751
    },
    {
      "epoch": 1.2761102478122188,
      "grad_norm": 11.473310470581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22905537486076355,
      "logits/rejected": 0.07371897995471954,
      "logps/chosen": -363.18310546875,
      "logps/rejected": -440.0596923828125,
      "loss": 0.5376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9817461967468262,
      "rewards/margins": 1.237443208694458,
      "rewards/rejected": -2.219189167022705,
      "step": 9752
    },
    {
      "epoch": 1.2762411057495706,
      "grad_norm": 4.603023529052734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09444983303546906,
      "logits/rejected": -0.15213407576084137,
      "logps/chosen": -516.0874633789062,
      "logps/rejected": -529.2086181640625,
      "loss": 0.2127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7928287982940674,
      "rewards/margins": 1.9989933967590332,
      "rewards/rejected": -2.7918219566345215,
      "step": 9753
    },
    {
      "epoch": 1.2763719636869224,
      "grad_norm": 9.026961326599121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10963746905326843,
      "logits/rejected": -0.1476047784090042,
      "logps/chosen": -554.6661987304688,
      "logps/rejected": -518.4212646484375,
      "loss": 0.7757,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5847797393798828,
      "rewards/margins": 0.5063164234161377,
      "rewards/rejected": -1.091096043586731,
      "step": 9754
    },
    {
      "epoch": 1.2765028216242742,
      "grad_norm": 9.879273414611816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5148726105690002,
      "logits/rejected": -0.43040409684181213,
      "logps/chosen": -335.36932373046875,
      "logps/rejected": -390.09765625,
      "loss": 0.5372,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3882312774658203,
      "rewards/margins": 0.8063535690307617,
      "rewards/rejected": -2.194584846496582,
      "step": 9755
    },
    {
      "epoch": 1.276633679561626,
      "grad_norm": 7.608193874359131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2547048032283783,
      "logits/rejected": -0.42749226093292236,
      "logps/chosen": -421.5696716308594,
      "logps/rejected": -535.3248291015625,
      "loss": 0.4403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.212692141532898,
      "rewards/margins": 1.3037939071655273,
      "rewards/rejected": -2.516486167907715,
      "step": 9756
    },
    {
      "epoch": 1.2767645374989778,
      "grad_norm": 10.164590835571289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27007752656936646,
      "logits/rejected": -0.18004615604877472,
      "logps/chosen": -394.294189453125,
      "logps/rejected": -468.7608337402344,
      "loss": 0.7214,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3890632390975952,
      "rewards/margins": 0.8258633017539978,
      "rewards/rejected": -2.2149264812469482,
      "step": 9757
    },
    {
      "epoch": 1.2768953954363296,
      "grad_norm": 9.983057975769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03837507963180542,
      "logits/rejected": -0.4082798659801483,
      "logps/chosen": -582.254150390625,
      "logps/rejected": -477.4893798828125,
      "loss": 0.4664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6630317568778992,
      "rewards/margins": 1.726651906967163,
      "rewards/rejected": -2.389683723449707,
      "step": 9758
    },
    {
      "epoch": 1.2770262533736811,
      "grad_norm": 7.300382614135742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0989948958158493,
      "logits/rejected": -0.21907496452331543,
      "logps/chosen": -355.6223449707031,
      "logps/rejected": -413.48828125,
      "loss": 0.2951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9382250308990479,
      "rewards/margins": 1.6319692134857178,
      "rewards/rejected": -3.5701940059661865,
      "step": 9759
    },
    {
      "epoch": 1.277157111311033,
      "grad_norm": 14.233309745788574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18764127790927887,
      "logits/rejected": -0.29108700156211853,
      "logps/chosen": -363.3990783691406,
      "logps/rejected": -400.8646240234375,
      "loss": 0.7986,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.297310471534729,
      "rewards/margins": 0.36774736642837524,
      "rewards/rejected": -1.6650577783584595,
      "step": 9760
    },
    {
      "epoch": 1.2772879692483847,
      "grad_norm": 9.818729400634766,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03560441732406616,
      "logits/rejected": 0.021425653249025345,
      "logps/chosen": -508.57476806640625,
      "logps/rejected": -535.3671264648438,
      "loss": 0.4947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4292397499084473,
      "rewards/margins": 1.584373116493225,
      "rewards/rejected": -3.013612747192383,
      "step": 9761
    },
    {
      "epoch": 1.2774188271857365,
      "grad_norm": 10.597284317016602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0803125649690628,
      "logits/rejected": -0.1696123480796814,
      "logps/chosen": -405.5743408203125,
      "logps/rejected": -298.58416748046875,
      "loss": 0.6323,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5252667665481567,
      "rewards/margins": 0.5203176140785217,
      "rewards/rejected": -2.045584201812744,
      "step": 9762
    },
    {
      "epoch": 1.2775496851230883,
      "grad_norm": 5.473292350769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24102672934532166,
      "logits/rejected": -0.2133132964372635,
      "logps/chosen": -378.745849609375,
      "logps/rejected": -354.1810302734375,
      "loss": 0.1819,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7166814208030701,
      "rewards/margins": 2.9897398948669434,
      "rewards/rejected": -3.706421136856079,
      "step": 9763
    },
    {
      "epoch": 1.2776805430604399,
      "grad_norm": 6.662318229675293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04068353772163391,
      "logits/rejected": -0.1088387593626976,
      "logps/chosen": -606.57568359375,
      "logps/rejected": -479.46807861328125,
      "loss": 0.2814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8148239254951477,
      "rewards/margins": 2.0688507556915283,
      "rewards/rejected": -2.8836748600006104,
      "step": 9764
    },
    {
      "epoch": 1.2778114009977917,
      "grad_norm": 4.593719005584717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09005855768918991,
      "logits/rejected": -0.19276636838912964,
      "logps/chosen": -347.2794494628906,
      "logps/rejected": -330.92822265625,
      "loss": 0.2508,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3684247732162476,
      "rewards/margins": 1.6170508861541748,
      "rewards/rejected": -2.985475540161133,
      "step": 9765
    },
    {
      "epoch": 1.2779422589351435,
      "grad_norm": 6.371020793914795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02812708355486393,
      "logits/rejected": -0.20572727918624878,
      "logps/chosen": -875.779052734375,
      "logps/rejected": -675.2860107421875,
      "loss": 0.263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7082489132881165,
      "rewards/margins": 2.1676723957061768,
      "rewards/rejected": -2.8759212493896484,
      "step": 9766
    },
    {
      "epoch": 1.2780731168724953,
      "grad_norm": 11.707840919494629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07170942425727844,
      "logits/rejected": 0.05167418718338013,
      "logps/chosen": -405.1062927246094,
      "logps/rejected": -363.4019775390625,
      "loss": 0.4302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5921531319618225,
      "rewards/margins": 1.4174935817718506,
      "rewards/rejected": -2.0096468925476074,
      "step": 9767
    },
    {
      "epoch": 1.278203974809847,
      "grad_norm": 8.537613868713379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.061695728451013565,
      "logits/rejected": -0.0833425521850586,
      "logps/chosen": -365.8922424316406,
      "logps/rejected": -459.80462646484375,
      "loss": 0.4566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.138911247253418,
      "rewards/margins": 1.4824001789093018,
      "rewards/rejected": -2.621311664581299,
      "step": 9768
    },
    {
      "epoch": 1.2783348327471988,
      "grad_norm": 12.605822563171387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.049723051488399506,
      "logits/rejected": -0.10921482741832733,
      "logps/chosen": -580.8991088867188,
      "logps/rejected": -543.194091796875,
      "loss": 0.7284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0205318927764893,
      "rewards/margins": 0.3682456612586975,
      "rewards/rejected": -2.388777494430542,
      "step": 9769
    },
    {
      "epoch": 1.2784656906845506,
      "grad_norm": 3.284574270248413,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4982302784919739,
      "logits/rejected": -0.42407527565956116,
      "logps/chosen": -320.6957092285156,
      "logps/rejected": -571.1734619140625,
      "loss": 0.1572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6752253174781799,
      "rewards/margins": 3.397264242172241,
      "rewards/rejected": -4.072489261627197,
      "step": 9770
    },
    {
      "epoch": 1.2785965486219024,
      "grad_norm": 9.149458885192871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3173117935657501,
      "logits/rejected": -0.35848551988601685,
      "logps/chosen": -293.67266845703125,
      "logps/rejected": -352.0540466308594,
      "loss": 0.5086,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5840103626251221,
      "rewards/margins": 1.1405291557312012,
      "rewards/rejected": -1.7245395183563232,
      "step": 9771
    },
    {
      "epoch": 1.2787274065592542,
      "grad_norm": 7.954880237579346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21190014481544495,
      "logits/rejected": -0.2400333434343338,
      "logps/chosen": -414.5673828125,
      "logps/rejected": -354.1816711425781,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9976526498794556,
      "rewards/margins": 1.3172670602798462,
      "rewards/rejected": -2.3149197101593018,
      "step": 9772
    },
    {
      "epoch": 1.278858264496606,
      "grad_norm": 5.752766132354736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039210230112075806,
      "logits/rejected": -0.0709734857082367,
      "logps/chosen": -506.2320556640625,
      "logps/rejected": -502.9686279296875,
      "loss": 0.3291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30771583318710327,
      "rewards/margins": 1.8636395931243896,
      "rewards/rejected": -2.1713554859161377,
      "step": 9773
    },
    {
      "epoch": 1.2789891224339576,
      "grad_norm": 9.089034080505371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07546687126159668,
      "logits/rejected": -0.15212219953536987,
      "logps/chosen": -363.13043212890625,
      "logps/rejected": -417.2893981933594,
      "loss": 0.501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9798160791397095,
      "rewards/margins": 1.333864688873291,
      "rewards/rejected": -2.313680648803711,
      "step": 9774
    },
    {
      "epoch": 1.2791199803713094,
      "grad_norm": 7.132752895355225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10080521553754807,
      "logits/rejected": -0.31008240580558777,
      "logps/chosen": -480.03912353515625,
      "logps/rejected": -452.81121826171875,
      "loss": 0.374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5943808555603027,
      "rewards/margins": 1.4475476741790771,
      "rewards/rejected": -2.04192852973938,
      "step": 9775
    },
    {
      "epoch": 1.2792508383086612,
      "grad_norm": 8.123542785644531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4391520619392395,
      "logits/rejected": -0.47062674164772034,
      "logps/chosen": -286.5715637207031,
      "logps/rejected": -278.6916198730469,
      "loss": 0.3827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.659763216972351,
      "rewards/margins": 1.285582423210144,
      "rewards/rejected": -2.945345401763916,
      "step": 9776
    },
    {
      "epoch": 1.279381696246013,
      "grad_norm": 7.453453063964844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1289958655834198,
      "logits/rejected": -0.06107322871685028,
      "logps/chosen": -532.8779296875,
      "logps/rejected": -487.084228515625,
      "loss": 0.3282,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6064707040786743,
      "rewards/margins": 1.6243212223052979,
      "rewards/rejected": -2.2307918071746826,
      "step": 9777
    },
    {
      "epoch": 1.2795125541833647,
      "grad_norm": 5.970251083374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21626150608062744,
      "logits/rejected": -0.20310449600219727,
      "logps/chosen": -311.1548767089844,
      "logps/rejected": -368.3539123535156,
      "loss": 0.2743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1418367624282837,
      "rewards/margins": 1.555316686630249,
      "rewards/rejected": -1.4134798049926758,
      "step": 9778
    },
    {
      "epoch": 1.2796434121207165,
      "grad_norm": 11.039254188537598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.349860817193985,
      "logits/rejected": -0.5147213935852051,
      "logps/chosen": -523.9017333984375,
      "logps/rejected": -455.7589111328125,
      "loss": 0.4435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5613476037979126,
      "rewards/margins": 1.7528281211853027,
      "rewards/rejected": -2.314175844192505,
      "step": 9779
    },
    {
      "epoch": 1.279774270058068,
      "grad_norm": 6.781551361083984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3605360984802246,
      "logits/rejected": -0.3460313081741333,
      "logps/chosen": -583.65185546875,
      "logps/rejected": -546.2385864257812,
      "loss": 0.3507,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.033778190612793,
      "rewards/margins": 1.6740953922271729,
      "rewards/rejected": -2.707873582839966,
      "step": 9780
    },
    {
      "epoch": 1.2799051279954199,
      "grad_norm": 9.306802749633789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1450868397951126,
      "logits/rejected": -0.020109660923480988,
      "logps/chosen": -342.14373779296875,
      "logps/rejected": -362.9640808105469,
      "loss": 0.6311,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6268278360366821,
      "rewards/margins": 0.6633789539337158,
      "rewards/rejected": -2.2902069091796875,
      "step": 9781
    },
    {
      "epoch": 1.2800359859327717,
      "grad_norm": 9.845778465270996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12898413836956024,
      "logits/rejected": -0.19357481598854065,
      "logps/chosen": -526.6995239257812,
      "logps/rejected": -433.967041015625,
      "loss": 0.5514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0298089981079102,
      "rewards/margins": 1.3360670804977417,
      "rewards/rejected": -2.3658759593963623,
      "step": 9782
    },
    {
      "epoch": 1.2801668438701235,
      "grad_norm": 12.795892715454102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1988767683506012,
      "logits/rejected": -0.20499670505523682,
      "logps/chosen": -626.0634765625,
      "logps/rejected": -590.436279296875,
      "loss": 0.8434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8331658840179443,
      "rewards/margins": 0.3666931092739105,
      "rewards/rejected": -2.1998589038848877,
      "step": 9783
    },
    {
      "epoch": 1.2802977018074753,
      "grad_norm": 9.626869201660156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03757840394973755,
      "logits/rejected": -0.34901177883148193,
      "logps/chosen": -493.0977478027344,
      "logps/rejected": -423.00665283203125,
      "loss": 0.4228,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.36239492893219,
      "rewards/margins": 1.5718512535095215,
      "rewards/rejected": -2.934246301651001,
      "step": 9784
    },
    {
      "epoch": 1.280428559744827,
      "grad_norm": 9.45339584350586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15826329588890076,
      "logits/rejected": -0.2510155737400055,
      "logps/chosen": -405.94146728515625,
      "logps/rejected": -357.1813049316406,
      "loss": 0.4856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3198105096817017,
      "rewards/margins": 1.3597707748413086,
      "rewards/rejected": -2.6795811653137207,
      "step": 9785
    },
    {
      "epoch": 1.2805594176821788,
      "grad_norm": 10.765146255493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10046963393688202,
      "logits/rejected": -0.16461530327796936,
      "logps/chosen": -606.0606079101562,
      "logps/rejected": -596.572265625,
      "loss": 0.6763,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.524823009967804,
      "rewards/margins": 1.7514641284942627,
      "rewards/rejected": -2.2762868404388428,
      "step": 9786
    },
    {
      "epoch": 1.2806902756195306,
      "grad_norm": 4.418887615203857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09956324845552444,
      "logits/rejected": -0.41243040561676025,
      "logps/chosen": -401.31549072265625,
      "logps/rejected": -342.02606201171875,
      "loss": 0.2196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8637824058532715,
      "rewards/margins": 2.079753875732422,
      "rewards/rejected": -2.9435362815856934,
      "step": 9787
    },
    {
      "epoch": 1.2808211335568824,
      "grad_norm": 9.882370948791504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10811032354831696,
      "logits/rejected": -0.05723165348172188,
      "logps/chosen": -488.94097900390625,
      "logps/rejected": -407.2425537109375,
      "loss": 0.5603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4283573031425476,
      "rewards/margins": 0.8058921694755554,
      "rewards/rejected": -1.234249472618103,
      "step": 9788
    },
    {
      "epoch": 1.280951991494234,
      "grad_norm": 6.976649761199951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18787039816379547,
      "logits/rejected": -0.38341593742370605,
      "logps/chosen": -446.527099609375,
      "logps/rejected": -366.8498229980469,
      "loss": 0.4125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7363725304603577,
      "rewards/margins": 1.7098127603530884,
      "rewards/rejected": -2.446185350418091,
      "step": 9789
    },
    {
      "epoch": 1.2810828494315858,
      "grad_norm": 6.73933744430542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007576443254947662,
      "logits/rejected": -0.15439924597740173,
      "logps/chosen": -341.12322998046875,
      "logps/rejected": -410.5535888671875,
      "loss": 0.3056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1907347440719604,
      "rewards/margins": 1.576849102973938,
      "rewards/rejected": -2.7675838470458984,
      "step": 9790
    },
    {
      "epoch": 1.2812137073689376,
      "grad_norm": 6.6904215812683105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3804008960723877,
      "logits/rejected": 0.4113035500049591,
      "logps/chosen": -393.4534606933594,
      "logps/rejected": -403.3309326171875,
      "loss": 0.4883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0611492395401001,
      "rewards/margins": 1.4201933145523071,
      "rewards/rejected": -1.4813423156738281,
      "step": 9791
    },
    {
      "epoch": 1.2813445653062894,
      "grad_norm": 9.545441627502441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19038906693458557,
      "logits/rejected": -0.20165759325027466,
      "logps/chosen": -761.5142211914062,
      "logps/rejected": -489.19049072265625,
      "loss": 0.5845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8353271484375,
      "rewards/margins": 2.0196731090545654,
      "rewards/rejected": -2.8550004959106445,
      "step": 9792
    },
    {
      "epoch": 1.2814754232436412,
      "grad_norm": 9.344134330749512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1188240796327591,
      "logits/rejected": -0.30884596705436707,
      "logps/chosen": -653.2479858398438,
      "logps/rejected": -550.97607421875,
      "loss": 0.3418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4349387884140015,
      "rewards/margins": 1.289939284324646,
      "rewards/rejected": -2.7248780727386475,
      "step": 9793
    },
    {
      "epoch": 1.281606281180993,
      "grad_norm": 7.0646443367004395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1374872624874115,
      "logits/rejected": 0.037590112537145615,
      "logps/chosen": -793.8997192382812,
      "logps/rejected": -584.6548461914062,
      "loss": 0.3591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01737096905708313,
      "rewards/margins": 2.119318723678589,
      "rewards/rejected": -2.101947784423828,
      "step": 9794
    },
    {
      "epoch": 1.2817371391183445,
      "grad_norm": 8.43149471282959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07634399831295013,
      "logits/rejected": -0.09221339225769043,
      "logps/chosen": -330.2094421386719,
      "logps/rejected": -540.7853393554688,
      "loss": 0.3965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5186489224433899,
      "rewards/margins": 1.2149839401245117,
      "rewards/rejected": -1.7336329221725464,
      "step": 9795
    },
    {
      "epoch": 1.2818679970556963,
      "grad_norm": 7.793558120727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051315128803253174,
      "logits/rejected": -0.24382512271404266,
      "logps/chosen": -548.77880859375,
      "logps/rejected": -549.2899780273438,
      "loss": 0.305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8392641544342041,
      "rewards/margins": 1.815948724746704,
      "rewards/rejected": -2.655212640762329,
      "step": 9796
    },
    {
      "epoch": 1.281998854993048,
      "grad_norm": 7.187229633331299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3383191227912903,
      "logits/rejected": -0.3166263699531555,
      "logps/chosen": -437.28912353515625,
      "logps/rejected": -341.3335876464844,
      "loss": 0.3131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8232921361923218,
      "rewards/margins": 1.5005192756652832,
      "rewards/rejected": -2.3238115310668945,
      "step": 9797
    },
    {
      "epoch": 1.2821297129304,
      "grad_norm": 6.3795928955078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3896617591381073,
      "logits/rejected": -0.42937639355659485,
      "logps/chosen": -469.0840148925781,
      "logps/rejected": -466.37371826171875,
      "loss": 0.2618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6918508410453796,
      "rewards/margins": 1.6700413227081299,
      "rewards/rejected": -2.361891984939575,
      "step": 9798
    },
    {
      "epoch": 1.2822605708677517,
      "grad_norm": 11.959563255310059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006280349567532539,
      "logits/rejected": -0.1328224241733551,
      "logps/chosen": -532.4846801757812,
      "logps/rejected": -439.6194152832031,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1254448890686035,
      "rewards/margins": 1.163930892944336,
      "rewards/rejected": -2.2893757820129395,
      "step": 9799
    },
    {
      "epoch": 1.2823914288051035,
      "grad_norm": 6.292745590209961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08389082551002502,
      "logits/rejected": -0.05526430159807205,
      "logps/chosen": -583.667236328125,
      "logps/rejected": -580.53173828125,
      "loss": 0.2898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5979806184768677,
      "rewards/margins": 1.5902609825134277,
      "rewards/rejected": -3.188241720199585,
      "step": 9800
    },
    {
      "epoch": 1.2825222867424553,
      "grad_norm": 8.486207962036133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12727321684360504,
      "logits/rejected": -0.3325011432170868,
      "logps/chosen": -512.2379760742188,
      "logps/rejected": -392.41229248046875,
      "loss": 0.4146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9320785999298096,
      "rewards/margins": 1.1818058490753174,
      "rewards/rejected": -2.113884210586548,
      "step": 9801
    },
    {
      "epoch": 1.282653144679807,
      "grad_norm": 7.719543933868408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06270579248666763,
      "logits/rejected": -0.019188292324543,
      "logps/chosen": -479.4852600097656,
      "logps/rejected": -548.1294555664062,
      "loss": 0.4855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24816665053367615,
      "rewards/margins": 0.8301971554756165,
      "rewards/rejected": -1.0783638954162598,
      "step": 9802
    },
    {
      "epoch": 1.2827840026171589,
      "grad_norm": 7.58027982711792,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004843331873416901,
      "logits/rejected": -0.07530735433101654,
      "logps/chosen": -628.2548828125,
      "logps/rejected": -627.2686767578125,
      "loss": 0.3762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7321279644966125,
      "rewards/margins": 1.0218768119812012,
      "rewards/rejected": -1.754004716873169,
      "step": 9803
    },
    {
      "epoch": 1.2829148605545104,
      "grad_norm": 7.066450595855713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12015312910079956,
      "logits/rejected": -0.34959182143211365,
      "logps/chosen": -592.01318359375,
      "logps/rejected": -423.72222900390625,
      "loss": 0.3628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0420936644077301,
      "rewards/margins": 1.7024271488189697,
      "rewards/rejected": -1.6603333950042725,
      "step": 9804
    },
    {
      "epoch": 1.2830457184918622,
      "grad_norm": 10.07091236114502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018072228878736496,
      "logits/rejected": -0.3835858702659607,
      "logps/chosen": -537.0870361328125,
      "logps/rejected": -396.88201904296875,
      "loss": 0.6379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.498882532119751,
      "rewards/margins": 0.8007675409317017,
      "rewards/rejected": -2.299649953842163,
      "step": 9805
    },
    {
      "epoch": 1.283176576429214,
      "grad_norm": 15.736780166625977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29268497228622437,
      "logits/rejected": -0.23386834561824799,
      "logps/chosen": -591.6480102539062,
      "logps/rejected": -601.8014526367188,
      "loss": 0.3924,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23798678815364838,
      "rewards/margins": 1.6481307744979858,
      "rewards/rejected": -1.886117696762085,
      "step": 9806
    },
    {
      "epoch": 1.2833074343665658,
      "grad_norm": 5.137099742889404,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12389463186264038,
      "logits/rejected": -0.04784473031759262,
      "logps/chosen": -360.787353515625,
      "logps/rejected": -293.8033142089844,
      "loss": 0.4522,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08546915650367737,
      "rewards/margins": 1.492314338684082,
      "rewards/rejected": -1.406845211982727,
      "step": 9807
    },
    {
      "epoch": 1.2834382923039176,
      "grad_norm": 7.545289516448975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18686659634113312,
      "logits/rejected": -0.2690694332122803,
      "logps/chosen": -407.8873596191406,
      "logps/rejected": -428.5260314941406,
      "loss": 0.5036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.622113823890686,
      "rewards/margins": 0.8748066425323486,
      "rewards/rejected": -1.4969205856323242,
      "step": 9808
    },
    {
      "epoch": 1.2835691502412694,
      "grad_norm": 10.057662963867188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10694345831871033,
      "logits/rejected": 0.061068132519721985,
      "logps/chosen": -427.0128173828125,
      "logps/rejected": -357.0249938964844,
      "loss": 0.6636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0808587074279785,
      "rewards/margins": 0.5423440933227539,
      "rewards/rejected": -2.6232030391693115,
      "step": 9809
    },
    {
      "epoch": 1.283700008178621,
      "grad_norm": 14.43183708190918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32014885544776917,
      "logits/rejected": -0.3692587614059448,
      "logps/chosen": -351.8563232421875,
      "logps/rejected": -342.14031982421875,
      "loss": 0.8251,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5805166959762573,
      "rewards/margins": 0.18723584711551666,
      "rewards/rejected": -1.7677525281906128,
      "step": 9810
    },
    {
      "epoch": 1.2838308661159727,
      "grad_norm": 3.740413188934326,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1537870168685913,
      "logits/rejected": -0.3723085820674896,
      "logps/chosen": -558.71826171875,
      "logps/rejected": -594.6551513671875,
      "loss": 0.1457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.377480149269104,
      "rewards/margins": 2.510547399520874,
      "rewards/rejected": -2.8880274295806885,
      "step": 9811
    },
    {
      "epoch": 1.2839617240533245,
      "grad_norm": 6.093672275543213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49288254976272583,
      "logits/rejected": -0.3717268705368042,
      "logps/chosen": -524.9827270507812,
      "logps/rejected": -341.7776794433594,
      "loss": 0.2939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30400294065475464,
      "rewards/margins": 2.5429205894470215,
      "rewards/rejected": -2.846923828125,
      "step": 9812
    },
    {
      "epoch": 1.2840925819906763,
      "grad_norm": 9.398197174072266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1295548975467682,
      "logits/rejected": -0.17561081051826477,
      "logps/chosen": -491.4032287597656,
      "logps/rejected": -437.9142150878906,
      "loss": 0.5983,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9721814393997192,
      "rewards/margins": 0.829276442527771,
      "rewards/rejected": -1.8014578819274902,
      "step": 9813
    },
    {
      "epoch": 1.2842234399280281,
      "grad_norm": 8.991447448730469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2528066337108612,
      "logits/rejected": -0.22729669511318207,
      "logps/chosen": -541.852294921875,
      "logps/rejected": -582.2354736328125,
      "loss": 0.4793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30002516508102417,
      "rewards/margins": 1.0413657426834106,
      "rewards/rejected": -1.3413909673690796,
      "step": 9814
    },
    {
      "epoch": 1.28435429786538,
      "grad_norm": 7.575220584869385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37114015221595764,
      "logits/rejected": -0.29597803950309753,
      "logps/chosen": -346.1070861816406,
      "logps/rejected": -536.184326171875,
      "loss": 0.3894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6740764379501343,
      "rewards/margins": 1.2275893688201904,
      "rewards/rejected": -1.9016659259796143,
      "step": 9815
    },
    {
      "epoch": 1.2844851558027317,
      "grad_norm": 6.1198296546936035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08449015766382217,
      "logits/rejected": 0.19003205001354218,
      "logps/chosen": -483.38189697265625,
      "logps/rejected": -415.7773742675781,
      "loss": 0.2786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11124993860721588,
      "rewards/margins": 1.4780030250549316,
      "rewards/rejected": -1.5892529487609863,
      "step": 9816
    },
    {
      "epoch": 1.2846160137400835,
      "grad_norm": 6.402767181396484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05550553277134895,
      "logits/rejected": -0.05684925243258476,
      "logps/chosen": -409.07672119140625,
      "logps/rejected": -563.25634765625,
      "loss": 0.2523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4792371988296509,
      "rewards/margins": 1.9705370664596558,
      "rewards/rejected": -2.4497740268707275,
      "step": 9817
    },
    {
      "epoch": 1.2847468716774353,
      "grad_norm": 4.24233865737915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0765758603811264,
      "logits/rejected": -0.0754341185092926,
      "logps/chosen": -485.1623840332031,
      "logps/rejected": -301.15087890625,
      "loss": 0.1979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5792993307113647,
      "rewards/margins": 2.361682415008545,
      "rewards/rejected": -1.7823833227157593,
      "step": 9818
    },
    {
      "epoch": 1.284877729614787,
      "grad_norm": 7.897393226623535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07593438774347305,
      "logits/rejected": -0.3313753008842468,
      "logps/chosen": -472.8799133300781,
      "logps/rejected": -465.61871337890625,
      "loss": 0.2454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5730644464492798,
      "rewards/margins": 2.83467173576355,
      "rewards/rejected": -3.407736301422119,
      "step": 9819
    },
    {
      "epoch": 1.2850085875521386,
      "grad_norm": 11.08286190032959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.050374437123537064,
      "logits/rejected": -0.12950977683067322,
      "logps/chosen": -542.4361572265625,
      "logps/rejected": -509.19061279296875,
      "loss": 0.8597,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.6335588693618774,
      "rewards/margins": 0.19856643676757812,
      "rewards/rejected": -1.832125186920166,
      "step": 9820
    },
    {
      "epoch": 1.2851394454894904,
      "grad_norm": 11.246063232421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1453056037425995,
      "logits/rejected": -0.147343248128891,
      "logps/chosen": -420.87677001953125,
      "logps/rejected": -306.55401611328125,
      "loss": 0.8864,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.423633098602295,
      "rewards/margins": -0.2754312753677368,
      "rewards/rejected": -2.1482019424438477,
      "step": 9821
    },
    {
      "epoch": 1.2852703034268422,
      "grad_norm": 5.625641345977783,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007892973721027374,
      "logits/rejected": -0.024080872535705566,
      "logps/chosen": -518.7714233398438,
      "logps/rejected": -441.7153015136719,
      "loss": 0.3806,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3562406897544861,
      "rewards/margins": 1.1900798082351685,
      "rewards/rejected": -1.5463205575942993,
      "step": 9822
    },
    {
      "epoch": 1.285401161364194,
      "grad_norm": 9.353899955749512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5317130088806152,
      "logits/rejected": -0.5855597257614136,
      "logps/chosen": -466.5360107421875,
      "logps/rejected": -499.11309814453125,
      "loss": 0.5902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7752655148506165,
      "rewards/margins": 1.890641450881958,
      "rewards/rejected": -2.6659069061279297,
      "step": 9823
    },
    {
      "epoch": 1.2855320193015458,
      "grad_norm": 3.9963526725769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3266668915748596,
      "logits/rejected": -0.1434868723154068,
      "logps/chosen": -389.82000732421875,
      "logps/rejected": -431.0599060058594,
      "loss": 0.174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8902050852775574,
      "rewards/margins": 2.5245442390441895,
      "rewards/rejected": -3.4147491455078125,
      "step": 9824
    },
    {
      "epoch": 1.2856628772388974,
      "grad_norm": 10.960742950439453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3169909417629242,
      "logits/rejected": -0.37917661666870117,
      "logps/chosen": -359.0543518066406,
      "logps/rejected": -427.9239196777344,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3924127519130707,
      "rewards/margins": 0.9396645426750183,
      "rewards/rejected": -1.3320772647857666,
      "step": 9825
    },
    {
      "epoch": 1.2857937351762492,
      "grad_norm": 4.732715606689453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017617955803871155,
      "logits/rejected": 0.0801393985748291,
      "logps/chosen": -722.5369262695312,
      "logps/rejected": -524.990966796875,
      "loss": 0.2148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4785938560962677,
      "rewards/margins": 2.7023725509643555,
      "rewards/rejected": -2.22377872467041,
      "step": 9826
    },
    {
      "epoch": 1.285924593113601,
      "grad_norm": 12.240981101989746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13538071513175964,
      "logits/rejected": -0.1866982877254486,
      "logps/chosen": -389.5397644042969,
      "logps/rejected": -442.50341796875,
      "loss": 0.6834,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.315046787261963,
      "rewards/margins": 1.2912168502807617,
      "rewards/rejected": -2.6062636375427246,
      "step": 9827
    },
    {
      "epoch": 1.2860554510509528,
      "grad_norm": 4.400378704071045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11196498572826385,
      "logits/rejected": -0.31097549200057983,
      "logps/chosen": -529.9038696289062,
      "logps/rejected": -554.1640625,
      "loss": 0.1934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02862548828125,
      "rewards/margins": 2.2414584159851074,
      "rewards/rejected": -2.2128326892852783,
      "step": 9828
    },
    {
      "epoch": 1.2861863089883045,
      "grad_norm": 9.074962615966797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27575868368148804,
      "logits/rejected": -0.421873539686203,
      "logps/chosen": -364.8699035644531,
      "logps/rejected": -465.4080810546875,
      "loss": 0.5353,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7595360279083252,
      "rewards/margins": 1.2917492389678955,
      "rewards/rejected": -3.0512852668762207,
      "step": 9829
    },
    {
      "epoch": 1.2863171669256563,
      "grad_norm": 6.722438812255859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04609811305999756,
      "logits/rejected": -0.0242883563041687,
      "logps/chosen": -444.83831787109375,
      "logps/rejected": -374.8609313964844,
      "loss": 0.4664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7185497283935547,
      "rewards/margins": 1.610335350036621,
      "rewards/rejected": -2.328885078430176,
      "step": 9830
    },
    {
      "epoch": 1.2864480248630081,
      "grad_norm": 9.307849884033203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15698528289794922,
      "logits/rejected": -0.29049965739250183,
      "logps/chosen": -505.13226318359375,
      "logps/rejected": -408.7695617675781,
      "loss": 0.6378,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6438726186752319,
      "rewards/margins": 0.5541067123413086,
      "rewards/rejected": -1.1979793310165405,
      "step": 9831
    },
    {
      "epoch": 1.28657888280036,
      "grad_norm": 7.2191362380981445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29975369572639465,
      "logits/rejected": -0.35185137391090393,
      "logps/chosen": -356.1497802734375,
      "logps/rejected": -432.1845703125,
      "loss": 0.4867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3170087337493896,
      "rewards/margins": 0.6795023083686829,
      "rewards/rejected": -1.9965109825134277,
      "step": 9832
    },
    {
      "epoch": 1.2867097407377117,
      "grad_norm": 3.711940288543701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05313988775014877,
      "logits/rejected": -0.14895549416542053,
      "logps/chosen": -442.6683654785156,
      "logps/rejected": -439.0269470214844,
      "loss": 0.1465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6621354818344116,
      "rewards/margins": 2.792327404022217,
      "rewards/rejected": -2.1301918029785156,
      "step": 9833
    },
    {
      "epoch": 1.2868405986750635,
      "grad_norm": 5.934976100921631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2512548565864563,
      "logits/rejected": -0.33148229122161865,
      "logps/chosen": -402.8110656738281,
      "logps/rejected": -629.4083862304688,
      "loss": 0.392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0098991394042969,
      "rewards/margins": 1.4479279518127441,
      "rewards/rejected": -2.45782732963562,
      "step": 9834
    },
    {
      "epoch": 1.286971456612415,
      "grad_norm": 3.872063636779785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4211614429950714,
      "logits/rejected": -0.4007459282875061,
      "logps/chosen": -354.0785217285156,
      "logps/rejected": -352.1807556152344,
      "loss": 0.2298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.249267578125,
      "rewards/margins": 2.176509380340576,
      "rewards/rejected": -2.425776720046997,
      "step": 9835
    },
    {
      "epoch": 1.2871023145497669,
      "grad_norm": 7.7399139404296875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19887083768844604,
      "logits/rejected": 0.18724732100963593,
      "logps/chosen": -415.77764892578125,
      "logps/rejected": -440.9537353515625,
      "loss": 0.3363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4495478868484497,
      "rewards/margins": 1.7569875717163086,
      "rewards/rejected": -3.206535577774048,
      "step": 9836
    },
    {
      "epoch": 1.2872331724871187,
      "grad_norm": 10.444307327270508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2735269367694855,
      "logits/rejected": -0.24345801770687103,
      "logps/chosen": -389.9081726074219,
      "logps/rejected": -336.1170349121094,
      "loss": 0.6219,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.157211422920227,
      "rewards/margins": 0.5455469489097595,
      "rewards/rejected": -1.7027583122253418,
      "step": 9837
    },
    {
      "epoch": 1.2873640304244705,
      "grad_norm": 9.466181755065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07821249961853027,
      "logits/rejected": -0.29823410511016846,
      "logps/chosen": -512.156982421875,
      "logps/rejected": -457.4227600097656,
      "loss": 0.6659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0941663980484009,
      "rewards/margins": 0.5178394317626953,
      "rewards/rejected": -1.6120058298110962,
      "step": 9838
    },
    {
      "epoch": 1.2874948883618222,
      "grad_norm": 5.9904680252075195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01933872699737549,
      "logits/rejected": -0.09358017891645432,
      "logps/chosen": -404.3949890136719,
      "logps/rejected": -365.63189697265625,
      "loss": 0.1772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5479114651679993,
      "rewards/margins": 2.0021767616271973,
      "rewards/rejected": -2.550088405609131,
      "step": 9839
    },
    {
      "epoch": 1.287625746299174,
      "grad_norm": 8.33076000213623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1186564564704895,
      "logits/rejected": -0.07218378037214279,
      "logps/chosen": -580.2931518554688,
      "logps/rejected": -736.043701171875,
      "loss": 0.4072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007938027381896973,
      "rewards/margins": 1.9066953659057617,
      "rewards/rejected": -1.9146332740783691,
      "step": 9840
    },
    {
      "epoch": 1.2877566042365256,
      "grad_norm": 11.313570976257324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11531952023506165,
      "logits/rejected": -0.10785602033138275,
      "logps/chosen": -505.921142578125,
      "logps/rejected": -489.0947265625,
      "loss": 0.437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8452461361885071,
      "rewards/margins": 1.7621406316757202,
      "rewards/rejected": -2.607386589050293,
      "step": 9841
    },
    {
      "epoch": 1.2878874621738774,
      "grad_norm": 7.899341583251953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18591487407684326,
      "logits/rejected": -0.24774613976478577,
      "logps/chosen": -320.5989990234375,
      "logps/rejected": -319.5885009765625,
      "loss": 0.5165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8211254477500916,
      "rewards/margins": 0.5850416421890259,
      "rewards/rejected": -1.4061670303344727,
      "step": 9842
    },
    {
      "epoch": 1.2880183201112292,
      "grad_norm": 10.12334156036377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40502268075942993,
      "logits/rejected": -0.18177403509616852,
      "logps/chosen": -448.39434814453125,
      "logps/rejected": -412.07330322265625,
      "loss": 0.5092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5644626617431641,
      "rewards/margins": 1.625387191772461,
      "rewards/rejected": -2.189849853515625,
      "step": 9843
    },
    {
      "epoch": 1.288149178048581,
      "grad_norm": 6.438419818878174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3074151873588562,
      "logits/rejected": -0.33776170015335083,
      "logps/chosen": -400.2464599609375,
      "logps/rejected": -387.0919494628906,
      "loss": 0.4391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5192714333534241,
      "rewards/margins": 1.4300554990768433,
      "rewards/rejected": -1.9493268728256226,
      "step": 9844
    },
    {
      "epoch": 1.2882800359859328,
      "grad_norm": 7.267818450927734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21363475918769836,
      "logits/rejected": -0.09021714329719543,
      "logps/chosen": -473.3770446777344,
      "logps/rejected": -391.7772216796875,
      "loss": 0.3625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2015666961669922,
      "rewards/margins": 1.9099242687225342,
      "rewards/rejected": -2.1114909648895264,
      "step": 9845
    },
    {
      "epoch": 1.2884108939232846,
      "grad_norm": 10.788534164428711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12738294899463654,
      "logits/rejected": -0.2925843596458435,
      "logps/chosen": -630.185791015625,
      "logps/rejected": -507.8258056640625,
      "loss": 0.4638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3229999840259552,
      "rewards/margins": 1.160468578338623,
      "rewards/rejected": -1.4834685325622559,
      "step": 9846
    },
    {
      "epoch": 1.2885417518606364,
      "grad_norm": 8.797545433044434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3242732584476471,
      "logits/rejected": -0.4394932985305786,
      "logps/chosen": -481.50909423828125,
      "logps/rejected": -390.5106506347656,
      "loss": 0.4025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8932756185531616,
      "rewards/margins": 1.7890174388885498,
      "rewards/rejected": -2.68229341506958,
      "step": 9847
    },
    {
      "epoch": 1.2886726097979881,
      "grad_norm": 5.395547389984131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14535239338874817,
      "logits/rejected": -0.37843963503837585,
      "logps/chosen": -485.6166687011719,
      "logps/rejected": -440.92333984375,
      "loss": 0.2825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.366771936416626,
      "rewards/margins": 2.030890941619873,
      "rewards/rejected": -2.397662878036499,
      "step": 9848
    },
    {
      "epoch": 1.28880346773534,
      "grad_norm": 6.986486911773682,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3661384582519531,
      "logits/rejected": 0.2298477739095688,
      "logps/chosen": -548.3427734375,
      "logps/rejected": -678.6090087890625,
      "loss": 0.3549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6833980679512024,
      "rewards/margins": 1.827728509902954,
      "rewards/rejected": -2.5111265182495117,
      "step": 9849
    },
    {
      "epoch": 1.2889343256726915,
      "grad_norm": 4.597564220428467,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19029904901981354,
      "logits/rejected": -0.27032536268234253,
      "logps/chosen": -595.6322631835938,
      "logps/rejected": -569.486083984375,
      "loss": 0.1599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8359246850013733,
      "rewards/margins": 2.2704429626464844,
      "rewards/rejected": -3.106367826461792,
      "step": 9850
    },
    {
      "epoch": 1.2890651836100433,
      "grad_norm": 5.558115005493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04269421100616455,
      "logits/rejected": 0.1382618248462677,
      "logps/chosen": -636.6485595703125,
      "logps/rejected": -510.8224792480469,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14031413197517395,
      "rewards/margins": 1.5129855871200562,
      "rewards/rejected": -1.6532995700836182,
      "step": 9851
    },
    {
      "epoch": 1.289196041547395,
      "grad_norm": 10.944348335266113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19244328141212463,
      "logits/rejected": -0.26107239723205566,
      "logps/chosen": -467.4203796386719,
      "logps/rejected": -644.5280151367188,
      "loss": 0.595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.661980390548706,
      "rewards/margins": 0.9288493394851685,
      "rewards/rejected": -1.5908297300338745,
      "step": 9852
    },
    {
      "epoch": 1.2893268994847469,
      "grad_norm": 7.928609848022461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43060585856437683,
      "logits/rejected": -0.38472452759742737,
      "logps/chosen": -284.52972412109375,
      "logps/rejected": -321.0043029785156,
      "loss": 0.6032,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7299095392227173,
      "rewards/margins": 0.7626180648803711,
      "rewards/rejected": -2.492527484893799,
      "step": 9853
    },
    {
      "epoch": 1.2894577574220987,
      "grad_norm": 8.258145332336426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2866169810295105,
      "logits/rejected": -0.22148370742797852,
      "logps/chosen": -355.7330322265625,
      "logps/rejected": -348.6981201171875,
      "loss": 0.4019,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2844022512435913,
      "rewards/margins": 1.691031575202942,
      "rewards/rejected": -2.9754340648651123,
      "step": 9854
    },
    {
      "epoch": 1.2895886153594505,
      "grad_norm": 6.013566017150879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07844984531402588,
      "logits/rejected": -0.453132688999176,
      "logps/chosen": -708.7225952148438,
      "logps/rejected": -654.6257934570312,
      "loss": 0.2589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21291962265968323,
      "rewards/margins": 2.156193256378174,
      "rewards/rejected": -2.3691132068634033,
      "step": 9855
    },
    {
      "epoch": 1.289719473296802,
      "grad_norm": 7.44550895690918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33201178908348083,
      "logits/rejected": -0.22974297404289246,
      "logps/chosen": -274.8056640625,
      "logps/rejected": -423.3868103027344,
      "loss": 0.4442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4690238833427429,
      "rewards/margins": 0.7950050830841064,
      "rewards/rejected": -1.2640290260314941,
      "step": 9856
    },
    {
      "epoch": 1.2898503312341538,
      "grad_norm": 11.991202354431152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.401946485042572,
      "logits/rejected": -0.42988935112953186,
      "logps/chosen": -424.87261962890625,
      "logps/rejected": -528.446533203125,
      "loss": 0.8188,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9435657262802124,
      "rewards/margins": 0.2856670618057251,
      "rewards/rejected": -2.2292327880859375,
      "step": 9857
    },
    {
      "epoch": 1.2899811891715056,
      "grad_norm": 8.387840270996094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0894189327955246,
      "logits/rejected": 0.035868823528289795,
      "logps/chosen": -411.9640808105469,
      "logps/rejected": -386.9371643066406,
      "loss": 0.5075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7711483240127563,
      "rewards/margins": 1.2708795070648193,
      "rewards/rejected": -2.0420279502868652,
      "step": 9858
    },
    {
      "epoch": 1.2901120471088574,
      "grad_norm": 8.063737869262695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07876116037368774,
      "logits/rejected": 0.006084960885345936,
      "logps/chosen": -406.84490966796875,
      "logps/rejected": -422.81207275390625,
      "loss": 0.4236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7344673871994019,
      "rewards/margins": 1.206388235092163,
      "rewards/rejected": -1.940855622291565,
      "step": 9859
    },
    {
      "epoch": 1.2902429050462092,
      "grad_norm": 8.109724044799805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15056532621383667,
      "logits/rejected": 0.010203737765550613,
      "logps/chosen": -338.76123046875,
      "logps/rejected": -373.6895751953125,
      "loss": 0.469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8322197198867798,
      "rewards/margins": 1.4160256385803223,
      "rewards/rejected": -2.2482454776763916,
      "step": 9860
    },
    {
      "epoch": 1.290373762983561,
      "grad_norm": 3.65069580078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13581237196922302,
      "logits/rejected": -0.2666490077972412,
      "logps/chosen": -504.12969970703125,
      "logps/rejected": -474.42596435546875,
      "loss": 0.1698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6702160239219666,
      "rewards/margins": 2.1058526039123535,
      "rewards/rejected": -2.7760684490203857,
      "step": 9861
    },
    {
      "epoch": 1.2905046209209128,
      "grad_norm": 7.8179779052734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18405665457248688,
      "logits/rejected": -0.10704298317432404,
      "logps/chosen": -463.7757568359375,
      "logps/rejected": -515.1409912109375,
      "loss": 0.4589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029718022793531418,
      "rewards/margins": 1.3514668941497803,
      "rewards/rejected": -1.321748971939087,
      "step": 9862
    },
    {
      "epoch": 1.2906354788582646,
      "grad_norm": 3.4857254028320312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024554215371608734,
      "logits/rejected": -0.2508280575275421,
      "logps/chosen": -448.0384216308594,
      "logps/rejected": -455.2254943847656,
      "loss": 0.1364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05684584379196167,
      "rewards/margins": 2.5225400924682617,
      "rewards/rejected": -2.465693950653076,
      "step": 9863
    },
    {
      "epoch": 1.2907663367956164,
      "grad_norm": 11.276556968688965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4599941670894623,
      "logits/rejected": -0.7366860508918762,
      "logps/chosen": -477.95281982421875,
      "logps/rejected": -355.173828125,
      "loss": 0.5363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5394716262817383,
      "rewards/margins": 0.7481809854507446,
      "rewards/rejected": -2.2876527309417725,
      "step": 9864
    },
    {
      "epoch": 1.2908971947329682,
      "grad_norm": 8.74351692199707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10436513274908066,
      "logits/rejected": 0.12284477055072784,
      "logps/chosen": -388.04248046875,
      "logps/rejected": -410.2174072265625,
      "loss": 0.6519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7052814364433289,
      "rewards/margins": 0.5301845669746399,
      "rewards/rejected": -1.2354660034179688,
      "step": 9865
    },
    {
      "epoch": 1.2910280526703197,
      "grad_norm": 16.784822463989258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23895232379436493,
      "logits/rejected": -0.22641338407993317,
      "logps/chosen": -578.281982421875,
      "logps/rejected": -534.4338989257812,
      "loss": 0.6999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3871143162250519,
      "rewards/margins": 0.6663054823875427,
      "rewards/rejected": -1.0534199476242065,
      "step": 9866
    },
    {
      "epoch": 1.2911589106076715,
      "grad_norm": 9.138714790344238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18055644631385803,
      "logits/rejected": -0.05522748827934265,
      "logps/chosen": -334.646484375,
      "logps/rejected": -404.9073181152344,
      "loss": 0.4739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1417083740234375,
      "rewards/margins": 1.0440939664840698,
      "rewards/rejected": -2.185802459716797,
      "step": 9867
    },
    {
      "epoch": 1.2912897685450233,
      "grad_norm": 9.895682334899902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026181809604167938,
      "logits/rejected": -0.012445302680134773,
      "logps/chosen": -530.3604125976562,
      "logps/rejected": -618.9114990234375,
      "loss": 0.4862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1901731491088867,
      "rewards/margins": 1.5006513595581055,
      "rewards/rejected": -2.690824270248413,
      "step": 9868
    },
    {
      "epoch": 1.291420626482375,
      "grad_norm": 12.839714050292969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.306615948677063,
      "logits/rejected": -0.1372334212064743,
      "logps/chosen": -395.0098876953125,
      "logps/rejected": -367.4530944824219,
      "loss": 0.7122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3281421661376953,
      "rewards/margins": 0.8083160519599915,
      "rewards/rejected": -2.136458158493042,
      "step": 9869
    },
    {
      "epoch": 1.291551484419727,
      "grad_norm": 8.048202514648438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3361959159374237,
      "logits/rejected": -0.2654404640197754,
      "logps/chosen": -445.2108459472656,
      "logps/rejected": -467.4696960449219,
      "loss": 0.6011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5991026163101196,
      "rewards/margins": 0.7440086603164673,
      "rewards/rejected": -1.343111276626587,
      "step": 9870
    },
    {
      "epoch": 1.2916823423570785,
      "grad_norm": 2.5844998359680176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13442325592041016,
      "logits/rejected": -0.27175140380859375,
      "logps/chosen": -401.3948669433594,
      "logps/rejected": -486.6641540527344,
      "loss": 0.0932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22918303310871124,
      "rewards/margins": 2.889173984527588,
      "rewards/rejected": -2.6599907875061035,
      "step": 9871
    },
    {
      "epoch": 1.2918132002944303,
      "grad_norm": 5.866986274719238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044761113822460175,
      "logits/rejected": -0.37783461809158325,
      "logps/chosen": -765.6881103515625,
      "logps/rejected": -692.4231567382812,
      "loss": 0.1916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42476388812065125,
      "rewards/margins": 1.9681885242462158,
      "rewards/rejected": -2.3929524421691895,
      "step": 9872
    },
    {
      "epoch": 1.291944058231782,
      "grad_norm": 9.419051170349121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.124429851770401,
      "logits/rejected": -0.24409613013267517,
      "logps/chosen": -377.73565673828125,
      "logps/rejected": -425.0570983886719,
      "loss": 0.6567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8709551095962524,
      "rewards/margins": 1.2953314781188965,
      "rewards/rejected": -2.1662864685058594,
      "step": 9873
    },
    {
      "epoch": 1.2920749161691338,
      "grad_norm": 9.70779037475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2761164903640747,
      "logits/rejected": -0.29790109395980835,
      "logps/chosen": -565.4003295898438,
      "logps/rejected": -552.6109619140625,
      "loss": 0.5313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0250475406646729,
      "rewards/margins": 2.051748752593994,
      "rewards/rejected": -3.076796293258667,
      "step": 9874
    },
    {
      "epoch": 1.2922057741064856,
      "grad_norm": 11.662261009216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046663396060466766,
      "logits/rejected": -0.28580331802368164,
      "logps/chosen": -414.6535339355469,
      "logps/rejected": -463.61126708984375,
      "loss": 0.6752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2520380020141602,
      "rewards/margins": 0.9467706680297852,
      "rewards/rejected": -2.1988086700439453,
      "step": 9875
    },
    {
      "epoch": 1.2923366320438374,
      "grad_norm": 6.509909152984619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4341416358947754,
      "logits/rejected": -0.43355199694633484,
      "logps/chosen": -447.3868408203125,
      "logps/rejected": -476.36651611328125,
      "loss": 0.4763,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9979828596115112,
      "rewards/margins": 0.8450407385826111,
      "rewards/rejected": -1.843023419380188,
      "step": 9876
    },
    {
      "epoch": 1.2924674899811892,
      "grad_norm": 10.604872703552246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04887596517801285,
      "logits/rejected": 0.12220795452594757,
      "logps/chosen": -352.2203674316406,
      "logps/rejected": -344.0606994628906,
      "loss": 0.5974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8492590188980103,
      "rewards/margins": 0.9250472784042358,
      "rewards/rejected": -1.774306297302246,
      "step": 9877
    },
    {
      "epoch": 1.292598347918541,
      "grad_norm": 8.876498222351074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0651908665895462,
      "logits/rejected": -0.23333102464675903,
      "logps/chosen": -366.43133544921875,
      "logps/rejected": -431.88922119140625,
      "loss": 0.4989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0177291631698608,
      "rewards/margins": 1.087360143661499,
      "rewards/rejected": -2.1050894260406494,
      "step": 9878
    },
    {
      "epoch": 1.2927292058558928,
      "grad_norm": 11.741181373596191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2590676248073578,
      "logits/rejected": -0.07011155039072037,
      "logps/chosen": -375.24517822265625,
      "logps/rejected": -302.872314453125,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3957186937332153,
      "rewards/margins": 0.7745370268821716,
      "rewards/rejected": -2.170255661010742,
      "step": 9879
    },
    {
      "epoch": 1.2928600637932446,
      "grad_norm": 9.165240287780762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19138577580451965,
      "logits/rejected": -0.3426959812641144,
      "logps/chosen": -635.7157592773438,
      "logps/rejected": -460.269287109375,
      "loss": 0.3579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2693607807159424,
      "rewards/margins": 1.1929243803024292,
      "rewards/rejected": -2.462285280227661,
      "step": 9880
    },
    {
      "epoch": 1.2929909217305962,
      "grad_norm": 8.139434814453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19461911916732788,
      "logits/rejected": -0.2895223796367645,
      "logps/chosen": -554.2315063476562,
      "logps/rejected": -426.8564453125,
      "loss": 0.4437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6180006265640259,
      "rewards/margins": 1.8928221464157104,
      "rewards/rejected": -2.5108227729797363,
      "step": 9881
    },
    {
      "epoch": 1.293121779667948,
      "grad_norm": 6.633854389190674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10081423074007034,
      "logits/rejected": -0.15193989872932434,
      "logps/chosen": -501.3763732910156,
      "logps/rejected": -499.53009033203125,
      "loss": 0.2896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2383397817611694,
      "rewards/margins": 1.2786351442337036,
      "rewards/rejected": -2.516974925994873,
      "step": 9882
    },
    {
      "epoch": 1.2932526376052997,
      "grad_norm": 7.358889102935791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17280882596969604,
      "logits/rejected": 0.08008700609207153,
      "logps/chosen": -310.7253112792969,
      "logps/rejected": -451.93255615234375,
      "loss": 0.3287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1367566585540771,
      "rewards/margins": 2.0153348445892334,
      "rewards/rejected": -3.1520912647247314,
      "step": 9883
    },
    {
      "epoch": 1.2933834955426515,
      "grad_norm": 8.123823165893555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18649256229400635,
      "logits/rejected": 0.13416031002998352,
      "logps/chosen": -473.3731384277344,
      "logps/rejected": -410.0716552734375,
      "loss": 0.5754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5545482635498047,
      "rewards/margins": 0.5442491173744202,
      "rewards/rejected": -1.0987974405288696,
      "step": 9884
    },
    {
      "epoch": 1.2935143534800033,
      "grad_norm": 6.113284587860107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43266433477401733,
      "logits/rejected": -0.11253819614648819,
      "logps/chosen": -558.001708984375,
      "logps/rejected": -480.6732177734375,
      "loss": 0.223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.618155300617218,
      "rewards/margins": 1.9815576076507568,
      "rewards/rejected": -2.59971284866333,
      "step": 9885
    },
    {
      "epoch": 1.2936452114173551,
      "grad_norm": 8.650554656982422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23769646883010864,
      "logits/rejected": -0.05309157818555832,
      "logps/chosen": -440.91217041015625,
      "logps/rejected": -506.52545166015625,
      "loss": 0.4449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0721355676651,
      "rewards/margins": 0.7554094791412354,
      "rewards/rejected": -1.8275450468063354,
      "step": 9886
    },
    {
      "epoch": 1.2937760693547067,
      "grad_norm": 5.192155838012695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4039267301559448,
      "logits/rejected": -0.5326249599456787,
      "logps/chosen": -381.919189453125,
      "logps/rejected": -362.40289306640625,
      "loss": 0.364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.08515202999115,
      "rewards/margins": 1.2903424501419067,
      "rewards/rejected": -2.3754944801330566,
      "step": 9887
    },
    {
      "epoch": 1.2939069272920585,
      "grad_norm": 7.651966571807861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17929062247276306,
      "logits/rejected": -0.21924076974391937,
      "logps/chosen": -481.04705810546875,
      "logps/rejected": -405.8337097167969,
      "loss": 0.3825,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35561445355415344,
      "rewards/margins": 1.5425596237182617,
      "rewards/rejected": -1.8981741666793823,
      "step": 9888
    },
    {
      "epoch": 1.2940377852294103,
      "grad_norm": 8.405327796936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2587326765060425,
      "logits/rejected": -0.2686111330986023,
      "logps/chosen": -513.251220703125,
      "logps/rejected": -515.6344604492188,
      "loss": 0.6519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1132876873016357,
      "rewards/margins": 1.435711145401001,
      "rewards/rejected": -2.5489988327026367,
      "step": 9889
    },
    {
      "epoch": 1.294168643166762,
      "grad_norm": 10.704374313354492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0821709930896759,
      "logits/rejected": -0.07832445204257965,
      "logps/chosen": -463.20703125,
      "logps/rejected": -375.9389953613281,
      "loss": 0.6724,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5163614749908447,
      "rewards/margins": 0.38992422819137573,
      "rewards/rejected": -1.9062858819961548,
      "step": 9890
    },
    {
      "epoch": 1.2942995011041138,
      "grad_norm": 7.522241115570068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04880870506167412,
      "logits/rejected": -0.17518119513988495,
      "logps/chosen": -694.0780029296875,
      "logps/rejected": -527.010009765625,
      "loss": 0.3286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29287856817245483,
      "rewards/margins": 1.5000357627868652,
      "rewards/rejected": -1.7929145097732544,
      "step": 9891
    },
    {
      "epoch": 1.2944303590414656,
      "grad_norm": 8.895988464355469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22290052473545074,
      "logits/rejected": -0.32899948954582214,
      "logps/chosen": -360.639892578125,
      "logps/rejected": -531.9661254882812,
      "loss": 0.385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2742270231246948,
      "rewards/margins": 1.1167492866516113,
      "rewards/rejected": -2.3909764289855957,
      "step": 9892
    },
    {
      "epoch": 1.2945612169788174,
      "grad_norm": 11.220585823059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3160606920719147,
      "logits/rejected": -0.38213494420051575,
      "logps/chosen": -429.6459655761719,
      "logps/rejected": -549.6258544921875,
      "loss": 0.5042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6585262417793274,
      "rewards/margins": 1.5366830825805664,
      "rewards/rejected": -2.195209264755249,
      "step": 9893
    },
    {
      "epoch": 1.2946920749161692,
      "grad_norm": 10.561187744140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3379441797733307,
      "logits/rejected": -0.4777368903160095,
      "logps/chosen": -406.4120178222656,
      "logps/rejected": -416.05987548828125,
      "loss": 0.5654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0629345178604126,
      "rewards/margins": 1.1362664699554443,
      "rewards/rejected": -2.1992006301879883,
      "step": 9894
    },
    {
      "epoch": 1.294822932853521,
      "grad_norm": 8.334084510803223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36182552576065063,
      "logits/rejected": -0.1946205198764801,
      "logps/chosen": -492.720703125,
      "logps/rejected": -458.18206787109375,
      "loss": 0.4195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27453577518463135,
      "rewards/margins": 1.6442843675613403,
      "rewards/rejected": -1.9188202619552612,
      "step": 9895
    },
    {
      "epoch": 1.2949537907908726,
      "grad_norm": 7.290369987487793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2147025465965271,
      "logits/rejected": -0.37497565150260925,
      "logps/chosen": -419.6727294921875,
      "logps/rejected": -404.38836669921875,
      "loss": 0.4301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8948156237602234,
      "rewards/margins": 1.2831416130065918,
      "rewards/rejected": -2.17795729637146,
      "step": 9896
    },
    {
      "epoch": 1.2950846487282244,
      "grad_norm": 6.0017900466918945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2202230542898178,
      "logits/rejected": -0.45510995388031006,
      "logps/chosen": -392.7929992675781,
      "logps/rejected": -389.3072509765625,
      "loss": 0.3424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9287697076797485,
      "rewards/margins": 1.7406010627746582,
      "rewards/rejected": -2.669370651245117,
      "step": 9897
    },
    {
      "epoch": 1.2952155066655762,
      "grad_norm": 8.86137866973877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02027866244316101,
      "logits/rejected": -0.06785528361797333,
      "logps/chosen": -616.47998046875,
      "logps/rejected": -467.83984375,
      "loss": 0.4171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11956064403057098,
      "rewards/margins": 1.4605555534362793,
      "rewards/rejected": -1.5801162719726562,
      "step": 9898
    },
    {
      "epoch": 1.295346364602928,
      "grad_norm": 12.825004577636719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33006882667541504,
      "logits/rejected": -0.35524341464042664,
      "logps/chosen": -501.27593994140625,
      "logps/rejected": -492.5566711425781,
      "loss": 0.7895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.364624261856079,
      "rewards/margins": 0.6285111904144287,
      "rewards/rejected": -1.9931353330612183,
      "step": 9899
    },
    {
      "epoch": 1.2954772225402797,
      "grad_norm": 4.80153226852417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04425952583551407,
      "logits/rejected": -0.347190260887146,
      "logps/chosen": -599.8187255859375,
      "logps/rejected": -387.365966796875,
      "loss": 0.1772,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5178686380386353,
      "rewards/margins": 2.5138838291168213,
      "rewards/rejected": -1.9960153102874756,
      "step": 9900
    },
    {
      "epoch": 1.2956080804776315,
      "grad_norm": 8.229318618774414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0017643384635448456,
      "logits/rejected": -0.06305176764726639,
      "logps/chosen": -445.8316650390625,
      "logps/rejected": -463.55548095703125,
      "loss": 0.5934,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14907300472259521,
      "rewards/margins": 0.7455585598945618,
      "rewards/rejected": -0.5964854955673218,
      "step": 9901
    },
    {
      "epoch": 1.295738938414983,
      "grad_norm": 8.560433387756348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10104880481958389,
      "logits/rejected": -0.37100598216056824,
      "logps/chosen": -605.3092041015625,
      "logps/rejected": -368.9429626464844,
      "loss": 0.3857,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.603079080581665,
      "rewards/margins": 1.8256930112838745,
      "rewards/rejected": -2.42877197265625,
      "step": 9902
    },
    {
      "epoch": 1.295869796352335,
      "grad_norm": 12.27332878112793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3194187879562378,
      "logits/rejected": -0.2469043880701065,
      "logps/chosen": -421.2148742675781,
      "logps/rejected": -583.1395874023438,
      "loss": 0.7321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9255378842353821,
      "rewards/margins": 1.2046864032745361,
      "rewards/rejected": -2.1302244663238525,
      "step": 9903
    },
    {
      "epoch": 1.2960006542896867,
      "grad_norm": 8.421513557434082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14964675903320312,
      "logits/rejected": -0.21187502145767212,
      "logps/chosen": -526.8718872070312,
      "logps/rejected": -413.842041015625,
      "loss": 0.4989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.81636381149292,
      "rewards/margins": 0.7551087141036987,
      "rewards/rejected": -2.571472644805908,
      "step": 9904
    },
    {
      "epoch": 1.2961315122270385,
      "grad_norm": 9.027405738830566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15081776678562164,
      "logits/rejected": -0.013356760144233704,
      "logps/chosen": -508.4443664550781,
      "logps/rejected": -446.65240478515625,
      "loss": 0.4335,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49605029821395874,
      "rewards/margins": 1.5673381090164185,
      "rewards/rejected": -2.0633883476257324,
      "step": 9905
    },
    {
      "epoch": 1.2962623701643903,
      "grad_norm": 4.840348720550537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2625271677970886,
      "logits/rejected": -0.18598000705242157,
      "logps/chosen": -441.646240234375,
      "logps/rejected": -522.889892578125,
      "loss": 0.1676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9714369177818298,
      "rewards/margins": 2.296967029571533,
      "rewards/rejected": -3.2684037685394287,
      "step": 9906
    },
    {
      "epoch": 1.296393228101742,
      "grad_norm": 6.976568222045898,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13018590211868286,
      "logits/rejected": 0.09271013736724854,
      "logps/chosen": -332.63482666015625,
      "logps/rejected": -409.0367126464844,
      "loss": 0.3558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.232377052307129,
      "rewards/margins": 1.528010368347168,
      "rewards/rejected": -2.760387659072876,
      "step": 9907
    },
    {
      "epoch": 1.2965240860390939,
      "grad_norm": 10.25620174407959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18204998970031738,
      "logits/rejected": -0.04456665739417076,
      "logps/chosen": -358.6162414550781,
      "logps/rejected": -349.70587158203125,
      "loss": 0.5434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4085843563079834,
      "rewards/margins": 1.0832264423370361,
      "rewards/rejected": -2.4918110370635986,
      "step": 9908
    },
    {
      "epoch": 1.2966549439764457,
      "grad_norm": 5.048463344573975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2192336618900299,
      "logits/rejected": -0.21657386422157288,
      "logps/chosen": -391.4393005371094,
      "logps/rejected": -341.81787109375,
      "loss": 0.369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0381021574139595,
      "rewards/margins": 1.8076536655426025,
      "rewards/rejected": -1.8457555770874023,
      "step": 9909
    },
    {
      "epoch": 1.2967858019137974,
      "grad_norm": 10.459092140197754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3884732723236084,
      "logits/rejected": -0.554850697517395,
      "logps/chosen": -387.11029052734375,
      "logps/rejected": -450.23236083984375,
      "loss": 0.5474,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3623050451278687,
      "rewards/margins": 0.5739046931266785,
      "rewards/rejected": -1.936209797859192,
      "step": 9910
    },
    {
      "epoch": 1.2969166598511492,
      "grad_norm": 7.895875930786133,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04014323279261589,
      "logits/rejected": -0.2053477019071579,
      "logps/chosen": -343.67169189453125,
      "logps/rejected": -538.010986328125,
      "loss": 0.559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0877021551132202,
      "rewards/margins": 1.7706937789916992,
      "rewards/rejected": -2.85839581489563,
      "step": 9911
    },
    {
      "epoch": 1.2970475177885008,
      "grad_norm": 6.675788402557373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15429480373859406,
      "logits/rejected": -0.23726657032966614,
      "logps/chosen": -381.45013427734375,
      "logps/rejected": -345.8568115234375,
      "loss": 0.4938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4695484638214111,
      "rewards/margins": 1.076220989227295,
      "rewards/rejected": -2.545769453048706,
      "step": 9912
    },
    {
      "epoch": 1.2971783757258526,
      "grad_norm": 9.702651023864746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018365703523159027,
      "logits/rejected": -0.24318167567253113,
      "logps/chosen": -563.5943603515625,
      "logps/rejected": -424.0894775390625,
      "loss": 0.5748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0889531373977661,
      "rewards/margins": 0.6097086668014526,
      "rewards/rejected": -1.6986619234085083,
      "step": 9913
    },
    {
      "epoch": 1.2973092336632044,
      "grad_norm": 8.748706817626953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2826214134693146,
      "logits/rejected": -0.22236396372318268,
      "logps/chosen": -420.896728515625,
      "logps/rejected": -442.5042419433594,
      "loss": 0.4219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44442495703697205,
      "rewards/margins": 1.3239513635635376,
      "rewards/rejected": -1.768376350402832,
      "step": 9914
    },
    {
      "epoch": 1.2974400916005562,
      "grad_norm": 11.21397876739502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3906719386577606,
      "logits/rejected": -0.36810874938964844,
      "logps/chosen": -543.3648071289062,
      "logps/rejected": -546.15185546875,
      "loss": 0.5878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2997339367866516,
      "rewards/margins": 1.7010046243667603,
      "rewards/rejected": -2.0007386207580566,
      "step": 9915
    },
    {
      "epoch": 1.297570949537908,
      "grad_norm": 14.345131874084473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09892572462558746,
      "logits/rejected": 0.050214942544698715,
      "logps/chosen": -631.3057250976562,
      "logps/rejected": -416.4304504394531,
      "loss": 0.8936,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1668310165405273,
      "rewards/margins": 0.26883697509765625,
      "rewards/rejected": -1.4356679916381836,
      "step": 9916
    },
    {
      "epoch": 1.2977018074752595,
      "grad_norm": 10.125096321105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004686150699853897,
      "logits/rejected": -0.10326869785785675,
      "logps/chosen": -581.1548461914062,
      "logps/rejected": -543.0531005859375,
      "loss": 0.5516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7618169784545898,
      "rewards/margins": 0.7467092871665955,
      "rewards/rejected": -1.5085262060165405,
      "step": 9917
    },
    {
      "epoch": 1.2978326654126113,
      "grad_norm": 5.367119789123535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02869599312543869,
      "logits/rejected": -0.31319907307624817,
      "logps/chosen": -461.6134338378906,
      "logps/rejected": -407.03924560546875,
      "loss": 0.2445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3226849138736725,
      "rewards/margins": 1.3681870698928833,
      "rewards/rejected": -1.690872073173523,
      "step": 9918
    },
    {
      "epoch": 1.2979635233499631,
      "grad_norm": 4.860537528991699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1636948585510254,
      "logits/rejected": -0.046301327645778656,
      "logps/chosen": -586.0359497070312,
      "logps/rejected": -439.1052551269531,
      "loss": 0.2307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7139860987663269,
      "rewards/margins": 1.7775800228118896,
      "rewards/rejected": -2.4915664196014404,
      "step": 9919
    },
    {
      "epoch": 1.298094381287315,
      "grad_norm": 9.204599380493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11039102077484131,
      "logits/rejected": -0.11722102761268616,
      "logps/chosen": -493.14581298828125,
      "logps/rejected": -589.5909423828125,
      "loss": 0.5313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7501543760299683,
      "rewards/margins": 0.8733603954315186,
      "rewards/rejected": -1.6235147714614868,
      "step": 9920
    },
    {
      "epoch": 1.2982252392246667,
      "grad_norm": 14.04240894317627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2070610523223877,
      "logits/rejected": -0.3410561680793762,
      "logps/chosen": -597.8421630859375,
      "logps/rejected": -473.70831298828125,
      "loss": 1.0136,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.685072660446167,
      "rewards/margins": 0.2525416910648346,
      "rewards/rejected": -1.9376143217086792,
      "step": 9921
    },
    {
      "epoch": 1.2983560971620185,
      "grad_norm": 9.136260986328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08476629108190536,
      "logits/rejected": -0.014950618147850037,
      "logps/chosen": -371.3270263671875,
      "logps/rejected": -402.4347839355469,
      "loss": 0.6974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8759145736694336,
      "rewards/margins": 0.2941072881221771,
      "rewards/rejected": -1.170021891593933,
      "step": 9922
    },
    {
      "epoch": 1.2984869550993703,
      "grad_norm": 9.528246879577637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16205669939517975,
      "logits/rejected": 0.1275799721479416,
      "logps/chosen": -606.8375854492188,
      "logps/rejected": -552.9761352539062,
      "loss": 0.6674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2117509841918945,
      "rewards/margins": 0.7238839864730835,
      "rewards/rejected": -1.935634970664978,
      "step": 9923
    },
    {
      "epoch": 1.298617813036722,
      "grad_norm": 9.314367294311523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10841459035873413,
      "logits/rejected": 0.01979963108897209,
      "logps/chosen": -531.5310668945312,
      "logps/rejected": -543.291748046875,
      "loss": 0.6572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.695044755935669,
      "rewards/margins": 0.767542839050293,
      "rewards/rejected": -1.4625874757766724,
      "step": 9924
    },
    {
      "epoch": 1.2987486709740739,
      "grad_norm": 5.428823947906494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14667780697345734,
      "logits/rejected": -0.32359081506729126,
      "logps/chosen": -503.87554931640625,
      "logps/rejected": -396.791748046875,
      "loss": 0.2696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4029128551483154,
      "rewards/margins": 1.3430213928222656,
      "rewards/rejected": -2.74593448638916,
      "step": 9925
    },
    {
      "epoch": 1.2988795289114257,
      "grad_norm": 9.277311325073242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003149699419736862,
      "logits/rejected": -0.21944735944271088,
      "logps/chosen": -566.6370849609375,
      "logps/rejected": -397.338623046875,
      "loss": 0.534,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9305075407028198,
      "rewards/margins": 0.62176114320755,
      "rewards/rejected": -1.5522687435150146,
      "step": 9926
    },
    {
      "epoch": 1.2990103868487772,
      "grad_norm": 6.61799955368042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19415079057216644,
      "logits/rejected": 0.04306532442569733,
      "logps/chosen": -493.96356201171875,
      "logps/rejected": -548.2875366210938,
      "loss": 0.2554,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.041200242936611176,
      "rewards/margins": 1.859476923942566,
      "rewards/rejected": -1.90067720413208,
      "step": 9927
    },
    {
      "epoch": 1.299141244786129,
      "grad_norm": 12.577582359313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11665209382772446,
      "logits/rejected": -0.15897352993488312,
      "logps/chosen": -458.063232421875,
      "logps/rejected": -351.78448486328125,
      "loss": 0.5848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2059011459350586,
      "rewards/margins": 0.6430016160011292,
      "rewards/rejected": -2.848902940750122,
      "step": 9928
    },
    {
      "epoch": 1.2992721027234808,
      "grad_norm": 11.267712593078613,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24029210209846497,
      "logits/rejected": 0.06761413812637329,
      "logps/chosen": -331.3659973144531,
      "logps/rejected": -406.2413330078125,
      "loss": 0.6954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8082304000854492,
      "rewards/margins": 0.6329303979873657,
      "rewards/rejected": -1.441160798072815,
      "step": 9929
    },
    {
      "epoch": 1.2994029606608326,
      "grad_norm": 6.428918361663818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21743015944957733,
      "logits/rejected": -0.3995375335216522,
      "logps/chosen": -496.03839111328125,
      "logps/rejected": -513.0328979492188,
      "loss": 0.3136,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47815126180648804,
      "rewards/margins": 1.8318711519241333,
      "rewards/rejected": -2.3100225925445557,
      "step": 9930
    },
    {
      "epoch": 1.2995338185981844,
      "grad_norm": 14.03072452545166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39887529611587524,
      "logits/rejected": -0.47718262672424316,
      "logps/chosen": -570.9989624023438,
      "logps/rejected": -566.1224975585938,
      "loss": 0.5174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7183271646499634,
      "rewards/margins": 0.7873921394348145,
      "rewards/rejected": -1.5057191848754883,
      "step": 9931
    },
    {
      "epoch": 1.2996646765355362,
      "grad_norm": 9.869617462158203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06505852937698364,
      "logits/rejected": -0.1849346160888672,
      "logps/chosen": -547.2845458984375,
      "logps/rejected": -428.5445861816406,
      "loss": 0.5021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16459272801876068,
      "rewards/margins": 0.9150055050849915,
      "rewards/rejected": -1.0795981884002686,
      "step": 9932
    },
    {
      "epoch": 1.2997955344728878,
      "grad_norm": 10.745954513549805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3088679611682892,
      "logits/rejected": -0.16666290163993835,
      "logps/chosen": -437.87091064453125,
      "logps/rejected": -419.0175476074219,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6836267709732056,
      "rewards/margins": 0.38968491554260254,
      "rewards/rejected": -1.073311686515808,
      "step": 9933
    },
    {
      "epoch": 1.2999263924102396,
      "grad_norm": 10.420008659362793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36777472496032715,
      "logits/rejected": -0.19983085989952087,
      "logps/chosen": -392.9930114746094,
      "logps/rejected": -369.95526123046875,
      "loss": 0.6262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7858186960220337,
      "rewards/margins": 0.7516170740127563,
      "rewards/rejected": -1.53743577003479,
      "step": 9934
    },
    {
      "epoch": 1.3000572503475913,
      "grad_norm": 5.938016891479492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13140025734901428,
      "logits/rejected": -0.20009878277778625,
      "logps/chosen": -552.3396606445312,
      "logps/rejected": -506.8473205566406,
      "loss": 0.2716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41052645444869995,
      "rewards/margins": 1.7370731830596924,
      "rewards/rejected": -2.147599697113037,
      "step": 9935
    },
    {
      "epoch": 1.3001881082849431,
      "grad_norm": 10.38205623626709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16138611733913422,
      "logits/rejected": -0.3231467008590698,
      "logps/chosen": -307.08251953125,
      "logps/rejected": -482.5704650878906,
      "loss": 1.0847,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6811392307281494,
      "rewards/margins": 0.47745802998542786,
      "rewards/rejected": -2.158597230911255,
      "step": 9936
    },
    {
      "epoch": 1.300318966222295,
      "grad_norm": 5.864096641540527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.142353355884552,
      "logits/rejected": -0.21247287094593048,
      "logps/chosen": -570.2739868164062,
      "logps/rejected": -414.0135498046875,
      "loss": 0.3961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.837658166885376,
      "rewards/margins": 1.5785013437271118,
      "rewards/rejected": -2.4161596298217773,
      "step": 9937
    },
    {
      "epoch": 1.3004498241596467,
      "grad_norm": 5.36529016494751,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05867624655365944,
      "logits/rejected": -0.08437967300415039,
      "logps/chosen": -560.589599609375,
      "logps/rejected": -477.5960998535156,
      "loss": 0.2681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1853588819503784,
      "rewards/margins": 1.6926748752593994,
      "rewards/rejected": -2.8780336380004883,
      "step": 9938
    },
    {
      "epoch": 1.3005806820969985,
      "grad_norm": 6.990251541137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1167794018983841,
      "logits/rejected": -0.2365088313817978,
      "logps/chosen": -649.3846435546875,
      "logps/rejected": -469.83160400390625,
      "loss": 0.3305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.241460919380188,
      "rewards/margins": 1.443356990814209,
      "rewards/rejected": -2.6848175525665283,
      "step": 9939
    },
    {
      "epoch": 1.3007115400343503,
      "grad_norm": 8.108199119567871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04539933428168297,
      "logits/rejected": -0.04738975688815117,
      "logps/chosen": -481.781982421875,
      "logps/rejected": -498.61932373046875,
      "loss": 0.5728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17696820199489594,
      "rewards/margins": 0.7358316779136658,
      "rewards/rejected": -0.5588635206222534,
      "step": 9940
    },
    {
      "epoch": 1.300842397971702,
      "grad_norm": 12.508464813232422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19154280424118042,
      "logits/rejected": -0.16281622648239136,
      "logps/chosen": -399.48687744140625,
      "logps/rejected": -543.8948364257812,
      "loss": 0.8104,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.641503930091858,
      "rewards/margins": 0.1626487523317337,
      "rewards/rejected": -1.8041527271270752,
      "step": 9941
    },
    {
      "epoch": 1.3009732559090537,
      "grad_norm": 9.264388084411621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1016479879617691,
      "logits/rejected": -0.1754145324230194,
      "logps/chosen": -610.4154052734375,
      "logps/rejected": -490.5478210449219,
      "loss": 0.4831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0657925605773926,
      "rewards/margins": 0.7353862524032593,
      "rewards/rejected": -2.8011786937713623,
      "step": 9942
    },
    {
      "epoch": 1.3011041138464055,
      "grad_norm": 9.38618278503418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22784985601902008,
      "logits/rejected": -0.3369295597076416,
      "logps/chosen": -393.199462890625,
      "logps/rejected": -575.1942138671875,
      "loss": 0.5135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.312369465827942,
      "rewards/margins": 0.9625930190086365,
      "rewards/rejected": -2.2749624252319336,
      "step": 9943
    },
    {
      "epoch": 1.3012349717837572,
      "grad_norm": 7.874614238739014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09030124545097351,
      "logits/rejected": -0.10307591408491135,
      "logps/chosen": -426.03350830078125,
      "logps/rejected": -519.2387084960938,
      "loss": 0.5642,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7882621884346008,
      "rewards/margins": 1.2987290620803833,
      "rewards/rejected": -2.08699107170105,
      "step": 9944
    },
    {
      "epoch": 1.301365829721109,
      "grad_norm": 7.701801776885986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03021205961704254,
      "logits/rejected": -0.18585801124572754,
      "logps/chosen": -525.52001953125,
      "logps/rejected": -559.1395874023438,
      "loss": 0.3433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2940632104873657,
      "rewards/margins": 1.4484915733337402,
      "rewards/rejected": -2.7425549030303955,
      "step": 9945
    },
    {
      "epoch": 1.3014966876584608,
      "grad_norm": 12.371406555175781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002505322452634573,
      "logits/rejected": 0.0006979778409004211,
      "logps/chosen": -617.1581420898438,
      "logps/rejected": -633.3187255859375,
      "loss": 0.6659,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4990333914756775,
      "rewards/margins": 0.9123144149780273,
      "rewards/rejected": -1.41134774684906,
      "step": 9946
    },
    {
      "epoch": 1.3016275455958126,
      "grad_norm": 9.646364212036133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14853085577487946,
      "logits/rejected": -0.4930228292942047,
      "logps/chosen": -405.407470703125,
      "logps/rejected": -378.61395263671875,
      "loss": 0.4877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.267223834991455,
      "rewards/margins": 0.7921469211578369,
      "rewards/rejected": -2.059370994567871,
      "step": 9947
    },
    {
      "epoch": 1.3017584035331642,
      "grad_norm": 7.366940021514893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23608970642089844,
      "logits/rejected": -0.2046479880809784,
      "logps/chosen": -502.9857482910156,
      "logps/rejected": -425.6718444824219,
      "loss": 0.56,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6964723467826843,
      "rewards/margins": 0.8160648345947266,
      "rewards/rejected": -1.5125371217727661,
      "step": 9948
    },
    {
      "epoch": 1.301889261470516,
      "grad_norm": 9.867847442626953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2672954499721527,
      "logits/rejected": -0.2820265591144562,
      "logps/chosen": -357.43701171875,
      "logps/rejected": -357.534912109375,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3500893115997314,
      "rewards/margins": 0.8993389010429382,
      "rewards/rejected": -2.2494285106658936,
      "step": 9949
    },
    {
      "epoch": 1.3020201194078678,
      "grad_norm": 14.170600891113281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29718267917633057,
      "logits/rejected": -0.29895347356796265,
      "logps/chosen": -458.416748046875,
      "logps/rejected": -638.5448608398438,
      "loss": 0.7444,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9064256548881531,
      "rewards/margins": 0.24701666831970215,
      "rewards/rejected": -1.1534423828125,
      "step": 9950
    },
    {
      "epoch": 1.3021509773452196,
      "grad_norm": 5.798094749450684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09982942044734955,
      "logits/rejected": -0.176747128367424,
      "logps/chosen": -356.1224365234375,
      "logps/rejected": -340.1246643066406,
      "loss": 0.3441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0466487407684326,
      "rewards/margins": 1.3911410570144653,
      "rewards/rejected": -2.4377894401550293,
      "step": 9951
    },
    {
      "epoch": 1.3022818352825714,
      "grad_norm": 11.5185546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35587236285209656,
      "logits/rejected": -0.32308846712112427,
      "logps/chosen": -573.8837280273438,
      "logps/rejected": -527.6287841796875,
      "loss": 0.6577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.649505376815796,
      "rewards/margins": 1.7476786375045776,
      "rewards/rejected": -3.397183895111084,
      "step": 9952
    },
    {
      "epoch": 1.3024126932199231,
      "grad_norm": 7.390161991119385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.349341481924057,
      "logits/rejected": -0.3195788562297821,
      "logps/chosen": -533.9370727539062,
      "logps/rejected": -506.91888427734375,
      "loss": 0.4032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8054403066635132,
      "rewards/margins": 1.9514119625091553,
      "rewards/rejected": -2.756852388381958,
      "step": 9953
    },
    {
      "epoch": 1.302543551157275,
      "grad_norm": 7.398239612579346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3725864589214325,
      "logits/rejected": -0.49227577447891235,
      "logps/chosen": -535.1688232421875,
      "logps/rejected": -576.9479370117188,
      "loss": 0.3257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.91525799036026,
      "rewards/margins": 1.8522083759307861,
      "rewards/rejected": -2.7674663066864014,
      "step": 9954
    },
    {
      "epoch": 1.3026744090946267,
      "grad_norm": 7.934042453765869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2936933636665344,
      "logits/rejected": -0.17477044463157654,
      "logps/chosen": -382.1681213378906,
      "logps/rejected": -418.90997314453125,
      "loss": 0.4824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7979092001914978,
      "rewards/margins": 1.2615021467208862,
      "rewards/rejected": -2.0594112873077393,
      "step": 9955
    },
    {
      "epoch": 1.3028052670319785,
      "grad_norm": 16.010828018188477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05058275908231735,
      "logits/rejected": -0.21516725420951843,
      "logps/chosen": -641.8690795898438,
      "logps/rejected": -577.9559326171875,
      "loss": 0.8345,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9435540437698364,
      "rewards/margins": 0.6153655648231506,
      "rewards/rejected": -1.5589197874069214,
      "step": 9956
    },
    {
      "epoch": 1.30293612496933,
      "grad_norm": 8.668204307556152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10709631443023682,
      "logits/rejected": -0.03648408129811287,
      "logps/chosen": -466.486572265625,
      "logps/rejected": -440.2056884765625,
      "loss": 0.4454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6049467325210571,
      "rewards/margins": 1.4446899890899658,
      "rewards/rejected": -2.0496366024017334,
      "step": 9957
    },
    {
      "epoch": 1.3030669829066819,
      "grad_norm": 5.612931251525879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.049924686551094055,
      "logits/rejected": 0.08703206479549408,
      "logps/chosen": -389.8695068359375,
      "logps/rejected": -475.10406494140625,
      "loss": 0.3271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1163485050201416,
      "rewards/margins": 1.139900803565979,
      "rewards/rejected": -2.25624942779541,
      "step": 9958
    },
    {
      "epoch": 1.3031978408440337,
      "grad_norm": 7.379579544067383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6015282869338989,
      "logits/rejected": -0.15524394810199738,
      "logps/chosen": -580.29345703125,
      "logps/rejected": -702.279052734375,
      "loss": 0.3806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1523774862289429,
      "rewards/margins": 1.2488021850585938,
      "rewards/rejected": -2.401179552078247,
      "step": 9959
    },
    {
      "epoch": 1.3033286987813855,
      "grad_norm": 6.36818790435791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08032116293907166,
      "logits/rejected": -0.19111648201942444,
      "logps/chosen": -401.16143798828125,
      "logps/rejected": -461.2607116699219,
      "loss": 0.3086,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.90476393699646,
      "rewards/margins": 1.5691787004470825,
      "rewards/rejected": -2.473942756652832,
      "step": 9960
    },
    {
      "epoch": 1.3034595567187373,
      "grad_norm": 7.6035332679748535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012722358107566833,
      "logits/rejected": -0.1421581506729126,
      "logps/chosen": -336.8186950683594,
      "logps/rejected": -401.9320068359375,
      "loss": 0.4424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5299463272094727,
      "rewards/margins": 2.4598934650421143,
      "rewards/rejected": -2.989839553833008,
      "step": 9961
    },
    {
      "epoch": 1.303590414656089,
      "grad_norm": 7.314561367034912,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3795980215072632,
      "logits/rejected": -0.23592069745063782,
      "logps/chosen": -218.3697967529297,
      "logps/rejected": -239.81124877929688,
      "loss": 0.2936,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8019654750823975,
      "rewards/margins": 2.1383233070373535,
      "rewards/rejected": -2.940288782119751,
      "step": 9962
    },
    {
      "epoch": 1.3037212725934406,
      "grad_norm": 8.918808937072754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08869251608848572,
      "logits/rejected": -0.13931286334991455,
      "logps/chosen": -265.1409606933594,
      "logps/rejected": -326.8831481933594,
      "loss": 0.5359,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.516027808189392,
      "rewards/margins": 1.786083459854126,
      "rewards/rejected": -3.3021109104156494,
      "step": 9963
    },
    {
      "epoch": 1.3038521305307924,
      "grad_norm": 6.060835361480713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3841323256492615,
      "logits/rejected": -0.43616944551467896,
      "logps/chosen": -492.1293029785156,
      "logps/rejected": -526.58642578125,
      "loss": 0.177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8994428515434265,
      "rewards/margins": 2.419522762298584,
      "rewards/rejected": -3.3189656734466553,
      "step": 9964
    },
    {
      "epoch": 1.3039829884681442,
      "grad_norm": 8.75354290008545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3453078866004944,
      "logits/rejected": -0.26180365681648254,
      "logps/chosen": -402.4974670410156,
      "logps/rejected": -383.134033203125,
      "loss": 0.368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9178592562675476,
      "rewards/margins": 1.5785901546478271,
      "rewards/rejected": -2.4964494705200195,
      "step": 9965
    },
    {
      "epoch": 1.304113846405496,
      "grad_norm": 7.3379902839660645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09680074453353882,
      "logits/rejected": -0.10582855343818665,
      "logps/chosen": -520.1683349609375,
      "logps/rejected": -461.4238586425781,
      "loss": 0.3975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6239437460899353,
      "rewards/margins": 1.583832025527954,
      "rewards/rejected": -2.207775592803955,
      "step": 9966
    },
    {
      "epoch": 1.3042447043428478,
      "grad_norm": 5.975865840911865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3410683870315552,
      "logits/rejected": -0.12077157199382782,
      "logps/chosen": -403.5470275878906,
      "logps/rejected": -496.3880615234375,
      "loss": 0.318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.594041645526886,
      "rewards/margins": 1.3613277673721313,
      "rewards/rejected": -1.9553695917129517,
      "step": 9967
    },
    {
      "epoch": 1.3043755622801996,
      "grad_norm": 7.024695873260498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05384375527501106,
      "logits/rejected": -0.10956420749425888,
      "logps/chosen": -326.110107421875,
      "logps/rejected": -367.71148681640625,
      "loss": 0.4469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.278981328010559,
      "rewards/margins": 0.9864928722381592,
      "rewards/rejected": -2.2654740810394287,
      "step": 9968
    },
    {
      "epoch": 1.3045064202175514,
      "grad_norm": 8.072416305541992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11299166083335876,
      "logits/rejected": -0.05793561413884163,
      "logps/chosen": -484.70318603515625,
      "logps/rejected": -503.48248291015625,
      "loss": 0.3963,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7355526685714722,
      "rewards/margins": 1.5807812213897705,
      "rewards/rejected": -2.316333770751953,
      "step": 9969
    },
    {
      "epoch": 1.3046372781549032,
      "grad_norm": 4.866308212280273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21771597862243652,
      "logits/rejected": -0.2898910939693451,
      "logps/chosen": -402.7339172363281,
      "logps/rejected": -397.521240234375,
      "loss": 0.3346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9586610794067383,
      "rewards/margins": 1.5932711362838745,
      "rewards/rejected": -2.5519320964813232,
      "step": 9970
    },
    {
      "epoch": 1.304768136092255,
      "grad_norm": 8.952192306518555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2540372610092163,
      "logits/rejected": -0.10201926529407501,
      "logps/chosen": -273.079833984375,
      "logps/rejected": -311.2688903808594,
      "loss": 0.5044,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7226440906524658,
      "rewards/margins": 0.692700982093811,
      "rewards/rejected": -1.4153450727462769,
      "step": 9971
    },
    {
      "epoch": 1.3048989940296067,
      "grad_norm": 6.114611625671387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2092645913362503,
      "logits/rejected": -0.421676367521286,
      "logps/chosen": -621.83837890625,
      "logps/rejected": -451.24609375,
      "loss": 0.2627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5648788213729858,
      "rewards/margins": 2.1532645225524902,
      "rewards/rejected": -2.7181432247161865,
      "step": 9972
    },
    {
      "epoch": 1.3050298519669583,
      "grad_norm": 4.178023815155029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28572434186935425,
      "logits/rejected": -0.2680526077747345,
      "logps/chosen": -487.220703125,
      "logps/rejected": -446.5145263671875,
      "loss": 0.2027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0744667649269104,
      "rewards/margins": 2.7311995029449463,
      "rewards/rejected": -2.6567327976226807,
      "step": 9973
    },
    {
      "epoch": 1.30516070990431,
      "grad_norm": 8.065741539001465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4494129419326782,
      "logits/rejected": -0.48327210545539856,
      "logps/chosen": -471.54412841796875,
      "logps/rejected": -474.86395263671875,
      "loss": 0.4552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.042755126953125,
      "rewards/margins": 1.147550344467163,
      "rewards/rejected": -2.190305471420288,
      "step": 9974
    },
    {
      "epoch": 1.305291567841662,
      "grad_norm": 6.979123115539551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058952972292900085,
      "logits/rejected": -0.05991050601005554,
      "logps/chosen": -388.9776611328125,
      "logps/rejected": -428.33123779296875,
      "loss": 0.3691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.242390513420105,
      "rewards/margins": 1.4260095357894897,
      "rewards/rejected": -2.668400287628174,
      "step": 9975
    },
    {
      "epoch": 1.3054224257790137,
      "grad_norm": 11.292827606201172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24004638195037842,
      "logits/rejected": 0.05103103071451187,
      "logps/chosen": -457.2087097167969,
      "logps/rejected": -583.9893798828125,
      "loss": 0.758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9881849884986877,
      "rewards/margins": 1.4335130453109741,
      "rewards/rejected": -2.4216978549957275,
      "step": 9976
    },
    {
      "epoch": 1.3055532837163655,
      "grad_norm": 15.438690185546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13802817463874817,
      "logits/rejected": -0.2442447692155838,
      "logps/chosen": -375.7600402832031,
      "logps/rejected": -449.10687255859375,
      "loss": 0.5352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2258379459381104,
      "rewards/margins": 0.7084470987319946,
      "rewards/rejected": -1.9342851638793945,
      "step": 9977
    },
    {
      "epoch": 1.305684141653717,
      "grad_norm": 6.107376575469971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04175742715597153,
      "logits/rejected": -0.2649037539958954,
      "logps/chosen": -463.3999328613281,
      "logps/rejected": -353.53582763671875,
      "loss": 0.37,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2098093032836914,
      "rewards/margins": 1.892973780632019,
      "rewards/rejected": -2.102782964706421,
      "step": 9978
    },
    {
      "epoch": 1.3058149995910688,
      "grad_norm": 9.382026672363281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035596445202827454,
      "logits/rejected": -0.2762029469013214,
      "logps/chosen": -410.10931396484375,
      "logps/rejected": -377.2660217285156,
      "loss": 0.5416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0454826354980469,
      "rewards/margins": 0.8153113126754761,
      "rewards/rejected": -1.8607938289642334,
      "step": 9979
    },
    {
      "epoch": 1.3059458575284206,
      "grad_norm": 6.646156311035156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2439335286617279,
      "logits/rejected": -0.5365559458732605,
      "logps/chosen": -221.94760131835938,
      "logps/rejected": -380.2254638671875,
      "loss": 0.3145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.708693027496338,
      "rewards/margins": 2.62268328666687,
      "rewards/rejected": -4.331376552581787,
      "step": 9980
    },
    {
      "epoch": 1.3060767154657724,
      "grad_norm": 6.696296215057373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1402595490217209,
      "logits/rejected": -0.2439119815826416,
      "logps/chosen": -331.47998046875,
      "logps/rejected": -334.2408752441406,
      "loss": 0.4491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1292704343795776,
      "rewards/margins": 0.8958398103713989,
      "rewards/rejected": -2.0251100063323975,
      "step": 9981
    },
    {
      "epoch": 1.3062075734031242,
      "grad_norm": 7.533633232116699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06489408016204834,
      "logits/rejected": -0.07412231713533401,
      "logps/chosen": -425.323974609375,
      "logps/rejected": -364.706298828125,
      "loss": 0.4817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.884274959564209,
      "rewards/margins": 1.1028804779052734,
      "rewards/rejected": -1.9871554374694824,
      "step": 9982
    },
    {
      "epoch": 1.306338431340476,
      "grad_norm": 11.710992813110352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2339046150445938,
      "logits/rejected": -0.2927798926830292,
      "logps/chosen": -504.6980285644531,
      "logps/rejected": -461.1312255859375,
      "loss": 0.6732,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.418408989906311,
      "rewards/margins": 0.23339897394180298,
      "rewards/rejected": -1.6518080234527588,
      "step": 9983
    },
    {
      "epoch": 1.3064692892778278,
      "grad_norm": 8.535435676574707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12092577666044235,
      "logits/rejected": -0.09947460889816284,
      "logps/chosen": -511.2826843261719,
      "logps/rejected": -455.32940673828125,
      "loss": 0.5299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4500883221626282,
      "rewards/margins": 2.3965466022491455,
      "rewards/rejected": -2.846634864807129,
      "step": 9984
    },
    {
      "epoch": 1.3066001472151796,
      "grad_norm": 5.48508358001709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26301270723342896,
      "logits/rejected": -0.31418585777282715,
      "logps/chosen": -326.35333251953125,
      "logps/rejected": -376.3094482421875,
      "loss": 0.3379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4485573172569275,
      "rewards/margins": 1.9013880491256714,
      "rewards/rejected": -2.349945068359375,
      "step": 9985
    },
    {
      "epoch": 1.3067310051525314,
      "grad_norm": 7.41340970993042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35589903593063354,
      "logits/rejected": -0.4283237159252167,
      "logps/chosen": -516.847900390625,
      "logps/rejected": -436.4081115722656,
      "loss": 0.3411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0986507385969162,
      "rewards/margins": 1.418009638786316,
      "rewards/rejected": -1.516660451889038,
      "step": 9986
    },
    {
      "epoch": 1.3068618630898832,
      "grad_norm": 4.423093795776367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48537522554397583,
      "logits/rejected": -0.28531450033187866,
      "logps/chosen": -479.044189453125,
      "logps/rejected": -534.5443725585938,
      "loss": 0.2132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7647266387939453,
      "rewards/margins": 1.7410420179367065,
      "rewards/rejected": -2.5057687759399414,
      "step": 9987
    },
    {
      "epoch": 1.3069927210272347,
      "grad_norm": 4.666927814483643,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3055357336997986,
      "logits/rejected": -0.14389276504516602,
      "logps/chosen": -525.697021484375,
      "logps/rejected": -504.833251953125,
      "loss": 0.341,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26416015625,
      "rewards/margins": 1.5415172576904297,
      "rewards/rejected": -1.8056774139404297,
      "step": 9988
    },
    {
      "epoch": 1.3071235789645865,
      "grad_norm": 7.162912368774414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007684752345085144,
      "logits/rejected": 0.09235795587301254,
      "logps/chosen": -405.4969482421875,
      "logps/rejected": -356.1526184082031,
      "loss": 0.3444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6485127806663513,
      "rewards/margins": 1.180497407913208,
      "rewards/rejected": -1.829010248184204,
      "step": 9989
    },
    {
      "epoch": 1.3072544369019383,
      "grad_norm": 3.1975622177124023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11957356333732605,
      "logits/rejected": 0.05422484874725342,
      "logps/chosen": -339.1174011230469,
      "logps/rejected": -324.27880859375,
      "loss": 0.1838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9042531847953796,
      "rewards/margins": 1.9425182342529297,
      "rewards/rejected": -2.846771240234375,
      "step": 9990
    },
    {
      "epoch": 1.3073852948392901,
      "grad_norm": 8.633116722106934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4699910283088684,
      "logits/rejected": -0.5453165769577026,
      "logps/chosen": -320.2927551269531,
      "logps/rejected": -309.303955078125,
      "loss": 0.6487,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7474015951156616,
      "rewards/margins": 0.3625665605068207,
      "rewards/rejected": -1.1099680662155151,
      "step": 9991
    },
    {
      "epoch": 1.307516152776642,
      "grad_norm": 4.419416427612305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30812621116638184,
      "logits/rejected": -0.23967230319976807,
      "logps/chosen": -453.8827209472656,
      "logps/rejected": -440.7391662597656,
      "loss": 0.2486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5605090856552124,
      "rewards/margins": 2.2405846118927,
      "rewards/rejected": -2.8010940551757812,
      "step": 9992
    },
    {
      "epoch": 1.3076470107139937,
      "grad_norm": 14.5724458694458,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031366802752017975,
      "logits/rejected": 0.05264894291758537,
      "logps/chosen": -584.2115478515625,
      "logps/rejected": -499.5037841796875,
      "loss": 0.817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.169669508934021,
      "rewards/margins": 0.6741007566452026,
      "rewards/rejected": -1.843770146369934,
      "step": 9993
    },
    {
      "epoch": 1.3077778686513453,
      "grad_norm": 9.749919891357422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013059243559837341,
      "logits/rejected": -0.049509551376104355,
      "logps/chosen": -399.38055419921875,
      "logps/rejected": -344.4924011230469,
      "loss": 0.7217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7441310882568359,
      "rewards/margins": 0.937099814414978,
      "rewards/rejected": -1.6812310218811035,
      "step": 9994
    },
    {
      "epoch": 1.307908726588697,
      "grad_norm": 7.70706033706665,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013255953788757324,
      "logits/rejected": -0.22545742988586426,
      "logps/chosen": -303.76422119140625,
      "logps/rejected": -390.1455993652344,
      "loss": 0.6708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2131667137145996,
      "rewards/margins": 1.4731824398040771,
      "rewards/rejected": -2.686349391937256,
      "step": 9995
    },
    {
      "epoch": 1.3080395845260488,
      "grad_norm": 9.646448135375977,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009830115363001823,
      "logits/rejected": -0.18981026113033295,
      "logps/chosen": -578.0855712890625,
      "logps/rejected": -606.6425170898438,
      "loss": 0.7033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8310425281524658,
      "rewards/margins": 0.8651705980300903,
      "rewards/rejected": -2.6962132453918457,
      "step": 9996
    },
    {
      "epoch": 1.3081704424634006,
      "grad_norm": 9.923068046569824,
      "learning_rate": 5e-06,
      "logits/chosen": 0.196297749876976,
      "logits/rejected": 0.054905641824007034,
      "logps/chosen": -460.16986083984375,
      "logps/rejected": -447.2449951171875,
      "loss": 0.4802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3171093463897705,
      "rewards/margins": 1.922804832458496,
      "rewards/rejected": -3.2399141788482666,
      "step": 9997
    },
    {
      "epoch": 1.3083013004007524,
      "grad_norm": 6.005857944488525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0402710847556591,
      "logits/rejected": -0.1437646448612213,
      "logps/chosen": -352.4011535644531,
      "logps/rejected": -379.7887268066406,
      "loss": 0.3106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6116681694984436,
      "rewards/margins": 1.2258167266845703,
      "rewards/rejected": -1.8374848365783691,
      "step": 9998
    },
    {
      "epoch": 1.3084321583381042,
      "grad_norm": 8.58810806274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.265758752822876,
      "logits/rejected": -0.2708607316017151,
      "logps/chosen": -470.49005126953125,
      "logps/rejected": -617.146484375,
      "loss": 0.4167,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0999376773834229,
      "rewards/margins": 1.4781733751296997,
      "rewards/rejected": -2.578110933303833,
      "step": 9999
    },
    {
      "epoch": 1.308563016275456,
      "grad_norm": 6.719044208526611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11540541797876358,
      "logits/rejected": -0.3867833614349365,
      "logps/chosen": -430.6104736328125,
      "logps/rejected": -388.143798828125,
      "loss": 0.2931,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25553834438323975,
      "rewards/margins": 1.895092248916626,
      "rewards/rejected": -2.150630474090576,
      "step": 10000
    },
    {
      "epoch": 1.3086938742128078,
      "grad_norm": 7.571943283081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15209540724754333,
      "logits/rejected": 0.056018538773059845,
      "logps/chosen": -428.00457763671875,
      "logps/rejected": -549.5344848632812,
      "loss": 0.4615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.688174843788147,
      "rewards/margins": 0.8510028719902039,
      "rewards/rejected": -1.5391777753829956,
      "step": 10001
    },
    {
      "epoch": 1.3088247321501596,
      "grad_norm": 8.921792030334473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04810088127851486,
      "logits/rejected": -0.031196296215057373,
      "logps/chosen": -438.5066833496094,
      "logps/rejected": -462.4783020019531,
      "loss": 0.3943,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8656668663024902,
      "rewards/margins": 1.1622540950775146,
      "rewards/rejected": -2.027920961380005,
      "step": 10002
    },
    {
      "epoch": 1.3089555900875112,
      "grad_norm": 2.9236156940460205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2713529169559479,
      "logits/rejected": -0.20399363338947296,
      "logps/chosen": -470.6072692871094,
      "logps/rejected": -347.7060852050781,
      "loss": 0.141,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.210284024477005,
      "rewards/margins": 3.88120698928833,
      "rewards/rejected": -3.6709227561950684,
      "step": 10003
    },
    {
      "epoch": 1.309086448024863,
      "grad_norm": 10.00052261352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2610621750354767,
      "logits/rejected": -0.3427228033542633,
      "logps/chosen": -603.7084350585938,
      "logps/rejected": -566.2989501953125,
      "loss": 0.6173,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6039533615112305,
      "rewards/margins": 0.637982964515686,
      "rewards/rejected": -2.241936206817627,
      "step": 10004
    },
    {
      "epoch": 1.3092173059622148,
      "grad_norm": 5.739398002624512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14238251745700836,
      "logits/rejected": 0.06114106625318527,
      "logps/chosen": -463.87359619140625,
      "logps/rejected": -381.91864013671875,
      "loss": 0.2939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7397849559783936,
      "rewards/margins": 1.6019651889801025,
      "rewards/rejected": -2.341750383377075,
      "step": 10005
    },
    {
      "epoch": 1.3093481638995665,
      "grad_norm": 6.501455307006836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03635157644748688,
      "logits/rejected": -0.19320932030677795,
      "logps/chosen": -367.4027404785156,
      "logps/rejected": -341.47174072265625,
      "loss": 0.3507,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4524957835674286,
      "rewards/margins": 1.5125396251678467,
      "rewards/rejected": -1.965035319328308,
      "step": 10006
    },
    {
      "epoch": 1.3094790218369183,
      "grad_norm": 9.041833877563477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15533214807510376,
      "logits/rejected": -0.25937899947166443,
      "logps/chosen": -612.5311889648438,
      "logps/rejected": -576.9142456054688,
      "loss": 0.4908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05556221306324005,
      "rewards/margins": 1.5632994174957275,
      "rewards/rejected": -1.507737159729004,
      "step": 10007
    },
    {
      "epoch": 1.3096098797742701,
      "grad_norm": 10.723383903503418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03312821686267853,
      "logits/rejected": -0.044134270399808884,
      "logps/chosen": -352.789794921875,
      "logps/rejected": -410.8111877441406,
      "loss": 0.6097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6715240478515625,
      "rewards/margins": 1.4177762269973755,
      "rewards/rejected": -2.0893001556396484,
      "step": 10008
    },
    {
      "epoch": 1.3097407377116217,
      "grad_norm": 9.459444046020508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10070481896400452,
      "logits/rejected": -0.1815052181482315,
      "logps/chosen": -387.0205993652344,
      "logps/rejected": -423.70831298828125,
      "loss": 0.4789,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4734084606170654,
      "rewards/margins": 1.4277223348617554,
      "rewards/rejected": -2.9011309146881104,
      "step": 10009
    },
    {
      "epoch": 1.3098715956489735,
      "grad_norm": 10.349984169006348,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11942945420742035,
      "logits/rejected": -0.04209651052951813,
      "logps/chosen": -554.216064453125,
      "logps/rejected": -547.405517578125,
      "loss": 0.6579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5283945202827454,
      "rewards/margins": 1.5260361433029175,
      "rewards/rejected": -2.0544309616088867,
      "step": 10010
    },
    {
      "epoch": 1.3100024535863253,
      "grad_norm": 5.5942606925964355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.062171146273612976,
      "logits/rejected": -0.02067001722753048,
      "logps/chosen": -648.8546752929688,
      "logps/rejected": -512.9896240234375,
      "loss": 0.2913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5333667993545532,
      "rewards/margins": 1.5609774589538574,
      "rewards/rejected": -2.094344139099121,
      "step": 10011
    },
    {
      "epoch": 1.310133311523677,
      "grad_norm": 7.860828876495361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11774256825447083,
      "logits/rejected": -0.28871646523475647,
      "logps/chosen": -636.8888549804688,
      "logps/rejected": -640.2077026367188,
      "loss": 0.3668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20588549971580505,
      "rewards/margins": 1.6254749298095703,
      "rewards/rejected": -1.4195895195007324,
      "step": 10012
    },
    {
      "epoch": 1.3102641694610289,
      "grad_norm": 3.976323366165161,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3975067138671875,
      "logits/rejected": 0.042722199112176895,
      "logps/chosen": -504.6208801269531,
      "logps/rejected": -356.718505859375,
      "loss": 0.1563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4829586148262024,
      "rewards/margins": 2.258884906768799,
      "rewards/rejected": -2.7418437004089355,
      "step": 10013
    },
    {
      "epoch": 1.3103950273983807,
      "grad_norm": 5.610474109649658,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008265377022325993,
      "logits/rejected": -0.229538232088089,
      "logps/chosen": -538.0274658203125,
      "logps/rejected": -523.800537109375,
      "loss": 0.2288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4223823547363281,
      "rewards/margins": 3.829535961151123,
      "rewards/rejected": -3.407154083251953,
      "step": 10014
    },
    {
      "epoch": 1.3105258853357324,
      "grad_norm": 4.371959686279297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19936037063598633,
      "logits/rejected": -0.16525962948799133,
      "logps/chosen": -498.01123046875,
      "logps/rejected": -622.2650146484375,
      "loss": 0.1996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04331434890627861,
      "rewards/margins": 2.399725914001465,
      "rewards/rejected": -2.443040370941162,
      "step": 10015
    },
    {
      "epoch": 1.3106567432730842,
      "grad_norm": 12.429722785949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26666459441185,
      "logits/rejected": -0.29094576835632324,
      "logps/chosen": -450.9737548828125,
      "logps/rejected": -478.6468505859375,
      "loss": 0.7715,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4570305049419403,
      "rewards/margins": 0.6598539352416992,
      "rewards/rejected": -1.116884469985962,
      "step": 10016
    },
    {
      "epoch": 1.310787601210436,
      "grad_norm": 10.228586196899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19124191999435425,
      "logits/rejected": -0.08931475877761841,
      "logps/chosen": -534.2523193359375,
      "logps/rejected": -482.82208251953125,
      "loss": 0.4407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.339227557182312,
      "rewards/margins": 0.9892381429672241,
      "rewards/rejected": -2.328465700149536,
      "step": 10017
    },
    {
      "epoch": 1.3109184591477878,
      "grad_norm": 7.832136631011963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3228270709514618,
      "logits/rejected": -0.38499152660369873,
      "logps/chosen": -413.1445007324219,
      "logps/rejected": -409.0516052246094,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8847225308418274,
      "rewards/margins": 1.037850022315979,
      "rewards/rejected": -1.9225727319717407,
      "step": 10018
    },
    {
      "epoch": 1.3110493170851394,
      "grad_norm": 10.523210525512695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.260742723941803,
      "logits/rejected": -0.24604836106300354,
      "logps/chosen": -547.1881103515625,
      "logps/rejected": -552.0338745117188,
      "loss": 0.6884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42281559109687805,
      "rewards/margins": 0.3165287375450134,
      "rewards/rejected": -0.7393442392349243,
      "step": 10019
    },
    {
      "epoch": 1.3111801750224912,
      "grad_norm": 6.203271389007568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06846419721841812,
      "logits/rejected": -0.017209213227033615,
      "logps/chosen": -395.3089904785156,
      "logps/rejected": -379.3504333496094,
      "loss": 0.3687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6238030195236206,
      "rewards/margins": 1.6544829607009888,
      "rewards/rejected": -2.2782859802246094,
      "step": 10020
    },
    {
      "epoch": 1.311311032959843,
      "grad_norm": 13.23301887512207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010404229164123535,
      "logits/rejected": -0.13208571076393127,
      "logps/chosen": -463.76544189453125,
      "logps/rejected": -398.131103515625,
      "loss": 0.6287,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5206629037857056,
      "rewards/margins": 0.7362396717071533,
      "rewards/rejected": -1.2569025754928589,
      "step": 10021
    },
    {
      "epoch": 1.3114418908971948,
      "grad_norm": 12.266375541687012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09550956636667252,
      "logits/rejected": -0.3353568911552429,
      "logps/chosen": -478.1872253417969,
      "logps/rejected": -486.0523376464844,
      "loss": 0.8587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2928214073181152,
      "rewards/margins": 1.0476741790771484,
      "rewards/rejected": -2.3404955863952637,
      "step": 10022
    },
    {
      "epoch": 1.3115727488345466,
      "grad_norm": 6.177648067474365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31088006496429443,
      "logits/rejected": -0.30889692902565,
      "logps/chosen": -496.9537658691406,
      "logps/rejected": -878.3478393554688,
      "loss": 0.3609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37619346380233765,
      "rewards/margins": 1.2798982858657837,
      "rewards/rejected": -0.903704822063446,
      "step": 10023
    },
    {
      "epoch": 1.3117036067718981,
      "grad_norm": 7.624782562255859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28084033727645874,
      "logits/rejected": -0.12662623822689056,
      "logps/chosen": -524.575927734375,
      "logps/rejected": -603.0353393554688,
      "loss": 0.4223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4245273470878601,
      "rewards/margins": 0.8900409936904907,
      "rewards/rejected": -1.3145684003829956,
      "step": 10024
    },
    {
      "epoch": 1.31183446470925,
      "grad_norm": 5.193592548370361,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1352023333311081,
      "logits/rejected": -0.03300876170396805,
      "logps/chosen": -371.14923095703125,
      "logps/rejected": -483.63543701171875,
      "loss": 0.2616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15761621296405792,
      "rewards/margins": 2.022181272506714,
      "rewards/rejected": -2.179797410964966,
      "step": 10025
    },
    {
      "epoch": 1.3119653226466017,
      "grad_norm": 5.760132312774658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2336411029100418,
      "logits/rejected": -0.3697583079338074,
      "logps/chosen": -417.09759521484375,
      "logps/rejected": -444.3515930175781,
      "loss": 0.2919,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14966753125190735,
      "rewards/margins": 2.047199010848999,
      "rewards/rejected": -2.196866750717163,
      "step": 10026
    },
    {
      "epoch": 1.3120961805839535,
      "grad_norm": 6.278636932373047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02883918210864067,
      "logits/rejected": 0.09260417520999908,
      "logps/chosen": -325.51702880859375,
      "logps/rejected": -371.06317138671875,
      "loss": 0.3767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21873760223388672,
      "rewards/margins": 1.677678108215332,
      "rewards/rejected": -1.4589405059814453,
      "step": 10027
    },
    {
      "epoch": 1.3122270385213053,
      "grad_norm": 4.438509464263916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1268916130065918,
      "logits/rejected": -0.31857097148895264,
      "logps/chosen": -439.30322265625,
      "logps/rejected": -468.7607727050781,
      "loss": 0.1702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2987852096557617,
      "rewards/margins": 2.3834056854248047,
      "rewards/rejected": -2.6821908950805664,
      "step": 10028
    },
    {
      "epoch": 1.312357896458657,
      "grad_norm": 7.5318732261657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10677843540906906,
      "logits/rejected": -0.2632581293582916,
      "logps/chosen": -318.8493347167969,
      "logps/rejected": -307.9796142578125,
      "loss": 0.5465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8846818804740906,
      "rewards/margins": 1.2934837341308594,
      "rewards/rejected": -2.1781656742095947,
      "step": 10029
    },
    {
      "epoch": 1.3124887543960089,
      "grad_norm": 10.0036039352417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41648387908935547,
      "logits/rejected": -0.04105156660079956,
      "logps/chosen": -505.7691650390625,
      "logps/rejected": -477.8166809082031,
      "loss": 0.5624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1242507696151733,
      "rewards/margins": 1.2915425300598145,
      "rewards/rejected": -2.4157931804656982,
      "step": 10030
    },
    {
      "epoch": 1.3126196123333607,
      "grad_norm": 9.015185356140137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09747669845819473,
      "logits/rejected": -0.09464947879314423,
      "logps/chosen": -367.101318359375,
      "logps/rejected": -409.341796875,
      "loss": 0.5578,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5533273816108704,
      "rewards/margins": 0.9122408032417297,
      "rewards/rejected": -1.4655683040618896,
      "step": 10031
    },
    {
      "epoch": 1.3127504702707125,
      "grad_norm": 6.041314601898193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06875157356262207,
      "logits/rejected": -0.13728147745132446,
      "logps/chosen": -393.39788818359375,
      "logps/rejected": -280.5723876953125,
      "loss": 0.3177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5691349506378174,
      "rewards/margins": 1.4229955673217773,
      "rewards/rejected": -1.9921303987503052,
      "step": 10032
    },
    {
      "epoch": 1.3128813282080642,
      "grad_norm": 7.99535608291626,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2729986906051636,
      "logits/rejected": -0.2483222633600235,
      "logps/chosen": -308.37689208984375,
      "logps/rejected": -462.3758850097656,
      "loss": 0.3848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6661152839660645,
      "rewards/margins": 1.678291916847229,
      "rewards/rejected": -2.344407320022583,
      "step": 10033
    },
    {
      "epoch": 1.3130121861454158,
      "grad_norm": 10.714912414550781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25232788920402527,
      "logits/rejected": -0.30760887265205383,
      "logps/chosen": -467.04815673828125,
      "logps/rejected": -418.4023132324219,
      "loss": 0.7033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5299515128135681,
      "rewards/margins": 0.6118011474609375,
      "rewards/rejected": -1.1417526006698608,
      "step": 10034
    },
    {
      "epoch": 1.3131430440827676,
      "grad_norm": 4.597513198852539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0856281965970993,
      "logits/rejected": -0.24641570448875427,
      "logps/chosen": -409.9149169921875,
      "logps/rejected": -516.3823852539062,
      "loss": 0.2863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3669013977050781,
      "rewards/margins": 1.645054578781128,
      "rewards/rejected": -1.2781531810760498,
      "step": 10035
    },
    {
      "epoch": 1.3132739020201194,
      "grad_norm": 5.070451259613037,
      "learning_rate": 5e-06,
      "logits/chosen": -0.182759091258049,
      "logits/rejected": -0.13746050000190735,
      "logps/chosen": -456.197998046875,
      "logps/rejected": -486.17279052734375,
      "loss": 0.2717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2813136875629425,
      "rewards/margins": 1.9486831426620483,
      "rewards/rejected": -2.229996919631958,
      "step": 10036
    },
    {
      "epoch": 1.3134047599574712,
      "grad_norm": 12.128325462341309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10543377697467804,
      "logits/rejected": -0.421143501996994,
      "logps/chosen": -461.58282470703125,
      "logps/rejected": -770.815673828125,
      "loss": 0.5802,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.19687819480896,
      "rewards/margins": 0.87274169921875,
      "rewards/rejected": -2.069620132446289,
      "step": 10037
    },
    {
      "epoch": 1.313535617894823,
      "grad_norm": 5.783725261688232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12926103174686432,
      "logits/rejected": -0.010950211435556412,
      "logps/chosen": -505.44940185546875,
      "logps/rejected": -546.4695434570312,
      "loss": 0.3044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10311925411224365,
      "rewards/margins": 1.6066925525665283,
      "rewards/rejected": -1.5035732984542847,
      "step": 10038
    },
    {
      "epoch": 1.3136664758321748,
      "grad_norm": 7.418337345123291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3359643816947937,
      "logits/rejected": -0.4884411096572876,
      "logps/chosen": -327.7234191894531,
      "logps/rejected": -383.8525085449219,
      "loss": 0.3692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8776705861091614,
      "rewards/margins": 1.4642088413238525,
      "rewards/rejected": -2.341879367828369,
      "step": 10039
    },
    {
      "epoch": 1.3137973337695263,
      "grad_norm": 9.113678932189941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19263266026973724,
      "logits/rejected": -0.200925812125206,
      "logps/chosen": -557.6185913085938,
      "logps/rejected": -540.5675659179688,
      "loss": 0.4772,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21126976609230042,
      "rewards/margins": 0.6265919208526611,
      "rewards/rejected": -0.8378615975379944,
      "step": 10040
    },
    {
      "epoch": 1.3139281917068781,
      "grad_norm": 5.878889083862305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1692616045475006,
      "logits/rejected": 0.09932677447795868,
      "logps/chosen": -445.5879821777344,
      "logps/rejected": -533.2928466796875,
      "loss": 0.4364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5790266394615173,
      "rewards/margins": 0.9709034562110901,
      "rewards/rejected": -1.5499299764633179,
      "step": 10041
    },
    {
      "epoch": 1.31405904964423,
      "grad_norm": 7.736964702606201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09596361219882965,
      "logits/rejected": -0.20935681462287903,
      "logps/chosen": -557.2474975585938,
      "logps/rejected": -583.4795532226562,
      "loss": 0.4364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5715256929397583,
      "rewards/margins": 0.8924534320831299,
      "rewards/rejected": -1.4639791250228882,
      "step": 10042
    },
    {
      "epoch": 1.3141899075815817,
      "grad_norm": 2.3955748081207275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2298966348171234,
      "logits/rejected": -0.3024546504020691,
      "logps/chosen": -477.0023498535156,
      "logps/rejected": -445.332275390625,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3922586739063263,
      "rewards/margins": 2.8780860900878906,
      "rewards/rejected": -2.4858272075653076,
      "step": 10043
    },
    {
      "epoch": 1.3143207655189335,
      "grad_norm": 9.271108627319336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07318204641342163,
      "logits/rejected": -0.09622500836849213,
      "logps/chosen": -288.90264892578125,
      "logps/rejected": -399.3004150390625,
      "loss": 0.4,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1205124855041504,
      "rewards/margins": 1.6348189115524292,
      "rewards/rejected": -2.75533127784729,
      "step": 10044
    },
    {
      "epoch": 1.3144516234562853,
      "grad_norm": 7.534157752990723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.326043963432312,
      "logits/rejected": -0.2274324744939804,
      "logps/chosen": -332.9934997558594,
      "logps/rejected": -656.26953125,
      "loss": 0.5239,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6749863624572754,
      "rewards/margins": 1.079527735710144,
      "rewards/rejected": -2.75451397895813,
      "step": 10045
    },
    {
      "epoch": 1.314582481393637,
      "grad_norm": 3.5595993995666504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14601702988147736,
      "logits/rejected": -0.3353944420814514,
      "logps/chosen": -599.5074462890625,
      "logps/rejected": -408.2983703613281,
      "loss": 0.1411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6876798272132874,
      "rewards/margins": 2.8619511127471924,
      "rewards/rejected": -3.549631118774414,
      "step": 10046
    },
    {
      "epoch": 1.3147133393309889,
      "grad_norm": 7.650601387023926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3639814853668213,
      "logits/rejected": -0.15937329828739166,
      "logps/chosen": -305.510498046875,
      "logps/rejected": -393.0411682128906,
      "loss": 0.3659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3428615629673004,
      "rewards/margins": 1.7872250080108643,
      "rewards/rejected": -2.130086660385132,
      "step": 10047
    },
    {
      "epoch": 1.3148441972683407,
      "grad_norm": 7.77492618560791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4094829261302948,
      "logits/rejected": -0.39458751678466797,
      "logps/chosen": -435.2425537109375,
      "logps/rejected": -420.7287902832031,
      "loss": 0.5851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3629919290542603,
      "rewards/margins": 0.5290586948394775,
      "rewards/rejected": -1.8920506238937378,
      "step": 10048
    },
    {
      "epoch": 1.3149750552056922,
      "grad_norm": 8.393645286560059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3792506158351898,
      "logits/rejected": -0.4634113311767578,
      "logps/chosen": -393.47821044921875,
      "logps/rejected": -352.0587463378906,
      "loss": 0.5304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9459616541862488,
      "rewards/margins": 0.7115954756736755,
      "rewards/rejected": -1.6575570106506348,
      "step": 10049
    },
    {
      "epoch": 1.315105913143044,
      "grad_norm": 5.674947261810303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15733802318572998,
      "logits/rejected": -0.32379150390625,
      "logps/chosen": -467.18109130859375,
      "logps/rejected": -409.37158203125,
      "loss": 0.2688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8186753988265991,
      "rewards/margins": 1.5261874198913574,
      "rewards/rejected": -2.344862937927246,
      "step": 10050
    },
    {
      "epoch": 1.3152367710803958,
      "grad_norm": 7.610276699066162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24761594831943512,
      "logits/rejected": -0.12483345717191696,
      "logps/chosen": -588.0562133789062,
      "logps/rejected": -609.0489501953125,
      "loss": 0.4771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07370737195014954,
      "rewards/margins": 2.1581287384033203,
      "rewards/rejected": -2.2318360805511475,
      "step": 10051
    },
    {
      "epoch": 1.3153676290177476,
      "grad_norm": 6.220680236816406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06638084352016449,
      "logits/rejected": -0.1696239858865738,
      "logps/chosen": -363.42974853515625,
      "logps/rejected": -403.45465087890625,
      "loss": 0.407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9093994498252869,
      "rewards/margins": 1.2062098979949951,
      "rewards/rejected": -2.1156091690063477,
      "step": 10052
    },
    {
      "epoch": 1.3154984869550994,
      "grad_norm": 10.150870323181152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22918850183486938,
      "logits/rejected": -0.15142382681369781,
      "logps/chosen": -336.29364013671875,
      "logps/rejected": -367.3161315917969,
      "loss": 0.6671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.787213921546936,
      "rewards/margins": 1.0537203550338745,
      "rewards/rejected": -1.8409342765808105,
      "step": 10053
    },
    {
      "epoch": 1.3156293448924512,
      "grad_norm": 5.874444484710693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15051724016666412,
      "logits/rejected": 0.031249884516000748,
      "logps/chosen": -421.7030334472656,
      "logps/rejected": -435.9640808105469,
      "loss": 0.4062,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49022889137268066,
      "rewards/margins": 1.5784378051757812,
      "rewards/rejected": -2.068666696548462,
      "step": 10054
    },
    {
      "epoch": 1.3157602028298028,
      "grad_norm": 12.909148216247559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05934473127126694,
      "logits/rejected": -0.19930517673492432,
      "logps/chosen": -505.9049987792969,
      "logps/rejected": -398.96759033203125,
      "loss": 0.9091,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2115023136138916,
      "rewards/margins": 0.11232396960258484,
      "rewards/rejected": -1.3238261938095093,
      "step": 10055
    },
    {
      "epoch": 1.3158910607671546,
      "grad_norm": 5.710630416870117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00801059603691101,
      "logits/rejected": -0.157790869474411,
      "logps/chosen": -558.95263671875,
      "logps/rejected": -536.9960327148438,
      "loss": 0.28,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08310945332050323,
      "rewards/margins": 1.9237654209136963,
      "rewards/rejected": -1.8406559228897095,
      "step": 10056
    },
    {
      "epoch": 1.3160219187045064,
      "grad_norm": 6.327239036560059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4374241828918457,
      "logits/rejected": -0.5320819616317749,
      "logps/chosen": -543.2863159179688,
      "logps/rejected": -484.29693603515625,
      "loss": 0.3493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3147367238998413,
      "rewards/margins": 1.1110748052597046,
      "rewards/rejected": -1.4258116483688354,
      "step": 10057
    },
    {
      "epoch": 1.3161527766418581,
      "grad_norm": 7.235833168029785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.361230731010437,
      "logits/rejected": -0.2672540247440338,
      "logps/chosen": -576.489013671875,
      "logps/rejected": -730.4811401367188,
      "loss": 0.4514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4882957637310028,
      "rewards/margins": 1.4495644569396973,
      "rewards/rejected": -1.9378602504730225,
      "step": 10058
    },
    {
      "epoch": 1.31628363457921,
      "grad_norm": 6.53102445602417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4262653887271881,
      "logits/rejected": -0.4143044948577881,
      "logps/chosen": -487.83795166015625,
      "logps/rejected": -424.15557861328125,
      "loss": 0.4041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8005155324935913,
      "rewards/margins": 1.0694321393966675,
      "rewards/rejected": -1.8699475526809692,
      "step": 10059
    },
    {
      "epoch": 1.3164144925165617,
      "grad_norm": 8.722084999084473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4107419550418854,
      "logits/rejected": -0.33025413751602173,
      "logps/chosen": -303.8995361328125,
      "logps/rejected": -388.2308349609375,
      "loss": 0.4557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2691948413848877,
      "rewards/margins": 1.2192697525024414,
      "rewards/rejected": -2.488464832305908,
      "step": 10060
    },
    {
      "epoch": 1.3165453504539135,
      "grad_norm": 5.151780128479004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16805052757263184,
      "logits/rejected": -0.14748699963092804,
      "logps/chosen": -378.45672607421875,
      "logps/rejected": -459.2406311035156,
      "loss": 0.2032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.44401073455810547,
      "rewards/margins": 1.808704137802124,
      "rewards/rejected": -2.2527148723602295,
      "step": 10061
    },
    {
      "epoch": 1.3166762083912653,
      "grad_norm": 8.61698055267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11787901818752289,
      "logits/rejected": -0.12842848896980286,
      "logps/chosen": -515.4287109375,
      "logps/rejected": -418.6534118652344,
      "loss": 0.4212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8944804072380066,
      "rewards/margins": 1.3713505268096924,
      "rewards/rejected": -2.2658307552337646,
      "step": 10062
    },
    {
      "epoch": 1.316807066328617,
      "grad_norm": 6.7002434730529785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20151108503341675,
      "logits/rejected": -0.09501457214355469,
      "logps/chosen": -426.0069885253906,
      "logps/rejected": -481.56536865234375,
      "loss": 0.1831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0697731003165245,
      "rewards/margins": 2.5098037719726562,
      "rewards/rejected": -2.5795769691467285,
      "step": 10063
    },
    {
      "epoch": 1.316937924265969,
      "grad_norm": 11.98862075805664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2979498505592346,
      "logits/rejected": -0.1481487900018692,
      "logps/chosen": -408.1418762207031,
      "logps/rejected": -600.5342407226562,
      "loss": 0.6064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8699559569358826,
      "rewards/margins": 0.7592878937721252,
      "rewards/rejected": -1.6292439699172974,
      "step": 10064
    },
    {
      "epoch": 1.3170687822033205,
      "grad_norm": 14.32773494720459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3719020485877991,
      "logits/rejected": -0.22580769658088684,
      "logps/chosen": -700.0488891601562,
      "logps/rejected": -408.9890441894531,
      "loss": 0.5509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4402871131896973,
      "rewards/margins": 0.8651199340820312,
      "rewards/rejected": -2.3054070472717285,
      "step": 10065
    },
    {
      "epoch": 1.3171996401406723,
      "grad_norm": 12.210556983947754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24158670008182526,
      "logits/rejected": -0.37346458435058594,
      "logps/chosen": -325.6756591796875,
      "logps/rejected": -270.8046569824219,
      "loss": 0.7961,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2860655784606934,
      "rewards/margins": 0.17499284446239471,
      "rewards/rejected": -2.4610583782196045,
      "step": 10066
    },
    {
      "epoch": 1.317330498078024,
      "grad_norm": 8.766989707946777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08931054174900055,
      "logits/rejected": -0.05476031452417374,
      "logps/chosen": -476.0830078125,
      "logps/rejected": -438.3428955078125,
      "loss": 0.4497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1276767253875732,
      "rewards/margins": 1.0827962160110474,
      "rewards/rejected": -2.210472822189331,
      "step": 10067
    },
    {
      "epoch": 1.3174613560153758,
      "grad_norm": 9.298298835754395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.363648921251297,
      "logits/rejected": -0.3420711159706116,
      "logps/chosen": -398.31549072265625,
      "logps/rejected": -368.2520751953125,
      "loss": 0.4998,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8201401233673096,
      "rewards/margins": 1.4145039319992065,
      "rewards/rejected": -2.2346441745758057,
      "step": 10068
    },
    {
      "epoch": 1.3175922139527276,
      "grad_norm": 12.20388412475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1287359595298767,
      "logits/rejected": -0.16319283843040466,
      "logps/chosen": -308.5169372558594,
      "logps/rejected": -426.04705810546875,
      "loss": 0.8044,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.5746960639953613,
      "rewards/margins": 0.47236716747283936,
      "rewards/rejected": -3.0470633506774902,
      "step": 10069
    },
    {
      "epoch": 1.3177230718900792,
      "grad_norm": 9.553488731384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5374711751937866,
      "logits/rejected": -0.5207439064979553,
      "logps/chosen": -629.585205078125,
      "logps/rejected": -576.2450561523438,
      "loss": 0.5661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8741302490234375,
      "rewards/margins": 0.906200647354126,
      "rewards/rejected": -2.7803308963775635,
      "step": 10070
    },
    {
      "epoch": 1.317853929827431,
      "grad_norm": 8.832014083862305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3580876588821411,
      "logits/rejected": -0.2783261239528656,
      "logps/chosen": -336.75439453125,
      "logps/rejected": -399.2357482910156,
      "loss": 0.5717,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1035807132720947,
      "rewards/margins": 0.9249357581138611,
      "rewards/rejected": -2.0285162925720215,
      "step": 10071
    },
    {
      "epoch": 1.3179847877647828,
      "grad_norm": 9.742730140686035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2470930814743042,
      "logits/rejected": -0.25058484077453613,
      "logps/chosen": -452.9378662109375,
      "logps/rejected": -475.16876220703125,
      "loss": 0.6033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8810501098632812,
      "rewards/margins": 0.6876907348632812,
      "rewards/rejected": -1.5687408447265625,
      "step": 10072
    },
    {
      "epoch": 1.3181156457021346,
      "grad_norm": 5.5749921798706055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6584151983261108,
      "logits/rejected": -0.49984297156333923,
      "logps/chosen": -356.43658447265625,
      "logps/rejected": -442.24652099609375,
      "loss": 0.3594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9529653787612915,
      "rewards/margins": 0.9406454563140869,
      "rewards/rejected": -1.8936108350753784,
      "step": 10073
    },
    {
      "epoch": 1.3182465036394864,
      "grad_norm": 8.41042709350586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3556308150291443,
      "logits/rejected": -0.2816036343574524,
      "logps/chosen": -530.0726928710938,
      "logps/rejected": -455.94879150390625,
      "loss": 0.4582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8476635217666626,
      "rewards/margins": 1.177972435951233,
      "rewards/rejected": -2.0256359577178955,
      "step": 10074
    },
    {
      "epoch": 1.3183773615768382,
      "grad_norm": 9.788702011108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16304929554462433,
      "logits/rejected": -0.1880950629711151,
      "logps/chosen": -384.5355224609375,
      "logps/rejected": -369.1645812988281,
      "loss": 0.6462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8350828886032104,
      "rewards/margins": 0.5187269449234009,
      "rewards/rejected": -2.3538098335266113,
      "step": 10075
    },
    {
      "epoch": 1.31850821951419,
      "grad_norm": 4.87668514251709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40816158056259155,
      "logits/rejected": -0.3924601674079895,
      "logps/chosen": -316.6847839355469,
      "logps/rejected": -455.287841796875,
      "loss": 0.2948,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46597984433174133,
      "rewards/margins": 1.8276093006134033,
      "rewards/rejected": -2.2935895919799805,
      "step": 10076
    },
    {
      "epoch": 1.3186390774515417,
      "grad_norm": 6.99991512298584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36709892749786377,
      "logits/rejected": -0.3535028100013733,
      "logps/chosen": -390.3371887207031,
      "logps/rejected": -356.3947448730469,
      "loss": 0.468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9620332717895508,
      "rewards/margins": 1.2115516662597656,
      "rewards/rejected": -2.1735851764678955,
      "step": 10077
    },
    {
      "epoch": 1.3187699353888935,
      "grad_norm": 12.016463279724121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05442969501018524,
      "logits/rejected": 0.04796300828456879,
      "logps/chosen": -336.3801574707031,
      "logps/rejected": -382.3900146484375,
      "loss": 0.7584,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7833192944526672,
      "rewards/margins": 1.1640665531158447,
      "rewards/rejected": -1.9473856687545776,
      "step": 10078
    },
    {
      "epoch": 1.3189007933262453,
      "grad_norm": 10.656513214111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26565122604370117,
      "logits/rejected": -0.44663938879966736,
      "logps/chosen": -489.31671142578125,
      "logps/rejected": -439.41845703125,
      "loss": 0.7744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5159671306610107,
      "rewards/margins": 1.6565864086151123,
      "rewards/rejected": -2.172553777694702,
      "step": 10079
    },
    {
      "epoch": 1.319031651263597,
      "grad_norm": 7.65269660949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40073874592781067,
      "logits/rejected": -0.2064339518547058,
      "logps/chosen": -300.8736572265625,
      "logps/rejected": -483.8716125488281,
      "loss": 0.4443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4604504108428955,
      "rewards/margins": 1.799198865890503,
      "rewards/rejected": -3.2596492767333984,
      "step": 10080
    },
    {
      "epoch": 1.3191625092009487,
      "grad_norm": 8.324992179870605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.016000017523765564,
      "logits/rejected": 0.13749545812606812,
      "logps/chosen": -483.5663757324219,
      "logps/rejected": -472.311279296875,
      "loss": 0.3753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4525991678237915,
      "rewards/margins": 1.5995991230010986,
      "rewards/rejected": -2.0521984100341797,
      "step": 10081
    },
    {
      "epoch": 1.3192933671383005,
      "grad_norm": 7.92312479019165,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4964148998260498,
      "logits/rejected": -0.2777417302131653,
      "logps/chosen": -494.91290283203125,
      "logps/rejected": -443.9636535644531,
      "loss": 0.4379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.721240758895874,
      "rewards/margins": 0.8620477914810181,
      "rewards/rejected": -1.5832886695861816,
      "step": 10082
    },
    {
      "epoch": 1.3194242250756523,
      "grad_norm": 15.531620979309082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23761308193206787,
      "logits/rejected": -0.2545631229877472,
      "logps/chosen": -485.0546569824219,
      "logps/rejected": -350.3995361328125,
      "loss": 0.8209,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4690524935722351,
      "rewards/margins": 0.1549827754497528,
      "rewards/rejected": -0.6240352988243103,
      "step": 10083
    },
    {
      "epoch": 1.319555083013004,
      "grad_norm": 9.767842292785645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06289045512676239,
      "logits/rejected": -0.2244405448436737,
      "logps/chosen": -422.9383850097656,
      "logps/rejected": -492.5951232910156,
      "loss": 0.4687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7229316830635071,
      "rewards/margins": 1.5016391277313232,
      "rewards/rejected": -2.2245709896087646,
      "step": 10084
    },
    {
      "epoch": 1.3196859409503559,
      "grad_norm": 10.82815170288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16900765895843506,
      "logits/rejected": -0.20853948593139648,
      "logps/chosen": -473.09564208984375,
      "logps/rejected": -496.6539306640625,
      "loss": 0.583,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8740525841712952,
      "rewards/margins": 0.7456741333007812,
      "rewards/rejected": -1.6197267770767212,
      "step": 10085
    },
    {
      "epoch": 1.3198167988877074,
      "grad_norm": 5.788688659667969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04749863222241402,
      "logits/rejected": -0.10798443853855133,
      "logps/chosen": -434.4256896972656,
      "logps/rejected": -427.432373046875,
      "loss": 0.2948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4307645857334137,
      "rewards/margins": 1.2677321434020996,
      "rewards/rejected": -1.6984965801239014,
      "step": 10086
    },
    {
      "epoch": 1.3199476568250592,
      "grad_norm": 4.522988796234131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007439741864800453,
      "logits/rejected": -0.16377000510692596,
      "logps/chosen": -502.03399658203125,
      "logps/rejected": -370.5089111328125,
      "loss": 0.2538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.555510401725769,
      "rewards/margins": 1.4284722805023193,
      "rewards/rejected": -1.9839826822280884,
      "step": 10087
    },
    {
      "epoch": 1.320078514762411,
      "grad_norm": 6.448364734649658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48129528760910034,
      "logits/rejected": -0.4123286008834839,
      "logps/chosen": -356.4166564941406,
      "logps/rejected": -442.7691650390625,
      "loss": 0.3684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11007939279079437,
      "rewards/margins": 1.5069899559020996,
      "rewards/rejected": -1.617069125175476,
      "step": 10088
    },
    {
      "epoch": 1.3202093726997628,
      "grad_norm": 4.480562210083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.511305570602417,
      "logits/rejected": -0.2815215587615967,
      "logps/chosen": -294.1611328125,
      "logps/rejected": -366.08251953125,
      "loss": 0.2514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4164198040962219,
      "rewards/margins": 2.1701414585113525,
      "rewards/rejected": -2.586561679840088,
      "step": 10089
    },
    {
      "epoch": 1.3203402306371146,
      "grad_norm": 4.2670464515686035,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2569097876548767,
      "logits/rejected": -0.3182237148284912,
      "logps/chosen": -545.3231201171875,
      "logps/rejected": -495.0495910644531,
      "loss": 0.1841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.41622599959373474,
      "rewards/margins": 2.2956926822662354,
      "rewards/rejected": -1.8794666528701782,
      "step": 10090
    },
    {
      "epoch": 1.3204710885744664,
      "grad_norm": 14.548929214477539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1926182061433792,
      "logits/rejected": -0.44012629985809326,
      "logps/chosen": -574.7833251953125,
      "logps/rejected": -496.5773010253906,
      "loss": 0.6988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1482021808624268,
      "rewards/margins": 0.9316638708114624,
      "rewards/rejected": -2.0798661708831787,
      "step": 10091
    },
    {
      "epoch": 1.3206019465118182,
      "grad_norm": 8.934194564819336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40727531909942627,
      "logits/rejected": -0.4605713188648224,
      "logps/chosen": -699.534912109375,
      "logps/rejected": -744.9441528320312,
      "loss": 0.4861,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6554019451141357,
      "rewards/margins": 0.7290288805961609,
      "rewards/rejected": -1.3844308853149414,
      "step": 10092
    },
    {
      "epoch": 1.32073280444917,
      "grad_norm": 8.053624153137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5921441316604614,
      "logits/rejected": -0.3912271559238434,
      "logps/chosen": -292.8622741699219,
      "logps/rejected": -361.4478759765625,
      "loss": 0.4663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1907755136489868,
      "rewards/margins": 1.9135782718658447,
      "rewards/rejected": -3.104353666305542,
      "step": 10093
    },
    {
      "epoch": 1.3208636623865218,
      "grad_norm": 7.460108280181885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3226696848869324,
      "logits/rejected": -0.2073008120059967,
      "logps/chosen": -564.1980590820312,
      "logps/rejected": -301.45574951171875,
      "loss": 0.4908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6833473443984985,
      "rewards/margins": 1.0536768436431885,
      "rewards/rejected": -1.7370240688323975,
      "step": 10094
    },
    {
      "epoch": 1.3209945203238733,
      "grad_norm": 6.268446445465088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24674300849437714,
      "logits/rejected": -0.37905529141426086,
      "logps/chosen": -395.6798400878906,
      "logps/rejected": -467.38336181640625,
      "loss": 0.3497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20855712890625,
      "rewards/margins": 1.9424517154693604,
      "rewards/rejected": -2.1510088443756104,
      "step": 10095
    },
    {
      "epoch": 1.3211253782612251,
      "grad_norm": 7.589801788330078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15671361982822418,
      "logits/rejected": -0.06568843126296997,
      "logps/chosen": -425.744384765625,
      "logps/rejected": -485.8792724609375,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1298622190952301,
      "rewards/margins": 1.1875219345092773,
      "rewards/rejected": -1.3173842430114746,
      "step": 10096
    },
    {
      "epoch": 1.321256236198577,
      "grad_norm": 6.099250793457031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04043189808726311,
      "logits/rejected": -0.4262002408504486,
      "logps/chosen": -530.810302734375,
      "logps/rejected": -435.8033142089844,
      "loss": 0.2932,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24394607543945312,
      "rewards/margins": 1.6399524211883545,
      "rewards/rejected": -1.883898377418518,
      "step": 10097
    },
    {
      "epoch": 1.3213870941359287,
      "grad_norm": 7.2525177001953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07838170975446701,
      "logits/rejected": -0.23614495992660522,
      "logps/chosen": -525.6890869140625,
      "logps/rejected": -614.3328247070312,
      "loss": 0.3298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8698175549507141,
      "rewards/margins": 1.56535005569458,
      "rewards/rejected": -2.4351675510406494,
      "step": 10098
    },
    {
      "epoch": 1.3215179520732805,
      "grad_norm": 10.296109199523926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05769950523972511,
      "logits/rejected": -0.11435642838478088,
      "logps/chosen": -738.2559204101562,
      "logps/rejected": -574.4093627929688,
      "loss": 0.6094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5681356191635132,
      "rewards/margins": 1.5078917741775513,
      "rewards/rejected": -2.0760273933410645,
      "step": 10099
    },
    {
      "epoch": 1.3216488100106323,
      "grad_norm": 5.629178047180176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2434009313583374,
      "logits/rejected": -0.6292211413383484,
      "logps/chosen": -459.1037902832031,
      "logps/rejected": -455.1192321777344,
      "loss": 0.3125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11127260327339172,
      "rewards/margins": 1.4888108968734741,
      "rewards/rejected": -1.6000837087631226,
      "step": 10100
    },
    {
      "epoch": 1.3217796679479838,
      "grad_norm": 3.856050968170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2817506194114685,
      "logits/rejected": -0.2917895019054413,
      "logps/chosen": -433.4684143066406,
      "logps/rejected": -413.8282165527344,
      "loss": 0.1846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2558309733867645,
      "rewards/margins": 2.5521066188812256,
      "rewards/rejected": -2.2962758541107178,
      "step": 10101
    },
    {
      "epoch": 1.3219105258853356,
      "grad_norm": 8.231058120727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13723361492156982,
      "logits/rejected": -0.3904087543487549,
      "logps/chosen": -471.9506530761719,
      "logps/rejected": -400.34930419921875,
      "loss": 0.4301,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9397140741348267,
      "rewards/margins": 1.3676817417144775,
      "rewards/rejected": -2.3073959350585938,
      "step": 10102
    },
    {
      "epoch": 1.3220413838226874,
      "grad_norm": 10.57236385345459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2926837205886841,
      "logits/rejected": -0.2129525989294052,
      "logps/chosen": -543.5712280273438,
      "logps/rejected": -490.69976806640625,
      "loss": 0.6745,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1254795789718628,
      "rewards/margins": 1.068595290184021,
      "rewards/rejected": -2.194075107574463,
      "step": 10103
    },
    {
      "epoch": 1.3221722417600392,
      "grad_norm": 8.214035987854004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11422360688447952,
      "logits/rejected": -0.10134990513324738,
      "logps/chosen": -566.3787231445312,
      "logps/rejected": -479.68243408203125,
      "loss": 0.4397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33053603768348694,
      "rewards/margins": 1.9640381336212158,
      "rewards/rejected": -2.29457426071167,
      "step": 10104
    },
    {
      "epoch": 1.322303099697391,
      "grad_norm": 6.001069068908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4350855052471161,
      "logits/rejected": -0.33738651871681213,
      "logps/chosen": -514.7443237304688,
      "logps/rejected": -411.8236083984375,
      "loss": 0.2183,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1199525594711304,
      "rewards/margins": 2.1649022102355957,
      "rewards/rejected": -3.2848546504974365,
      "step": 10105
    },
    {
      "epoch": 1.3224339576347428,
      "grad_norm": 6.473547458648682,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4516916871070862,
      "logits/rejected": -0.45775267481803894,
      "logps/chosen": -583.7869873046875,
      "logps/rejected": -565.9800415039062,
      "loss": 0.327,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43343985080718994,
      "rewards/margins": 1.4789124727249146,
      "rewards/rejected": -1.912352204322815,
      "step": 10106
    },
    {
      "epoch": 1.3225648155720946,
      "grad_norm": 10.2528657913208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2722510099411011,
      "logits/rejected": -0.1952296644449234,
      "logps/chosen": -507.56329345703125,
      "logps/rejected": -431.03118896484375,
      "loss": 0.452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0492932796478271,
      "rewards/margins": 1.4193105697631836,
      "rewards/rejected": -2.4686038494110107,
      "step": 10107
    },
    {
      "epoch": 1.3226956735094464,
      "grad_norm": 8.897370338439941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.049381934106349945,
      "logits/rejected": 0.034183256328105927,
      "logps/chosen": -511.4342041015625,
      "logps/rejected": -453.6910095214844,
      "loss": 0.5976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20415270328521729,
      "rewards/margins": 0.7221027612686157,
      "rewards/rejected": -0.926255464553833,
      "step": 10108
    },
    {
      "epoch": 1.3228265314467982,
      "grad_norm": 9.38371467590332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10863076150417328,
      "logits/rejected": 0.09473328292369843,
      "logps/chosen": -443.51715087890625,
      "logps/rejected": -579.6202392578125,
      "loss": 0.4497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8981055617332458,
      "rewards/margins": 1.6683495044708252,
      "rewards/rejected": -2.566455125808716,
      "step": 10109
    },
    {
      "epoch": 1.3229573893841498,
      "grad_norm": 11.015755653381348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2135041505098343,
      "logits/rejected": -0.2078554332256317,
      "logps/chosen": -614.1395874023438,
      "logps/rejected": -699.3756103515625,
      "loss": 0.4629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24333226680755615,
      "rewards/margins": 2.0355563163757324,
      "rewards/rejected": -2.278888702392578,
      "step": 10110
    },
    {
      "epoch": 1.3230882473215015,
      "grad_norm": 6.856493949890137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2444804459810257,
      "logits/rejected": -0.09908582270145416,
      "logps/chosen": -318.6783447265625,
      "logps/rejected": -335.2546691894531,
      "loss": 0.3137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1160314679145813,
      "rewards/margins": 1.939526081085205,
      "rewards/rejected": -1.8234946727752686,
      "step": 10111
    },
    {
      "epoch": 1.3232191052588533,
      "grad_norm": 10.6470365524292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1433584988117218,
      "logits/rejected": -0.05601773038506508,
      "logps/chosen": -498.47808837890625,
      "logps/rejected": -607.103515625,
      "loss": 0.5168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4755269289016724,
      "rewards/margins": 1.016706943511963,
      "rewards/rejected": -2.492233991622925,
      "step": 10112
    },
    {
      "epoch": 1.3233499631962051,
      "grad_norm": 3.794684648513794,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32489562034606934,
      "logits/rejected": -0.33304041624069214,
      "logps/chosen": -334.7130126953125,
      "logps/rejected": -306.2696228027344,
      "loss": 0.1886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3249311447143555,
      "rewards/margins": 1.7313082218170166,
      "rewards/rejected": -3.056239604949951,
      "step": 10113
    },
    {
      "epoch": 1.323480821133557,
      "grad_norm": 3.2765915393829346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1443261355161667,
      "logits/rejected": -0.31222179532051086,
      "logps/chosen": -447.5729064941406,
      "logps/rejected": -446.3254089355469,
      "loss": 0.1408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.44210225343704224,
      "rewards/margins": 2.970914363861084,
      "rewards/rejected": -3.4130167961120605,
      "step": 10114
    },
    {
      "epoch": 1.3236116790709087,
      "grad_norm": 11.300301551818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43626195192337036,
      "logits/rejected": -0.2670045495033264,
      "logps/chosen": -393.2417297363281,
      "logps/rejected": -528.1001586914062,
      "loss": 0.7978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0304079055786133,
      "rewards/margins": 0.22238484025001526,
      "rewards/rejected": -2.2527928352355957,
      "step": 10115
    },
    {
      "epoch": 1.3237425370082603,
      "grad_norm": 4.849100589752197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4108114242553711,
      "logits/rejected": -0.5049411058425903,
      "logps/chosen": -266.6322021484375,
      "logps/rejected": -401.99749755859375,
      "loss": 0.1385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0558677613735199,
      "rewards/margins": 2.6350326538085938,
      "rewards/rejected": -2.6909003257751465,
      "step": 10116
    },
    {
      "epoch": 1.323873394945612,
      "grad_norm": 9.661975860595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26900944113731384,
      "logits/rejected": -0.1294439733028412,
      "logps/chosen": -446.2318115234375,
      "logps/rejected": -491.744140625,
      "loss": 0.4142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6639034152030945,
      "rewards/margins": 1.7414171695709229,
      "rewards/rejected": -2.405320882797241,
      "step": 10117
    },
    {
      "epoch": 1.3240042528829639,
      "grad_norm": 6.016626358032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3203563690185547,
      "logits/rejected": -0.2266138792037964,
      "logps/chosen": -531.75146484375,
      "logps/rejected": -472.9010314941406,
      "loss": 0.3162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7241958975791931,
      "rewards/margins": 1.7492024898529053,
      "rewards/rejected": -2.473398447036743,
      "step": 10118
    },
    {
      "epoch": 1.3241351108203157,
      "grad_norm": 10.364871978759766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42709115147590637,
      "logits/rejected": -0.4950667917728424,
      "logps/chosen": -516.9620361328125,
      "logps/rejected": -547.6973266601562,
      "loss": 0.5865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1594514846801758,
      "rewards/margins": 0.6307131052017212,
      "rewards/rejected": -1.790164589881897,
      "step": 10119
    },
    {
      "epoch": 1.3242659687576674,
      "grad_norm": 6.892600059509277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15909238159656525,
      "logits/rejected": -0.09622159600257874,
      "logps/chosen": -683.6757202148438,
      "logps/rejected": -747.4532470703125,
      "loss": 0.2401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4468051493167877,
      "rewards/margins": 2.240922212600708,
      "rewards/rejected": -2.687727451324463,
      "step": 10120
    },
    {
      "epoch": 1.3243968266950192,
      "grad_norm": 12.053393363952637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1840723156929016,
      "logits/rejected": -0.2344236671924591,
      "logps/chosen": -469.8481750488281,
      "logps/rejected": -463.729736328125,
      "loss": 0.5708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6327798962593079,
      "rewards/margins": 1.9664655923843384,
      "rewards/rejected": -2.599245548248291,
      "step": 10121
    },
    {
      "epoch": 1.324527684632371,
      "grad_norm": 7.69351053237915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2834118604660034,
      "logits/rejected": -0.6104140877723694,
      "logps/chosen": -399.2681579589844,
      "logps/rejected": -402.6639709472656,
      "loss": 0.5016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5312919020652771,
      "rewards/margins": 0.9036504030227661,
      "rewards/rejected": -1.434942364692688,
      "step": 10122
    },
    {
      "epoch": 1.3246585425697228,
      "grad_norm": 8.598484992980957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14146788418293,
      "logits/rejected": -0.19386732578277588,
      "logps/chosen": -454.30853271484375,
      "logps/rejected": -536.4974975585938,
      "loss": 0.3627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4350581169128418,
      "rewards/margins": 1.6354310512542725,
      "rewards/rejected": -3.0704894065856934,
      "step": 10123
    },
    {
      "epoch": 1.3247894005070746,
      "grad_norm": 3.947026491165161,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04384119063615799,
      "logits/rejected": -0.05295100808143616,
      "logps/chosen": -582.0107421875,
      "logps/rejected": -431.94342041015625,
      "loss": 0.2245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4640117287635803,
      "rewards/margins": 2.2031912803649902,
      "rewards/rejected": -2.667202949523926,
      "step": 10124
    },
    {
      "epoch": 1.3249202584444264,
      "grad_norm": 8.315385818481445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12617629766464233,
      "logits/rejected": -0.05638442188501358,
      "logps/chosen": -446.5589294433594,
      "logps/rejected": -368.6699523925781,
      "loss": 0.5083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9410015344619751,
      "rewards/margins": 0.8633888363838196,
      "rewards/rejected": -1.80439031124115,
      "step": 10125
    },
    {
      "epoch": 1.325051116381778,
      "grad_norm": 10.046177864074707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.033821310847997665,
      "logits/rejected": -0.043941713869571686,
      "logps/chosen": -495.1351013183594,
      "logps/rejected": -570.9354248046875,
      "loss": 0.6443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2973976135253906,
      "rewards/margins": 1.2003003358840942,
      "rewards/rejected": -1.4976977109909058,
      "step": 10126
    },
    {
      "epoch": 1.3251819743191298,
      "grad_norm": 5.897691249847412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14064401388168335,
      "logits/rejected": -0.1631256341934204,
      "logps/chosen": -479.9501953125,
      "logps/rejected": -519.2044677734375,
      "loss": 0.229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7263935208320618,
      "rewards/margins": 1.882333755493164,
      "rewards/rejected": -2.60872745513916,
      "step": 10127
    },
    {
      "epoch": 1.3253128322564816,
      "grad_norm": 8.319903373718262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17116954922676086,
      "logits/rejected": -0.11247199773788452,
      "logps/chosen": -300.4209289550781,
      "logps/rejected": -312.24267578125,
      "loss": 0.4968,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9091593027114868,
      "rewards/margins": 0.6375092267990112,
      "rewards/rejected": -1.546668529510498,
      "step": 10128
    },
    {
      "epoch": 1.3254436901938333,
      "grad_norm": 5.221224308013916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5644503831863403,
      "logits/rejected": -0.38809749484062195,
      "logps/chosen": -297.81317138671875,
      "logps/rejected": -366.67852783203125,
      "loss": 0.3446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3018622398376465,
      "rewards/margins": 1.353121042251587,
      "rewards/rejected": -2.6549835205078125,
      "step": 10129
    },
    {
      "epoch": 1.3255745481311851,
      "grad_norm": 2.504349946975708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3535325527191162,
      "logits/rejected": -0.3204485774040222,
      "logps/chosen": -397.9319152832031,
      "logps/rejected": -630.9719848632812,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11629866808652878,
      "rewards/margins": 4.605166912078857,
      "rewards/rejected": -4.721465110778809,
      "step": 10130
    },
    {
      "epoch": 1.3257054060685367,
      "grad_norm": 9.507562637329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25335657596588135,
      "logits/rejected": -0.17765209078788757,
      "logps/chosen": -538.8831787109375,
      "logps/rejected": -670.1832275390625,
      "loss": 0.5346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4118797183036804,
      "rewards/margins": 0.5360778570175171,
      "rewards/rejected": -0.9479575157165527,
      "step": 10131
    },
    {
      "epoch": 1.3258362640058885,
      "grad_norm": 11.38951301574707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29106947779655457,
      "logits/rejected": -0.09472660720348358,
      "logps/chosen": -479.30682373046875,
      "logps/rejected": -559.3226318359375,
      "loss": 0.6358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.324884057044983,
      "rewards/margins": 0.30909615755081177,
      "rewards/rejected": -1.63398015499115,
      "step": 10132
    },
    {
      "epoch": 1.3259671219432403,
      "grad_norm": 12.467472076416016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28431522846221924,
      "logits/rejected": -0.14376315474510193,
      "logps/chosen": -440.9566345214844,
      "logps/rejected": -346.0048828125,
      "loss": 0.7381,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.570379614830017,
      "rewards/margins": 0.9555758237838745,
      "rewards/rejected": -2.5259554386138916,
      "step": 10133
    },
    {
      "epoch": 1.326097979880592,
      "grad_norm": 9.133733749389648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1357402801513672,
      "logits/rejected": 0.03969907760620117,
      "logps/chosen": -497.9955749511719,
      "logps/rejected": -495.7679748535156,
      "loss": 0.4716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3633440434932709,
      "rewards/margins": 1.2063353061676025,
      "rewards/rejected": -1.5696793794631958,
      "step": 10134
    },
    {
      "epoch": 1.3262288378179439,
      "grad_norm": 10.731616973876953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04568745940923691,
      "logits/rejected": -0.13456560671329498,
      "logps/chosen": -474.30780029296875,
      "logps/rejected": -507.97747802734375,
      "loss": 0.6224,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3454618752002716,
      "rewards/margins": 0.7976406216621399,
      "rewards/rejected": -1.1431025266647339,
      "step": 10135
    },
    {
      "epoch": 1.3263596957552957,
      "grad_norm": 7.42987585067749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44351646304130554,
      "logits/rejected": -0.28353193402290344,
      "logps/chosen": -483.0194396972656,
      "logps/rejected": -349.17926025390625,
      "loss": 0.4962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7219194769859314,
      "rewards/margins": 1.2269154787063599,
      "rewards/rejected": -1.948835015296936,
      "step": 10136
    },
    {
      "epoch": 1.3264905536926475,
      "grad_norm": 12.620651245117188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10463204234838486,
      "logits/rejected": -0.33473536372184753,
      "logps/chosen": -335.1202392578125,
      "logps/rejected": -372.0658874511719,
      "loss": 0.9293,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6831278800964355,
      "rewards/margins": 0.22324028611183167,
      "rewards/rejected": -0.9063681364059448,
      "step": 10137
    },
    {
      "epoch": 1.3266214116299992,
      "grad_norm": 9.762296676635742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14459365606307983,
      "logits/rejected": -0.1256614476442337,
      "logps/chosen": -408.33514404296875,
      "logps/rejected": -355.5359191894531,
      "loss": 0.772,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2396209239959717,
      "rewards/margins": 0.3238671123981476,
      "rewards/rejected": -1.563488245010376,
      "step": 10138
    },
    {
      "epoch": 1.326752269567351,
      "grad_norm": 5.945585250854492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01779690384864807,
      "logits/rejected": 0.012982705608010292,
      "logps/chosen": -446.1123352050781,
      "logps/rejected": -553.365234375,
      "loss": 0.2981,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1683729887008667,
      "rewards/margins": 1.5621238946914673,
      "rewards/rejected": -2.730496883392334,
      "step": 10139
    },
    {
      "epoch": 1.3268831275047028,
      "grad_norm": 9.418463706970215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3512570858001709,
      "logits/rejected": -0.4931211769580841,
      "logps/chosen": -464.14886474609375,
      "logps/rejected": -427.877197265625,
      "loss": 0.641,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4446601867675781,
      "rewards/margins": 0.8254955410957336,
      "rewards/rejected": -2.270155906677246,
      "step": 10140
    },
    {
      "epoch": 1.3270139854420544,
      "grad_norm": 5.669266223907471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33415457606315613,
      "logits/rejected": -0.31663113832473755,
      "logps/chosen": -502.4185791015625,
      "logps/rejected": -410.96368408203125,
      "loss": 0.258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9438577890396118,
      "rewards/margins": 2.0670692920684814,
      "rewards/rejected": -3.010927200317383,
      "step": 10141
    },
    {
      "epoch": 1.3271448433794062,
      "grad_norm": 7.077284812927246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28050315380096436,
      "logits/rejected": -0.4013230800628662,
      "logps/chosen": -305.5420837402344,
      "logps/rejected": -342.6513977050781,
      "loss": 0.3992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6965568661689758,
      "rewards/margins": 1.7544126510620117,
      "rewards/rejected": -2.4509694576263428,
      "step": 10142
    },
    {
      "epoch": 1.327275701316758,
      "grad_norm": 9.841278076171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2385341227054596,
      "logits/rejected": -0.37247997522354126,
      "logps/chosen": -712.8992309570312,
      "logps/rejected": -636.5216064453125,
      "loss": 0.5397,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5884354114532471,
      "rewards/margins": 0.9387658834457397,
      "rewards/rejected": -1.5272012948989868,
      "step": 10143
    },
    {
      "epoch": 1.3274065592541098,
      "grad_norm": 11.435529708862305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1918918490409851,
      "logits/rejected": -0.12875205278396606,
      "logps/chosen": -498.3672790527344,
      "logps/rejected": -496.61383056640625,
      "loss": 0.7666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9201486110687256,
      "rewards/margins": 0.5272455215454102,
      "rewards/rejected": -1.4473941326141357,
      "step": 10144
    },
    {
      "epoch": 1.3275374171914616,
      "grad_norm": 9.82840633392334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19191956520080566,
      "logits/rejected": -0.2865716218948364,
      "logps/chosen": -443.01995849609375,
      "logps/rejected": -501.0602111816406,
      "loss": 0.6623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03840998560190201,
      "rewards/margins": 1.6555440425872803,
      "rewards/rejected": -1.6171340942382812,
      "step": 10145
    },
    {
      "epoch": 1.3276682751288134,
      "grad_norm": 9.485538482666016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.592856764793396,
      "logits/rejected": 0.31094253063201904,
      "logps/chosen": -573.0371704101562,
      "logps/rejected": -519.6200561523438,
      "loss": 0.4006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7331615686416626,
      "rewards/margins": 1.4636530876159668,
      "rewards/rejected": -2.196814775466919,
      "step": 10146
    },
    {
      "epoch": 1.327799133066165,
      "grad_norm": 11.313525199890137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012892276048660278,
      "logits/rejected": -0.26920440793037415,
      "logps/chosen": -562.5514526367188,
      "logps/rejected": -383.6127624511719,
      "loss": 0.629,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6354627013206482,
      "rewards/margins": 1.2768855094909668,
      "rewards/rejected": -1.9123483896255493,
      "step": 10147
    },
    {
      "epoch": 1.3279299910035167,
      "grad_norm": 7.164376735687256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3275328576564789,
      "logits/rejected": -0.41414016485214233,
      "logps/chosen": -505.41229248046875,
      "logps/rejected": -399.1556701660156,
      "loss": 0.4107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36951637268066406,
      "rewards/margins": 1.0529571771621704,
      "rewards/rejected": -1.422473669052124,
      "step": 10148
    },
    {
      "epoch": 1.3280608489408685,
      "grad_norm": 8.123247146606445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18795403838157654,
      "logits/rejected": -0.3098718523979187,
      "logps/chosen": -408.8865051269531,
      "logps/rejected": -497.57965087890625,
      "loss": 0.4541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9066870212554932,
      "rewards/margins": 1.4103273153305054,
      "rewards/rejected": -2.317014455795288,
      "step": 10149
    },
    {
      "epoch": 1.3281917068782203,
      "grad_norm": 8.1462984085083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5020844340324402,
      "logits/rejected": -0.4233613908290863,
      "logps/chosen": -424.7210693359375,
      "logps/rejected": -563.8209228515625,
      "loss": 0.42,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9091631174087524,
      "rewards/margins": 1.1523133516311646,
      "rewards/rejected": -2.061476469039917,
      "step": 10150
    },
    {
      "epoch": 1.328322564815572,
      "grad_norm": 8.565771102905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3293108642101288,
      "logits/rejected": -0.3577785789966583,
      "logps/chosen": -433.1188049316406,
      "logps/rejected": -371.9450988769531,
      "loss": 0.4829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7396236658096313,
      "rewards/margins": 1.0851608514785767,
      "rewards/rejected": -1.824784517288208,
      "step": 10151
    },
    {
      "epoch": 1.3284534227529239,
      "grad_norm": 9.786628723144531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30200907588005066,
      "logits/rejected": -0.10967876017093658,
      "logps/chosen": -377.1964111328125,
      "logps/rejected": -509.3997802734375,
      "loss": 0.4351,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07093630731105804,
      "rewards/margins": 1.2255271673202515,
      "rewards/rejected": -1.2964633703231812,
      "step": 10152
    },
    {
      "epoch": 1.3285842806902757,
      "grad_norm": 7.14288854598999,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3554898202419281,
      "logits/rejected": -0.23158571124076843,
      "logps/chosen": -339.91143798828125,
      "logps/rejected": -311.9660949707031,
      "loss": 0.4892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2110702395439148,
      "rewards/margins": 0.7072025537490845,
      "rewards/rejected": -0.9182727336883545,
      "step": 10153
    },
    {
      "epoch": 1.3287151386276275,
      "grad_norm": 7.7980828285217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11451837420463562,
      "logits/rejected": -0.2146334946155548,
      "logps/chosen": -429.82818603515625,
      "logps/rejected": -497.396728515625,
      "loss": 0.3677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6108667850494385,
      "rewards/margins": 2.1979682445526123,
      "rewards/rejected": -2.8088347911834717,
      "step": 10154
    },
    {
      "epoch": 1.3288459965649793,
      "grad_norm": 7.484982967376709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03690291941165924,
      "logits/rejected": 0.10884984582662582,
      "logps/chosen": -547.2017822265625,
      "logps/rejected": -580.86474609375,
      "loss": 0.3103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8851556181907654,
      "rewards/margins": 1.256264567375183,
      "rewards/rejected": -2.141420364379883,
      "step": 10155
    },
    {
      "epoch": 1.3289768545023308,
      "grad_norm": 12.78754997253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10851704329252243,
      "logits/rejected": -0.06075849011540413,
      "logps/chosen": -687.02734375,
      "logps/rejected": -463.53961181640625,
      "loss": 0.5975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5522344708442688,
      "rewards/margins": 1.1209027767181396,
      "rewards/rejected": -1.6731373071670532,
      "step": 10156
    },
    {
      "epoch": 1.3291077124396826,
      "grad_norm": 3.874845504760742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1394883394241333,
      "logits/rejected": -0.2523737847805023,
      "logps/chosen": -492.29217529296875,
      "logps/rejected": -541.3768920898438,
      "loss": 0.1913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47159039974212646,
      "rewards/margins": 2.5621793270111084,
      "rewards/rejected": -3.0337696075439453,
      "step": 10157
    },
    {
      "epoch": 1.3292385703770344,
      "grad_norm": 12.32045841217041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2505735456943512,
      "logits/rejected": -0.31298696994781494,
      "logps/chosen": -423.70538330078125,
      "logps/rejected": -397.9600830078125,
      "loss": 0.6001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6859838366508484,
      "rewards/margins": 1.4720014333724976,
      "rewards/rejected": -2.157985210418701,
      "step": 10158
    },
    {
      "epoch": 1.3293694283143862,
      "grad_norm": 7.943310737609863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11851467937231064,
      "logits/rejected": -0.26770657300949097,
      "logps/chosen": -438.3886413574219,
      "logps/rejected": -425.7055969238281,
      "loss": 0.4304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5000681281089783,
      "rewards/margins": 1.5101450681686401,
      "rewards/rejected": -2.0102133750915527,
      "step": 10159
    },
    {
      "epoch": 1.329500286251738,
      "grad_norm": 3.568643569946289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1361592411994934,
      "logits/rejected": 0.08484749495983124,
      "logps/chosen": -524.207763671875,
      "logps/rejected": -543.6537475585938,
      "loss": 0.1747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.595036506652832,
      "rewards/margins": 2.03656005859375,
      "rewards/rejected": -2.631596326828003,
      "step": 10160
    },
    {
      "epoch": 1.3296311441890898,
      "grad_norm": 6.994185447692871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21409086883068085,
      "logits/rejected": -0.22926728427410126,
      "logps/chosen": -371.282958984375,
      "logps/rejected": -419.8394775390625,
      "loss": 0.4469,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7467048764228821,
      "rewards/margins": 1.3204288482666016,
      "rewards/rejected": -2.067133665084839,
      "step": 10161
    },
    {
      "epoch": 1.3297620021264414,
      "grad_norm": 9.7725248336792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5271936655044556,
      "logits/rejected": -0.5150694251060486,
      "logps/chosen": -461.0435485839844,
      "logps/rejected": -498.2129821777344,
      "loss": 0.4122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1735639572143555,
      "rewards/margins": 0.968241810798645,
      "rewards/rejected": -2.14180588722229,
      "step": 10162
    },
    {
      "epoch": 1.3298928600637931,
      "grad_norm": 5.108160495758057,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15501490235328674,
      "logits/rejected": -0.06329216808080673,
      "logps/chosen": -412.34228515625,
      "logps/rejected": -498.78863525390625,
      "loss": 0.222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8598090410232544,
      "rewards/margins": 1.919179916381836,
      "rewards/rejected": -2.77898907661438,
      "step": 10163
    },
    {
      "epoch": 1.330023718001145,
      "grad_norm": 14.724872589111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1782531887292862,
      "logits/rejected": -0.2562037408351898,
      "logps/chosen": -552.9208984375,
      "logps/rejected": -403.63470458984375,
      "loss": 0.5658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8459102511405945,
      "rewards/margins": 1.4681472778320312,
      "rewards/rejected": -2.3140573501586914,
      "step": 10164
    },
    {
      "epoch": 1.3301545759384967,
      "grad_norm": 7.414016246795654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19037839770317078,
      "logits/rejected": -0.18410971760749817,
      "logps/chosen": -539.0933837890625,
      "logps/rejected": -490.06646728515625,
      "loss": 0.3135,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.177148699760437,
      "rewards/margins": 1.6495087146759033,
      "rewards/rejected": -2.826657295227051,
      "step": 10165
    },
    {
      "epoch": 1.3302854338758485,
      "grad_norm": 6.608354568481445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051606547087430954,
      "logits/rejected": -0.18994908034801483,
      "logps/chosen": -609.3341674804688,
      "logps/rejected": -526.0319213867188,
      "loss": 0.3695,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15858523547649384,
      "rewards/margins": 1.8572160005569458,
      "rewards/rejected": -1.6986308097839355,
      "step": 10166
    },
    {
      "epoch": 1.3304162918132003,
      "grad_norm": 8.468496322631836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11494264006614685,
      "logits/rejected": -0.36161568760871887,
      "logps/chosen": -442.4820251464844,
      "logps/rejected": -418.7370300292969,
      "loss": 0.3835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3694956302642822,
      "rewards/margins": 1.3704378604888916,
      "rewards/rejected": -2.739933490753174,
      "step": 10167
    },
    {
      "epoch": 1.330547149750552,
      "grad_norm": 6.725549221038818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22312907874584198,
      "logits/rejected": -0.4142061471939087,
      "logps/chosen": -528.2567138671875,
      "logps/rejected": -387.8609619140625,
      "loss": 0.386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0488325357437134,
      "rewards/margins": 1.5043245553970337,
      "rewards/rejected": -2.553157091140747,
      "step": 10168
    },
    {
      "epoch": 1.330678007687904,
      "grad_norm": 7.692413330078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06221403181552887,
      "logits/rejected": -0.2039710283279419,
      "logps/chosen": -531.4278564453125,
      "logps/rejected": -423.92718505859375,
      "loss": 0.4038,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6644814014434814,
      "rewards/margins": 1.6558301448822021,
      "rewards/rejected": -2.3203115463256836,
      "step": 10169
    },
    {
      "epoch": 1.3308088656252557,
      "grad_norm": 7.339245319366455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09416675567626953,
      "logits/rejected": 0.016488509252667427,
      "logps/chosen": -362.0858459472656,
      "logps/rejected": -445.50604248046875,
      "loss": 0.3492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6956892609596252,
      "rewards/margins": 1.1291136741638184,
      "rewards/rejected": -1.8248027563095093,
      "step": 10170
    },
    {
      "epoch": 1.3309397235626075,
      "grad_norm": 6.662308216094971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41320544481277466,
      "logits/rejected": -0.3116222023963928,
      "logps/chosen": -460.9968566894531,
      "logps/rejected": -445.0977478027344,
      "loss": 0.3762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.998965859413147,
      "rewards/margins": 1.0305228233337402,
      "rewards/rejected": -2.0294885635375977,
      "step": 10171
    },
    {
      "epoch": 1.331070581499959,
      "grad_norm": 7.542446613311768,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09778239578008652,
      "logits/rejected": -0.26632532477378845,
      "logps/chosen": -396.20904541015625,
      "logps/rejected": -360.81268310546875,
      "loss": 0.4294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5339456796646118,
      "rewards/margins": 1.4032275676727295,
      "rewards/rejected": -1.9371730089187622,
      "step": 10172
    },
    {
      "epoch": 1.3312014394373108,
      "grad_norm": 15.310646057128906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21524283289909363,
      "logits/rejected": -0.29351621866226196,
      "logps/chosen": -554.6821899414062,
      "logps/rejected": -320.6717529296875,
      "loss": 1.0658,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0671300888061523,
      "rewards/margins": 0.026159286499023438,
      "rewards/rejected": -2.093289375305176,
      "step": 10173
    },
    {
      "epoch": 1.3313322973746626,
      "grad_norm": 7.058635234832764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.254024475812912,
      "logits/rejected": -0.5171153545379639,
      "logps/chosen": -366.7738952636719,
      "logps/rejected": -396.76300048828125,
      "loss": 0.3599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.023941159248352,
      "rewards/margins": 1.447465419769287,
      "rewards/rejected": -2.471406936645508,
      "step": 10174
    },
    {
      "epoch": 1.3314631553120144,
      "grad_norm": 6.607949256896973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11604394018650055,
      "logits/rejected": -0.22564038634300232,
      "logps/chosen": -436.27667236328125,
      "logps/rejected": -369.4610290527344,
      "loss": 0.3448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31723999977111816,
      "rewards/margins": 0.998805820941925,
      "rewards/rejected": -1.3160457611083984,
      "step": 10175
    },
    {
      "epoch": 1.3315940132493662,
      "grad_norm": 10.01516342163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3755584955215454,
      "logits/rejected": -0.27890509366989136,
      "logps/chosen": -407.32366943359375,
      "logps/rejected": -576.0107421875,
      "loss": 0.3742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6044408679008484,
      "rewards/margins": 2.5585901737213135,
      "rewards/rejected": -3.1630311012268066,
      "step": 10176
    },
    {
      "epoch": 1.3317248711867178,
      "grad_norm": 9.679390907287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07796768099069595,
      "logits/rejected": -0.17638935148715973,
      "logps/chosen": -687.487548828125,
      "logps/rejected": -475.0791320800781,
      "loss": 0.599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4014856219291687,
      "rewards/margins": 1.0677001476287842,
      "rewards/rejected": -1.4691859483718872,
      "step": 10177
    },
    {
      "epoch": 1.3318557291240696,
      "grad_norm": 7.956321716308594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5087977647781372,
      "logits/rejected": -0.5226906538009644,
      "logps/chosen": -366.09613037109375,
      "logps/rejected": -314.45953369140625,
      "loss": 0.4132,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0188153982162476,
      "rewards/margins": 1.3486000299453735,
      "rewards/rejected": -2.367415428161621,
      "step": 10178
    },
    {
      "epoch": 1.3319865870614214,
      "grad_norm": 6.1609110832214355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2497127652168274,
      "logits/rejected": -0.1504133641719818,
      "logps/chosen": -395.3954162597656,
      "logps/rejected": -469.4298400878906,
      "loss": 0.262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.029672637581825256,
      "rewards/margins": 1.8443820476531982,
      "rewards/rejected": -1.8147096633911133,
      "step": 10179
    },
    {
      "epoch": 1.3321174449987732,
      "grad_norm": 8.925898551940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.256458580493927,
      "logits/rejected": -0.19987362623214722,
      "logps/chosen": -375.8837890625,
      "logps/rejected": -498.0051574707031,
      "loss": 0.5786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.065672516822815,
      "rewards/margins": 1.0748475790023804,
      "rewards/rejected": -2.1405200958251953,
      "step": 10180
    },
    {
      "epoch": 1.332248302936125,
      "grad_norm": 5.924592018127441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27430981397628784,
      "logits/rejected": -0.3252037763595581,
      "logps/chosen": -331.7595520019531,
      "logps/rejected": -452.7675476074219,
      "loss": 0.3326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4933747947216034,
      "rewards/margins": 2.006758213043213,
      "rewards/rejected": -2.5001332759857178,
      "step": 10181
    },
    {
      "epoch": 1.3323791608734767,
      "grad_norm": 11.610268592834473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22840401530265808,
      "logits/rejected": -0.0057624876499176025,
      "logps/chosen": -463.9827575683594,
      "logps/rejected": -312.3326721191406,
      "loss": 0.5379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2931931018829346,
      "rewards/margins": 0.7954248189926147,
      "rewards/rejected": -2.088618040084839,
      "step": 10182
    },
    {
      "epoch": 1.3325100188108285,
      "grad_norm": 14.107941627502441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3329421877861023,
      "logits/rejected": -0.03060031682252884,
      "logps/chosen": -400.26385498046875,
      "logps/rejected": -448.40264892578125,
      "loss": 0.8427,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8353098034858704,
      "rewards/margins": 0.6952196359634399,
      "rewards/rejected": -1.530529499053955,
      "step": 10183
    },
    {
      "epoch": 1.3326408767481803,
      "grad_norm": 8.670024871826172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12959113717079163,
      "logits/rejected": -0.051975447684526443,
      "logps/chosen": -536.3471069335938,
      "logps/rejected": -536.2742919921875,
      "loss": 0.4846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41759949922561646,
      "rewards/margins": 0.5812292098999023,
      "rewards/rejected": -0.9988287091255188,
      "step": 10184
    },
    {
      "epoch": 1.3327717346855321,
      "grad_norm": 6.201230525970459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3083444833755493,
      "logits/rejected": -0.3907091021537781,
      "logps/chosen": -756.7900390625,
      "logps/rejected": -541.1732788085938,
      "loss": 0.311,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8266149759292603,
      "rewards/margins": 1.7028326988220215,
      "rewards/rejected": -2.529447555541992,
      "step": 10185
    },
    {
      "epoch": 1.332902592622884,
      "grad_norm": 3.785984992980957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08448569476604462,
      "logits/rejected": -0.4521731436252594,
      "logps/chosen": -443.9756774902344,
      "logps/rejected": -481.6036376953125,
      "loss": 0.1505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3364448547363281,
      "rewards/margins": 2.5519888401031494,
      "rewards/rejected": -2.8884336948394775,
      "step": 10186
    },
    {
      "epoch": 1.3330334505602355,
      "grad_norm": 5.791293144226074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28765928745269775,
      "logits/rejected": -0.3160751760005951,
      "logps/chosen": -526.6600952148438,
      "logps/rejected": -364.46356201171875,
      "loss": 0.2847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4277733266353607,
      "rewards/margins": 1.396996259689331,
      "rewards/rejected": -1.8247696161270142,
      "step": 10187
    },
    {
      "epoch": 1.3331643084975873,
      "grad_norm": 5.732300281524658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40722915530204773,
      "logits/rejected": -0.36907556653022766,
      "logps/chosen": -333.7732238769531,
      "logps/rejected": -262.955078125,
      "loss": 0.3372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05323600023984909,
      "rewards/margins": 1.2210644483566284,
      "rewards/rejected": -1.274300456047058,
      "step": 10188
    },
    {
      "epoch": 1.333295166434939,
      "grad_norm": 12.449422836303711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11317503452301025,
      "logits/rejected": -0.03287678211927414,
      "logps/chosen": -642.4263916015625,
      "logps/rejected": -550.178955078125,
      "loss": 0.7696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7241674661636353,
      "rewards/margins": 0.44089722633361816,
      "rewards/rejected": -1.1650646924972534,
      "step": 10189
    },
    {
      "epoch": 1.3334260243722909,
      "grad_norm": 6.078935146331787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22848796844482422,
      "logits/rejected": -0.11861450970172882,
      "logps/chosen": -441.07183837890625,
      "logps/rejected": -414.66522216796875,
      "loss": 0.4112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3813667297363281,
      "rewards/margins": 1.2068510055541992,
      "rewards/rejected": -1.5882177352905273,
      "step": 10190
    },
    {
      "epoch": 1.3335568823096426,
      "grad_norm": 5.714326858520508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11610535532236099,
      "logits/rejected": -0.14745914936065674,
      "logps/chosen": -518.6619873046875,
      "logps/rejected": -505.48095703125,
      "loss": 0.259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3400229215621948,
      "rewards/margins": 2.9258174896240234,
      "rewards/rejected": -4.265840530395508,
      "step": 10191
    },
    {
      "epoch": 1.3336877402469944,
      "grad_norm": 9.739148139953613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34560027718544006,
      "logits/rejected": -0.1646452397108078,
      "logps/chosen": -556.233154296875,
      "logps/rejected": -391.28924560546875,
      "loss": 0.5816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.462403655052185,
      "rewards/margins": 1.0464568138122559,
      "rewards/rejected": -2.5088603496551514,
      "step": 10192
    },
    {
      "epoch": 1.333818598184346,
      "grad_norm": 6.103989601135254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17045734822750092,
      "logits/rejected": -0.42503419518470764,
      "logps/chosen": -389.1090087890625,
      "logps/rejected": -550.4065551757812,
      "loss": 0.3134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.959673285484314,
      "rewards/margins": 2.2482738494873047,
      "rewards/rejected": -3.207947254180908,
      "step": 10193
    },
    {
      "epoch": 1.3339494561216978,
      "grad_norm": 7.660516262054443,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32945436239242554,
      "logits/rejected": -0.3418908715248108,
      "logps/chosen": -625.9791259765625,
      "logps/rejected": -365.6231384277344,
      "loss": 0.3252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.002259612083435,
      "rewards/margins": 1.3237429857254028,
      "rewards/rejected": -2.326002597808838,
      "step": 10194
    },
    {
      "epoch": 1.3340803140590496,
      "grad_norm": 9.707420349121094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33790960907936096,
      "logits/rejected": -0.15998344123363495,
      "logps/chosen": -335.2373962402344,
      "logps/rejected": -412.1385192871094,
      "loss": 0.5966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.290427803993225,
      "rewards/margins": 0.5081299543380737,
      "rewards/rejected": -1.7985578775405884,
      "step": 10195
    },
    {
      "epoch": 1.3342111719964014,
      "grad_norm": 6.923434257507324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17234277725219727,
      "logits/rejected": -0.2965036630630493,
      "logps/chosen": -374.84832763671875,
      "logps/rejected": -435.3070068359375,
      "loss": 0.2863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2809555530548096,
      "rewards/margins": 2.169630765914917,
      "rewards/rejected": -3.4505863189697266,
      "step": 10196
    },
    {
      "epoch": 1.3343420299337532,
      "grad_norm": 4.745713710784912,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14260989427566528,
      "logits/rejected": -0.17912565171718597,
      "logps/chosen": -595.3087158203125,
      "logps/rejected": -499.89825439453125,
      "loss": 0.2646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.255099892616272,
      "rewards/margins": 1.4369455575942993,
      "rewards/rejected": -1.6920453310012817,
      "step": 10197
    },
    {
      "epoch": 1.334472887871105,
      "grad_norm": 6.748321533203125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37363752722740173,
      "logits/rejected": -0.2852588891983032,
      "logps/chosen": -426.3556213378906,
      "logps/rejected": -457.72576904296875,
      "loss": 0.3707,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7993301749229431,
      "rewards/margins": 1.7590023279190063,
      "rewards/rejected": -2.5583324432373047,
      "step": 10198
    },
    {
      "epoch": 1.3346037458084568,
      "grad_norm": 7.118828296661377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2403518259525299,
      "logits/rejected": -0.10685737431049347,
      "logps/chosen": -361.3121643066406,
      "logps/rejected": -513.0620727539062,
      "loss": 0.3744,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1106197834014893,
      "rewards/margins": 1.4084478616714478,
      "rewards/rejected": -2.5190677642822266,
      "step": 10199
    },
    {
      "epoch": 1.3347346037458085,
      "grad_norm": 7.969395637512207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07774079591035843,
      "logits/rejected": -0.3133840560913086,
      "logps/chosen": -437.9001159667969,
      "logps/rejected": -373.304931640625,
      "loss": 0.5247,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.619293451309204,
      "rewards/margins": 2.078165292739868,
      "rewards/rejected": -3.6974587440490723,
      "step": 10200
    },
    {
      "epoch": 1.3348654616831603,
      "grad_norm": 5.79909610748291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5090106129646301,
      "logits/rejected": -0.5685076713562012,
      "logps/chosen": -487.5218505859375,
      "logps/rejected": -476.17083740234375,
      "loss": 0.2857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8455644845962524,
      "rewards/margins": 1.4308478832244873,
      "rewards/rejected": -2.27641224861145,
      "step": 10201
    },
    {
      "epoch": 1.334996319620512,
      "grad_norm": 4.710203170776367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24138642847537994,
      "logits/rejected": -0.32664161920547485,
      "logps/chosen": -446.0714416503906,
      "logps/rejected": -330.74517822265625,
      "loss": 0.2419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8347448110580444,
      "rewards/margins": 2.044224739074707,
      "rewards/rejected": -2.878969669342041,
      "step": 10202
    },
    {
      "epoch": 1.3351271775578637,
      "grad_norm": 10.663790702819824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23581019043922424,
      "logits/rejected": -0.5352585315704346,
      "logps/chosen": -535.7713623046875,
      "logps/rejected": -429.9867858886719,
      "loss": 0.8102,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2534586191177368,
      "rewards/margins": 0.8624241352081299,
      "rewards/rejected": -2.1158828735351562,
      "step": 10203
    },
    {
      "epoch": 1.3352580354952155,
      "grad_norm": 10.10611343383789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16241443157196045,
      "logits/rejected": -0.4206690490245819,
      "logps/chosen": -418.9324645996094,
      "logps/rejected": -292.27679443359375,
      "loss": 0.5127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3189085125923157,
      "rewards/margins": 1.4752635955810547,
      "rewards/rejected": -1.794171929359436,
      "step": 10204
    },
    {
      "epoch": 1.3353888934325673,
      "grad_norm": 9.960801124572754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40755122900009155,
      "logits/rejected": -0.2911309003829956,
      "logps/chosen": -556.6407470703125,
      "logps/rejected": -425.457763671875,
      "loss": 0.6048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5479351282119751,
      "rewards/margins": 0.968085765838623,
      "rewards/rejected": -1.5160207748413086,
      "step": 10205
    },
    {
      "epoch": 1.335519751369919,
      "grad_norm": 6.804731845855713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2937851548194885,
      "logits/rejected": -0.25643932819366455,
      "logps/chosen": -495.8861389160156,
      "logps/rejected": -694.7059936523438,
      "loss": 0.3205,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3644757270812988,
      "rewards/margins": 1.6152515411376953,
      "rewards/rejected": -2.979727268218994,
      "step": 10206
    },
    {
      "epoch": 1.3356506093072709,
      "grad_norm": 8.309736251831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18629670143127441,
      "logits/rejected": -0.23301583528518677,
      "logps/chosen": -447.48687744140625,
      "logps/rejected": -551.1951904296875,
      "loss": 0.4237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5780317783355713,
      "rewards/margins": 1.8286497592926025,
      "rewards/rejected": -4.406681537628174,
      "step": 10207
    },
    {
      "epoch": 1.3357814672446224,
      "grad_norm": 10.929327964782715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05343412607908249,
      "logits/rejected": -0.07209381461143494,
      "logps/chosen": -532.9603881835938,
      "logps/rejected": -423.23919677734375,
      "loss": 0.6278,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12747040390968323,
      "rewards/margins": 1.1497809886932373,
      "rewards/rejected": -1.2772513628005981,
      "step": 10208
    },
    {
      "epoch": 1.3359123251819742,
      "grad_norm": 7.239588260650635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3739248812198639,
      "logits/rejected": -0.5392076969146729,
      "logps/chosen": -384.7314147949219,
      "logps/rejected": -355.7087707519531,
      "loss": 0.4072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9860607385635376,
      "rewards/margins": 1.8502966165542603,
      "rewards/rejected": -2.836357593536377,
      "step": 10209
    },
    {
      "epoch": 1.336043183119326,
      "grad_norm": 8.796236038208008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23301105201244354,
      "logits/rejected": -0.322967529296875,
      "logps/chosen": -351.88421630859375,
      "logps/rejected": -440.4158020019531,
      "loss": 0.451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3501856327056885,
      "rewards/margins": 1.630998134613037,
      "rewards/rejected": -1.981183648109436,
      "step": 10210
    },
    {
      "epoch": 1.3361740410566778,
      "grad_norm": 4.605382442474365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48208361864089966,
      "logits/rejected": -0.31851744651794434,
      "logps/chosen": -499.40667724609375,
      "logps/rejected": -597.8790893554688,
      "loss": 0.2694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.74005126953125,
      "rewards/margins": 1.44639253616333,
      "rewards/rejected": -2.18644380569458,
      "step": 10211
    },
    {
      "epoch": 1.3363048989940296,
      "grad_norm": 9.137544631958008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21553048491477966,
      "logits/rejected": 0.15269409120082855,
      "logps/chosen": -558.1503295898438,
      "logps/rejected": -680.1080932617188,
      "loss": 0.3196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.35796058177948,
      "rewards/margins": 1.7432239055633545,
      "rewards/rejected": -3.101184606552124,
      "step": 10212
    },
    {
      "epoch": 1.3364357569313814,
      "grad_norm": 8.63467025756836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22485613822937012,
      "logits/rejected": -0.16913755238056183,
      "logps/chosen": -456.8711853027344,
      "logps/rejected": -433.15362548828125,
      "loss": 0.3914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4130971431732178,
      "rewards/margins": 1.1795860528945923,
      "rewards/rejected": -2.5926830768585205,
      "step": 10213
    },
    {
      "epoch": 1.3365666148687332,
      "grad_norm": 9.264244079589844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1013755053281784,
      "logits/rejected": 0.0632062628865242,
      "logps/chosen": -513.8834838867188,
      "logps/rejected": -542.2369384765625,
      "loss": 0.812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8071157336235046,
      "rewards/margins": 0.5745820999145508,
      "rewards/rejected": 0.23253366351127625,
      "step": 10214
    },
    {
      "epoch": 1.336697472806085,
      "grad_norm": 7.452754497528076,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02290293201804161,
      "logits/rejected": -0.24949580430984497,
      "logps/chosen": -457.93426513671875,
      "logps/rejected": -399.4698486328125,
      "loss": 0.2486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33113235235214233,
      "rewards/margins": 3.1452736854553223,
      "rewards/rejected": -3.4764060974121094,
      "step": 10215
    },
    {
      "epoch": 1.3368283307434368,
      "grad_norm": 7.246279239654541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05097413808107376,
      "logits/rejected": -0.15718266367912292,
      "logps/chosen": -622.0249633789062,
      "logps/rejected": -518.7733764648438,
      "loss": 0.4229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6864364147186279,
      "rewards/margins": 1.0837182998657227,
      "rewards/rejected": -1.7701549530029297,
      "step": 10216
    },
    {
      "epoch": 1.3369591886807886,
      "grad_norm": 13.700837135314941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08051969856023788,
      "logits/rejected": -0.20771834254264832,
      "logps/chosen": -470.3755187988281,
      "logps/rejected": -386.8220520019531,
      "loss": 0.6291,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9624593257904053,
      "rewards/margins": 0.8560582995414734,
      "rewards/rejected": -1.8185176849365234,
      "step": 10217
    },
    {
      "epoch": 1.3370900466181401,
      "grad_norm": 4.730661392211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16164089739322662,
      "logits/rejected": -0.319166362285614,
      "logps/chosen": -355.0223083496094,
      "logps/rejected": -499.93963623046875,
      "loss": 0.26,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1697520017623901,
      "rewards/margins": 2.128535747528076,
      "rewards/rejected": -3.298287868499756,
      "step": 10218
    },
    {
      "epoch": 1.337220904555492,
      "grad_norm": 8.554000854492188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04824451729655266,
      "logits/rejected": -0.29017361998558044,
      "logps/chosen": -613.98388671875,
      "logps/rejected": -572.9118041992188,
      "loss": 0.5001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.592392921447754,
      "rewards/margins": 1.1809370517730713,
      "rewards/rejected": -3.773329734802246,
      "step": 10219
    },
    {
      "epoch": 1.3373517624928437,
      "grad_norm": 6.112646102905273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09763140976428986,
      "logits/rejected": 0.04737911745905876,
      "logps/chosen": -383.10272216796875,
      "logps/rejected": -457.8701171875,
      "loss": 0.2742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15947988629341125,
      "rewards/margins": 1.9818360805511475,
      "rewards/rejected": -2.1413159370422363,
      "step": 10220
    },
    {
      "epoch": 1.3374826204301955,
      "grad_norm": 8.727884292602539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2695285379886627,
      "logits/rejected": -0.41569870710372925,
      "logps/chosen": -379.24456787109375,
      "logps/rejected": -540.5394287109375,
      "loss": 0.5472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1236523389816284,
      "rewards/margins": 1.349782943725586,
      "rewards/rejected": -2.473435401916504,
      "step": 10221
    },
    {
      "epoch": 1.3376134783675473,
      "grad_norm": 10.292251586914062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21465329825878143,
      "logits/rejected": -0.21851006150245667,
      "logps/chosen": -423.3102722167969,
      "logps/rejected": -443.27783203125,
      "loss": 0.5846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.016300082206726,
      "rewards/margins": 1.512521505355835,
      "rewards/rejected": -2.5288212299346924,
      "step": 10222
    },
    {
      "epoch": 1.3377443363048989,
      "grad_norm": 6.104899883270264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13020211458206177,
      "logits/rejected": 0.032873183488845825,
      "logps/chosen": -466.0423278808594,
      "logps/rejected": -476.1241455078125,
      "loss": 0.2141,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9954260587692261,
      "rewards/margins": 2.398775100708008,
      "rewards/rejected": -3.3942012786865234,
      "step": 10223
    },
    {
      "epoch": 1.3378751942422507,
      "grad_norm": 9.058016777038574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3528715968132019,
      "logits/rejected": -0.46803292632102966,
      "logps/chosen": -332.24346923828125,
      "logps/rejected": -345.3021240234375,
      "loss": 0.3837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2367676496505737,
      "rewards/margins": 1.1208685636520386,
      "rewards/rejected": -2.357635974884033,
      "step": 10224
    },
    {
      "epoch": 1.3380060521796024,
      "grad_norm": 7.459941864013672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7114289402961731,
      "logits/rejected": -0.7411640882492065,
      "logps/chosen": -402.58306884765625,
      "logps/rejected": -377.7648010253906,
      "loss": 0.4229,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9935110211372375,
      "rewards/margins": 1.2825980186462402,
      "rewards/rejected": -2.276108980178833,
      "step": 10225
    },
    {
      "epoch": 1.3381369101169542,
      "grad_norm": 7.912206649780273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14800409972667694,
      "logits/rejected": -0.07910879701375961,
      "logps/chosen": -377.8143615722656,
      "logps/rejected": -544.4011840820312,
      "loss": 0.4824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0896745920181274,
      "rewards/margins": 1.4894635677337646,
      "rewards/rejected": -2.5791380405426025,
      "step": 10226
    },
    {
      "epoch": 1.338267768054306,
      "grad_norm": 5.498274326324463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1428612470626831,
      "logits/rejected": 0.2822962701320648,
      "logps/chosen": -444.340576171875,
      "logps/rejected": -505.0076599121094,
      "loss": 0.2525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3381274342536926,
      "rewards/margins": 3.023961305618286,
      "rewards/rejected": -2.6858339309692383,
      "step": 10227
    },
    {
      "epoch": 1.3383986259916578,
      "grad_norm": 12.957169532775879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38128048181533813,
      "logits/rejected": -0.3660389184951782,
      "logps/chosen": -545.9033813476562,
      "logps/rejected": -524.868896484375,
      "loss": 0.9327,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.282410740852356,
      "rewards/margins": 0.3874971568584442,
      "rewards/rejected": -1.669908046722412,
      "step": 10228
    },
    {
      "epoch": 1.3385294839290096,
      "grad_norm": 9.41955852508545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38298875093460083,
      "logits/rejected": -0.4351390600204468,
      "logps/chosen": -562.0072021484375,
      "logps/rejected": -553.6223754882812,
      "loss": 0.6155,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6502944231033325,
      "rewards/margins": 1.483187198638916,
      "rewards/rejected": -3.133481502532959,
      "step": 10229
    },
    {
      "epoch": 1.3386603418663614,
      "grad_norm": 6.125267028808594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24052727222442627,
      "logits/rejected": -0.3148055970668793,
      "logps/chosen": -415.40533447265625,
      "logps/rejected": -546.9365234375,
      "loss": 0.2793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01742781139910221,
      "rewards/margins": 2.4133007526397705,
      "rewards/rejected": -2.4307286739349365,
      "step": 10230
    },
    {
      "epoch": 1.3387911998037132,
      "grad_norm": 10.300529479980469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05247107148170471,
      "logits/rejected": -0.285084992647171,
      "logps/chosen": -698.679443359375,
      "logps/rejected": -749.4393310546875,
      "loss": 0.6221,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0973505973815918,
      "rewards/margins": 0.9015901684761047,
      "rewards/rejected": -1.9989407062530518,
      "step": 10231
    },
    {
      "epoch": 1.338922057741065,
      "grad_norm": 7.256496906280518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12047966569662094,
      "logits/rejected": -0.4413765072822571,
      "logps/chosen": -633.8170166015625,
      "logps/rejected": -641.885986328125,
      "loss": 0.3325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5549810528755188,
      "rewards/margins": 1.2180832624435425,
      "rewards/rejected": -1.773064374923706,
      "step": 10232
    },
    {
      "epoch": 1.3390529156784166,
      "grad_norm": 8.204384803771973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29698002338409424,
      "logits/rejected": -0.40058043599128723,
      "logps/chosen": -470.87225341796875,
      "logps/rejected": -517.166015625,
      "loss": 0.4576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6311585903167725,
      "rewards/margins": 0.9040924310684204,
      "rewards/rejected": -2.5352511405944824,
      "step": 10233
    },
    {
      "epoch": 1.3391837736157683,
      "grad_norm": 9.722451210021973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3409503102302551,
      "logits/rejected": -0.4237172603607178,
      "logps/chosen": -597.3973388671875,
      "logps/rejected": -478.16412353515625,
      "loss": 0.6055,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47701433300971985,
      "rewards/margins": 1.0918865203857422,
      "rewards/rejected": -1.5689009428024292,
      "step": 10234
    },
    {
      "epoch": 1.3393146315531201,
      "grad_norm": 7.6920485496521,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08135942369699478,
      "logits/rejected": -0.30538931488990784,
      "logps/chosen": -702.1844482421875,
      "logps/rejected": -485.4959716796875,
      "loss": 0.4133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43622586131095886,
      "rewards/margins": 1.6103004217147827,
      "rewards/rejected": -1.174074649810791,
      "step": 10235
    },
    {
      "epoch": 1.339445489490472,
      "grad_norm": 8.410810470581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1490497887134552,
      "logits/rejected": -0.08579594641923904,
      "logps/chosen": -287.51312255859375,
      "logps/rejected": -359.60888671875,
      "loss": 0.7246,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7175005078315735,
      "rewards/margins": 0.4579814672470093,
      "rewards/rejected": -1.1754820346832275,
      "step": 10236
    },
    {
      "epoch": 1.3395763474278237,
      "grad_norm": 6.679961204528809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08622851967811584,
      "logits/rejected": -0.2162618637084961,
      "logps/chosen": -396.12750244140625,
      "logps/rejected": -357.33465576171875,
      "loss": 0.502,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5079115033149719,
      "rewards/margins": 1.2818074226379395,
      "rewards/rejected": -1.7897188663482666,
      "step": 10237
    },
    {
      "epoch": 1.3397072053651755,
      "grad_norm": 8.429766654968262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2229311168193817,
      "logits/rejected": -0.19772598147392273,
      "logps/chosen": -658.6408081054688,
      "logps/rejected": -667.952880859375,
      "loss": 0.5342,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11613407731056213,
      "rewards/margins": 1.142165184020996,
      "rewards/rejected": -1.0260311365127563,
      "step": 10238
    },
    {
      "epoch": 1.339838063302527,
      "grad_norm": 7.0283942222595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1287112683057785,
      "logits/rejected": -0.05258680880069733,
      "logps/chosen": -605.8714599609375,
      "logps/rejected": -670.905517578125,
      "loss": 0.4215,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2987808287143707,
      "rewards/margins": 1.2191524505615234,
      "rewards/rejected": -1.5179333686828613,
      "step": 10239
    },
    {
      "epoch": 1.3399689212398789,
      "grad_norm": 3.781947612762451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2866378724575043,
      "logits/rejected": -0.198248028755188,
      "logps/chosen": -452.2518310546875,
      "logps/rejected": -532.9373168945312,
      "loss": 0.151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.029557425528764725,
      "rewards/margins": 2.6915578842163086,
      "rewards/rejected": -2.6620006561279297,
      "step": 10240
    },
    {
      "epoch": 1.3400997791772307,
      "grad_norm": 8.486979484558105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021961623802781105,
      "logits/rejected": -0.4303241968154907,
      "logps/chosen": -591.1995239257812,
      "logps/rejected": -495.168701171875,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.203392505645752,
      "rewards/margins": 1.1715078353881836,
      "rewards/rejected": -2.3749003410339355,
      "step": 10241
    },
    {
      "epoch": 1.3402306371145825,
      "grad_norm": 11.958280563354492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30355238914489746,
      "logits/rejected": 0.09079821407794952,
      "logps/chosen": -435.4620361328125,
      "logps/rejected": -425.44342041015625,
      "loss": 0.8844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3844300508499146,
      "rewards/margins": 0.25901567935943604,
      "rewards/rejected": -1.6434454917907715,
      "step": 10242
    },
    {
      "epoch": 1.3403614950519342,
      "grad_norm": 8.271163940429688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4838041663169861,
      "logits/rejected": -0.7130244374275208,
      "logps/chosen": -366.743896484375,
      "logps/rejected": -456.8799743652344,
      "loss": 0.4846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5908351540565491,
      "rewards/margins": 1.2062749862670898,
      "rewards/rejected": -1.7971101999282837,
      "step": 10243
    },
    {
      "epoch": 1.340492352989286,
      "grad_norm": 12.752589225769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053166523575782776,
      "logits/rejected": -0.16835249960422516,
      "logps/chosen": -539.7384033203125,
      "logps/rejected": -489.9689025878906,
      "loss": 0.6519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3459076881408691,
      "rewards/margins": 0.7075815200805664,
      "rewards/rejected": -2.0534892082214355,
      "step": 10244
    },
    {
      "epoch": 1.3406232109266378,
      "grad_norm": 8.727383613586426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17133828997612,
      "logits/rejected": -0.27775686979293823,
      "logps/chosen": -356.84381103515625,
      "logps/rejected": -365.6043395996094,
      "loss": 0.4918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0035459995269775,
      "rewards/margins": 1.4975571632385254,
      "rewards/rejected": -2.501103162765503,
      "step": 10245
    },
    {
      "epoch": 1.3407540688639896,
      "grad_norm": 8.966815948486328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04416763409972191,
      "logits/rejected": -0.2387872040271759,
      "logps/chosen": -592.7794189453125,
      "logps/rejected": -445.3046875,
      "loss": 0.4103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4398807287216187,
      "rewards/margins": 0.9920284748077393,
      "rewards/rejected": -2.4319090843200684,
      "step": 10246
    },
    {
      "epoch": 1.3408849268013414,
      "grad_norm": 8.206579208374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2868109345436096,
      "logits/rejected": -0.29244980216026306,
      "logps/chosen": -323.1846618652344,
      "logps/rejected": -347.61346435546875,
      "loss": 0.3502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.150325894355774,
      "rewards/margins": 1.5603792667388916,
      "rewards/rejected": -2.710705280303955,
      "step": 10247
    },
    {
      "epoch": 1.341015784738693,
      "grad_norm": 13.086519241333008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07230842858552933,
      "logits/rejected": -0.3837919235229492,
      "logps/chosen": -325.96142578125,
      "logps/rejected": -418.2552795410156,
      "loss": 0.9023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6956307888031006,
      "rewards/margins": 0.8186374306678772,
      "rewards/rejected": -2.514268398284912,
      "step": 10248
    },
    {
      "epoch": 1.3411466426760448,
      "grad_norm": 5.741537570953369,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1288965791463852,
      "logits/rejected": 0.04988975077867508,
      "logps/chosen": -341.84375,
      "logps/rejected": -350.93927001953125,
      "loss": 0.3325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8904564380645752,
      "rewards/margins": 1.324386477470398,
      "rewards/rejected": -2.2148430347442627,
      "step": 10249
    },
    {
      "epoch": 1.3412775006133966,
      "grad_norm": 8.374017715454102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044563136994838715,
      "logits/rejected": -0.1441926807165146,
      "logps/chosen": -567.8739624023438,
      "logps/rejected": -592.5689086914062,
      "loss": 0.4899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9203174114227295,
      "rewards/margins": 1.025626301765442,
      "rewards/rejected": -2.945943832397461,
      "step": 10250
    },
    {
      "epoch": 1.3414083585507484,
      "grad_norm": 13.752729415893555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5555850267410278,
      "logits/rejected": -0.43887966871261597,
      "logps/chosen": -546.1228637695312,
      "logps/rejected": -416.7007141113281,
      "loss": 0.8023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4108905792236328,
      "rewards/margins": 0.7807950973510742,
      "rewards/rejected": -2.191685676574707,
      "step": 10251
    },
    {
      "epoch": 1.3415392164881002,
      "grad_norm": 10.067289352416992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20781512558460236,
      "logits/rejected": -0.19920532405376434,
      "logps/chosen": -467.4107666015625,
      "logps/rejected": -450.3061828613281,
      "loss": 0.6199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.236718773841858,
      "rewards/margins": 0.9207010269165039,
      "rewards/rejected": -2.1574199199676514,
      "step": 10252
    },
    {
      "epoch": 1.341670074425452,
      "grad_norm": 3.2155327796936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05393746867775917,
      "logits/rejected": -0.14734916388988495,
      "logps/chosen": -446.2334289550781,
      "logps/rejected": -459.6099548339844,
      "loss": 0.1514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1552295684814453,
      "rewards/margins": 2.4628636837005615,
      "rewards/rejected": -2.618093729019165,
      "step": 10253
    },
    {
      "epoch": 1.3418009323628035,
      "grad_norm": 12.649251937866211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4195050895214081,
      "logits/rejected": -0.2287500500679016,
      "logps/chosen": -503.3382568359375,
      "logps/rejected": -455.0799560546875,
      "loss": 0.5841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.53107488155365,
      "rewards/margins": 1.2409815788269043,
      "rewards/rejected": -2.7720565795898438,
      "step": 10254
    },
    {
      "epoch": 1.3419317903001553,
      "grad_norm": 12.912742614746094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11791086196899414,
      "logits/rejected": -0.2262798547744751,
      "logps/chosen": -645.4501953125,
      "logps/rejected": -479.4226379394531,
      "loss": 0.7571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2264654636383057,
      "rewards/margins": 0.5195273160934448,
      "rewards/rejected": -1.745992660522461,
      "step": 10255
    },
    {
      "epoch": 1.342062648237507,
      "grad_norm": 10.655616760253906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33161023259162903,
      "logits/rejected": -0.45429712533950806,
      "logps/chosen": -429.71136474609375,
      "logps/rejected": -335.92138671875,
      "loss": 0.4598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2635799646377563,
      "rewards/margins": 1.1614162921905518,
      "rewards/rejected": -2.4249963760375977,
      "step": 10256
    },
    {
      "epoch": 1.3421935061748589,
      "grad_norm": 7.610806941986084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3385167121887207,
      "logits/rejected": -0.09149129688739777,
      "logps/chosen": -335.08343505859375,
      "logps/rejected": -329.06121826171875,
      "loss": 0.4161,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0004909038543701,
      "rewards/margins": 1.278520941734314,
      "rewards/rejected": -2.2790117263793945,
      "step": 10257
    },
    {
      "epoch": 1.3423243641122107,
      "grad_norm": 9.2870454788208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042126115411520004,
      "logits/rejected": -0.0460645891726017,
      "logps/chosen": -446.4521179199219,
      "logps/rejected": -417.200927734375,
      "loss": 0.5875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2910287380218506,
      "rewards/margins": 0.625410795211792,
      "rewards/rejected": -2.9164397716522217,
      "step": 10258
    },
    {
      "epoch": 1.3424552220495625,
      "grad_norm": 9.429347038269043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0026987791061401367,
      "logits/rejected": -0.1444045603275299,
      "logps/chosen": -536.3201904296875,
      "logps/rejected": -493.26824951171875,
      "loss": 0.4917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8045666217803955,
      "rewards/margins": 1.1979823112487793,
      "rewards/rejected": -2.002548933029175,
      "step": 10259
    },
    {
      "epoch": 1.3425860799869143,
      "grad_norm": 10.974794387817383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30449649691581726,
      "logits/rejected": -0.2692594826221466,
      "logps/chosen": -421.2101745605469,
      "logps/rejected": -405.08233642578125,
      "loss": 0.6037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2681384086608887,
      "rewards/margins": 0.5074490904808044,
      "rewards/rejected": -2.775587558746338,
      "step": 10260
    },
    {
      "epoch": 1.342716937924266,
      "grad_norm": 11.526031494140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.227279394865036,
      "logits/rejected": -0.34060847759246826,
      "logps/chosen": -568.2921752929688,
      "logps/rejected": -479.0117492675781,
      "loss": 0.4988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2352802753448486,
      "rewards/margins": 1.3822600841522217,
      "rewards/rejected": -2.6175403594970703,
      "step": 10261
    },
    {
      "epoch": 1.3428477958616178,
      "grad_norm": 14.814586639404297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04432305693626404,
      "logits/rejected": 0.06689175963401794,
      "logps/chosen": -424.0461730957031,
      "logps/rejected": -460.6274719238281,
      "loss": 0.9809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2421377897262573,
      "rewards/margins": 0.13402286171913147,
      "rewards/rejected": -1.3761606216430664,
      "step": 10262
    },
    {
      "epoch": 1.3429786537989694,
      "grad_norm": 4.11152982711792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.512265145778656,
      "logits/rejected": -0.2777465879917145,
      "logps/chosen": -328.27545166015625,
      "logps/rejected": -319.7724609375,
      "loss": 0.2263,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8711041808128357,
      "rewards/margins": 2.262007474899292,
      "rewards/rejected": -3.1331119537353516,
      "step": 10263
    },
    {
      "epoch": 1.3431095117363212,
      "grad_norm": 7.32907772064209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04065496474504471,
      "logits/rejected": 0.16456574201583862,
      "logps/chosen": -442.0356750488281,
      "logps/rejected": -497.04693603515625,
      "loss": 0.3955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3403305113315582,
      "rewards/margins": 1.1327956914901733,
      "rewards/rejected": -1.4731261730194092,
      "step": 10264
    },
    {
      "epoch": 1.343240369673673,
      "grad_norm": 8.234678268432617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1712527573108673,
      "logits/rejected": -0.04700177162885666,
      "logps/chosen": -566.0665283203125,
      "logps/rejected": -622.7357177734375,
      "loss": 0.4328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1201492547988892,
      "rewards/margins": 1.3291791677474976,
      "rewards/rejected": -2.4493284225463867,
      "step": 10265
    },
    {
      "epoch": 1.3433712276110248,
      "grad_norm": 12.734293937683105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13772855699062347,
      "logits/rejected": -0.20797909796237946,
      "logps/chosen": -643.6556396484375,
      "logps/rejected": -452.2356262207031,
      "loss": 0.2655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33880615234375,
      "rewards/margins": 2.959990978240967,
      "rewards/rejected": -3.298797369003296,
      "step": 10266
    },
    {
      "epoch": 1.3435020855483766,
      "grad_norm": 6.788435459136963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13829892873764038,
      "logits/rejected": -0.34596818685531616,
      "logps/chosen": -323.25396728515625,
      "logps/rejected": -439.5965576171875,
      "loss": 0.325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0596354007720947,
      "rewards/margins": 1.6359639167785645,
      "rewards/rejected": -2.6955995559692383,
      "step": 10267
    },
    {
      "epoch": 1.3436329434857284,
      "grad_norm": 3.3137693405151367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14754149317741394,
      "logits/rejected": -0.22290757298469543,
      "logps/chosen": -672.9140625,
      "logps/rejected": -484.3543701171875,
      "loss": 0.1568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2953975796699524,
      "rewards/margins": 2.258427619934082,
      "rewards/rejected": -1.9630298614501953,
      "step": 10268
    },
    {
      "epoch": 1.34376380142308,
      "grad_norm": 8.044726371765137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33006367087364197,
      "logits/rejected": -0.27641409635543823,
      "logps/chosen": -317.0018005371094,
      "logps/rejected": -370.222900390625,
      "loss": 0.4389,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5381252765655518,
      "rewards/margins": 1.7912871837615967,
      "rewards/rejected": -3.3294124603271484,
      "step": 10269
    },
    {
      "epoch": 1.3438946593604317,
      "grad_norm": 4.917212009429932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4133211374282837,
      "logits/rejected": -0.6264439225196838,
      "logps/chosen": -398.51959228515625,
      "logps/rejected": -489.6772155761719,
      "loss": 0.278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8175915479660034,
      "rewards/margins": 1.533484697341919,
      "rewards/rejected": -3.351076364517212,
      "step": 10270
    },
    {
      "epoch": 1.3440255172977835,
      "grad_norm": 7.992016792297363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35338664054870605,
      "logits/rejected": -0.3279114067554474,
      "logps/chosen": -359.3460693359375,
      "logps/rejected": -405.93365478515625,
      "loss": 0.4414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5922655463218689,
      "rewards/margins": 1.4460201263427734,
      "rewards/rejected": -2.038285732269287,
      "step": 10271
    },
    {
      "epoch": 1.3441563752351353,
      "grad_norm": 10.208518981933594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13332109153270721,
      "logits/rejected": -0.059257738292217255,
      "logps/chosen": -421.83770751953125,
      "logps/rejected": -431.8385009765625,
      "loss": 0.7648,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6907171607017517,
      "rewards/margins": 0.41497668623924255,
      "rewards/rejected": -1.1056938171386719,
      "step": 10272
    },
    {
      "epoch": 1.344287233172487,
      "grad_norm": 10.447015762329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11004585772752762,
      "logits/rejected": -0.09011438488960266,
      "logps/chosen": -318.6475830078125,
      "logps/rejected": -420.80810546875,
      "loss": 0.7417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6057862043380737,
      "rewards/margins": 0.5653858184814453,
      "rewards/rejected": -2.1711719036102295,
      "step": 10273
    },
    {
      "epoch": 1.344418091109839,
      "grad_norm": 7.576263427734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18680430948734283,
      "logits/rejected": -0.34973037242889404,
      "logps/chosen": -387.214111328125,
      "logps/rejected": -417.8336181640625,
      "loss": 0.2518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4114222526550293,
      "rewards/margins": 1.936980128288269,
      "rewards/rejected": -3.348402500152588,
      "step": 10274
    },
    {
      "epoch": 1.3445489490471907,
      "grad_norm": 8.56287956237793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4282648265361786,
      "logits/rejected": -0.1802539825439453,
      "logps/chosen": -543.92919921875,
      "logps/rejected": -543.3992919921875,
      "loss": 0.4449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4921266734600067,
      "rewards/margins": 1.1961050033569336,
      "rewards/rejected": -1.6882317066192627,
      "step": 10275
    },
    {
      "epoch": 1.3446798069845425,
      "grad_norm": 17.468616485595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07438144087791443,
      "logits/rejected": -0.25835850834846497,
      "logps/chosen": -501.7174072265625,
      "logps/rejected": -510.0330505371094,
      "loss": 0.9775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1284594535827637,
      "rewards/margins": 0.49332308769226074,
      "rewards/rejected": -2.6217827796936035,
      "step": 10276
    },
    {
      "epoch": 1.3448106649218943,
      "grad_norm": 11.517987251281738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42173099517822266,
      "logits/rejected": -0.5532209277153015,
      "logps/chosen": -384.8362731933594,
      "logps/rejected": -516.4611206054688,
      "loss": 0.6759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.291993498802185,
      "rewards/margins": 0.7008480429649353,
      "rewards/rejected": -1.992841362953186,
      "step": 10277
    },
    {
      "epoch": 1.344941522859246,
      "grad_norm": 9.153578758239746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42828935384750366,
      "logits/rejected": -0.0915495753288269,
      "logps/chosen": -582.2042236328125,
      "logps/rejected": -393.01611328125,
      "loss": 0.5952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1760331392288208,
      "rewards/margins": 0.8516672253608704,
      "rewards/rejected": -2.027700185775757,
      "step": 10278
    },
    {
      "epoch": 1.3450723807965976,
      "grad_norm": 8.933393478393555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013473915867507458,
      "logits/rejected": 0.08387492597103119,
      "logps/chosen": -488.3102111816406,
      "logps/rejected": -456.64013671875,
      "loss": 0.6514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.302160382270813,
      "rewards/margins": 0.8019684553146362,
      "rewards/rejected": -2.104128837585449,
      "step": 10279
    },
    {
      "epoch": 1.3452032387339494,
      "grad_norm": 13.39528751373291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17863690853118896,
      "logits/rejected": -0.027043160051107407,
      "logps/chosen": -569.12158203125,
      "logps/rejected": -561.8951416015625,
      "loss": 0.6132,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4946436882019043,
      "rewards/margins": 1.4021199941635132,
      "rewards/rejected": -2.896763563156128,
      "step": 10280
    },
    {
      "epoch": 1.3453340966713012,
      "grad_norm": 8.240132331848145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21241441369056702,
      "logits/rejected": -0.1274919956922531,
      "logps/chosen": -515.7683715820312,
      "logps/rejected": -562.7586059570312,
      "loss": 0.4256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4005811810493469,
      "rewards/margins": 1.1014372110366821,
      "rewards/rejected": -1.5020184516906738,
      "step": 10281
    },
    {
      "epoch": 1.345464954608653,
      "grad_norm": 5.683282852172852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4178168475627899,
      "logits/rejected": -0.6959695219993591,
      "logps/chosen": -358.682373046875,
      "logps/rejected": -445.68707275390625,
      "loss": 0.2587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9684349298477173,
      "rewards/margins": 2.3826427459716797,
      "rewards/rejected": -3.3510777950286865,
      "step": 10282
    },
    {
      "epoch": 1.3455958125460048,
      "grad_norm": 9.143389701843262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4883280396461487,
      "logits/rejected": -0.2013249397277832,
      "logps/chosen": -422.2062072753906,
      "logps/rejected": -466.2794494628906,
      "loss": 0.3483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3805074691772461,
      "rewards/margins": 1.5066474676132202,
      "rewards/rejected": -1.8871550559997559,
      "step": 10283
    },
    {
      "epoch": 1.3457266704833564,
      "grad_norm": 4.638607978820801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11383013427257538,
      "logits/rejected": -0.2978481650352478,
      "logps/chosen": -499.3259582519531,
      "logps/rejected": -434.9342041015625,
      "loss": 0.1745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5596755743026733,
      "rewards/margins": 2.329641342163086,
      "rewards/rejected": -2.889317035675049,
      "step": 10284
    },
    {
      "epoch": 1.3458575284207082,
      "grad_norm": 15.02193832397461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18112652003765106,
      "logits/rejected": -0.23274190723896027,
      "logps/chosen": -412.8033142089844,
      "logps/rejected": -376.14105224609375,
      "loss": 1.1491,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.0320496559143066,
      "rewards/margins": 0.07344815135002136,
      "rewards/rejected": -2.1054980754852295,
      "step": 10285
    },
    {
      "epoch": 1.34598838635806,
      "grad_norm": 6.079986095428467,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1386280655860901,
      "logits/rejected": -0.45797133445739746,
      "logps/chosen": -778.6893310546875,
      "logps/rejected": -500.1734313964844,
      "loss": 0.1695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3524915874004364,
      "rewards/margins": 2.671964645385742,
      "rewards/rejected": -2.3194730281829834,
      "step": 10286
    },
    {
      "epoch": 1.3461192442954117,
      "grad_norm": 6.887642860412598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1251886487007141,
      "logits/rejected": -0.014307664707303047,
      "logps/chosen": -384.2386474609375,
      "logps/rejected": -446.5887756347656,
      "loss": 0.4144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7677821516990662,
      "rewards/margins": 0.9461879730224609,
      "rewards/rejected": -1.7139701843261719,
      "step": 10287
    },
    {
      "epoch": 1.3462501022327635,
      "grad_norm": 9.099054336547852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44196072220802307,
      "logits/rejected": -0.5960788726806641,
      "logps/chosen": -484.72021484375,
      "logps/rejected": -620.8701782226562,
      "loss": 0.7171,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6545605659484863,
      "rewards/margins": 0.2599026560783386,
      "rewards/rejected": -1.9144632816314697,
      "step": 10288
    },
    {
      "epoch": 1.3463809601701153,
      "grad_norm": 8.284625053405762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1375407576560974,
      "logits/rejected": -0.24373270571231842,
      "logps/chosen": -519.3167114257812,
      "logps/rejected": -569.3120727539062,
      "loss": 0.3902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0137087106704712,
      "rewards/margins": 1.5629647970199585,
      "rewards/rejected": -2.5766735076904297,
      "step": 10289
    },
    {
      "epoch": 1.3465118181074671,
      "grad_norm": 3.748723268508911,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08156140148639679,
      "logits/rejected": -0.26550614833831787,
      "logps/chosen": -259.4111328125,
      "logps/rejected": -428.9628601074219,
      "loss": 0.2201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.69091796875,
      "rewards/margins": 2.5576894283294678,
      "rewards/rejected": -3.2486071586608887,
      "step": 10290
    },
    {
      "epoch": 1.346642676044819,
      "grad_norm": 9.422623634338379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22585895657539368,
      "logits/rejected": -0.31599247455596924,
      "logps/chosen": -451.23968505859375,
      "logps/rejected": -392.166748046875,
      "loss": 0.7128,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.604212760925293,
      "rewards/margins": 0.25938892364501953,
      "rewards/rejected": -1.8636016845703125,
      "step": 10291
    },
    {
      "epoch": 1.3467735339821707,
      "grad_norm": 10.402225494384766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0969078317284584,
      "logits/rejected": -0.10917315632104874,
      "logps/chosen": -540.9215087890625,
      "logps/rejected": -494.5478820800781,
      "loss": 0.4745,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2931150197982788,
      "rewards/margins": 1.452600359916687,
      "rewards/rejected": -2.745715379714966,
      "step": 10292
    },
    {
      "epoch": 1.3469043919195225,
      "grad_norm": 10.66379165649414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15393029153347015,
      "logits/rejected": 0.020706510171294212,
      "logps/chosen": -411.74639892578125,
      "logps/rejected": -435.1452941894531,
      "loss": 0.6006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8634752035140991,
      "rewards/margins": 1.137689471244812,
      "rewards/rejected": -3.001164674758911,
      "step": 10293
    },
    {
      "epoch": 1.347035249856874,
      "grad_norm": 4.78529167175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20880910754203796,
      "logits/rejected": -0.27316170930862427,
      "logps/chosen": -293.9593505859375,
      "logps/rejected": -356.5804138183594,
      "loss": 0.2367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1594619750976562,
      "rewards/margins": 1.5872113704681396,
      "rewards/rejected": -2.746673583984375,
      "step": 10294
    },
    {
      "epoch": 1.3471661077942259,
      "grad_norm": 10.51030158996582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1707177460193634,
      "logits/rejected": -0.19403594732284546,
      "logps/chosen": -408.2558898925781,
      "logps/rejected": -358.12261962890625,
      "loss": 0.6463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.315096139907837,
      "rewards/margins": 1.2328834533691406,
      "rewards/rejected": -2.5479795932769775,
      "step": 10295
    },
    {
      "epoch": 1.3472969657315776,
      "grad_norm": 9.682982444763184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22123734652996063,
      "logits/rejected": -0.30602043867111206,
      "logps/chosen": -512.008544921875,
      "logps/rejected": -454.8955078125,
      "loss": 0.5025,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0987756252288818,
      "rewards/margins": 1.4403148889541626,
      "rewards/rejected": -2.539090156555176,
      "step": 10296
    },
    {
      "epoch": 1.3474278236689294,
      "grad_norm": 7.727514266967773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10610353201627731,
      "logits/rejected": -0.132192924618721,
      "logps/chosen": -524.8832397460938,
      "logps/rejected": -403.92755126953125,
      "loss": 0.4286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.061917781829834,
      "rewards/margins": 1.1819733381271362,
      "rewards/rejected": -2.2438910007476807,
      "step": 10297
    },
    {
      "epoch": 1.3475586816062812,
      "grad_norm": 10.581125259399414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6379188895225525,
      "logits/rejected": -0.5241245031356812,
      "logps/chosen": -515.7742919921875,
      "logps/rejected": -614.5545654296875,
      "loss": 0.6187,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5050355195999146,
      "rewards/margins": 0.8652204871177673,
      "rewards/rejected": -2.370255947113037,
      "step": 10298
    },
    {
      "epoch": 1.347689539543633,
      "grad_norm": 7.411834716796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19094738364219666,
      "logits/rejected": -0.2016480267047882,
      "logps/chosen": -477.7932434082031,
      "logps/rejected": -411.4981689453125,
      "loss": 0.4037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.393660545349121,
      "rewards/margins": 1.9353522062301636,
      "rewards/rejected": -3.329012870788574,
      "step": 10299
    },
    {
      "epoch": 1.3478203974809846,
      "grad_norm": 6.234612941741943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12817618250846863,
      "logits/rejected": -0.2674814462661743,
      "logps/chosen": -450.0962219238281,
      "logps/rejected": -311.621337890625,
      "loss": 0.4323,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0712664127349854,
      "rewards/margins": 1.0738611221313477,
      "rewards/rejected": -2.145127534866333,
      "step": 10300
    },
    {
      "epoch": 1.3479512554183364,
      "grad_norm": 11.205463409423828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17410264909267426,
      "logits/rejected": 0.0007651858031749725,
      "logps/chosen": -630.046142578125,
      "logps/rejected": -551.4810791015625,
      "loss": 0.5436,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0213825702667236,
      "rewards/margins": 0.9980417490005493,
      "rewards/rejected": -3.0194242000579834,
      "step": 10301
    },
    {
      "epoch": 1.3480821133556882,
      "grad_norm": 11.889118194580078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031866852194070816,
      "logits/rejected": -0.3172141909599304,
      "logps/chosen": -588.7434692382812,
      "logps/rejected": -620.3274536132812,
      "loss": 0.583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5246593952178955,
      "rewards/margins": 1.6815481185913086,
      "rewards/rejected": -3.206207752227783,
      "step": 10302
    },
    {
      "epoch": 1.34821297129304,
      "grad_norm": 6.537539005279541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3230838179588318,
      "logits/rejected": -0.4939376413822174,
      "logps/chosen": -462.5845947265625,
      "logps/rejected": -361.0621337890625,
      "loss": 0.2985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0216600894927979,
      "rewards/margins": 1.6142828464508057,
      "rewards/rejected": -2.6359426975250244,
      "step": 10303
    },
    {
      "epoch": 1.3483438292303918,
      "grad_norm": 14.175114631652832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.498725950717926,
      "logits/rejected": -0.5571156740188599,
      "logps/chosen": -511.7843322753906,
      "logps/rejected": -705.9319458007812,
      "loss": 1.0079,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7047159671783447,
      "rewards/margins": 0.33587783575057983,
      "rewards/rejected": -3.0405938625335693,
      "step": 10304
    },
    {
      "epoch": 1.3484746871677435,
      "grad_norm": 9.788562774658203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3727497458457947,
      "logits/rejected": -0.5569544434547424,
      "logps/chosen": -357.5007629394531,
      "logps/rejected": -439.57525634765625,
      "loss": 0.4702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.234631896018982,
      "rewards/margins": 1.2884504795074463,
      "rewards/rejected": -2.5230824947357178,
      "step": 10305
    },
    {
      "epoch": 1.3486055451050953,
      "grad_norm": 8.143318176269531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37152230739593506,
      "logits/rejected": -0.5697624683380127,
      "logps/chosen": -293.23455810546875,
      "logps/rejected": -305.3291320800781,
      "loss": 0.4827,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.053844928741455,
      "rewards/margins": 1.5429117679595947,
      "rewards/rejected": -2.596756935119629,
      "step": 10306
    },
    {
      "epoch": 1.3487364030424471,
      "grad_norm": 8.312191009521484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45099297165870667,
      "logits/rejected": -0.23410925269126892,
      "logps/chosen": -377.2879943847656,
      "logps/rejected": -465.9921875,
      "loss": 0.4541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8579614162445068,
      "rewards/margins": 1.286086916923523,
      "rewards/rejected": -2.1440482139587402,
      "step": 10307
    },
    {
      "epoch": 1.348867260979799,
      "grad_norm": 6.920875072479248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14299294352531433,
      "logits/rejected": -0.3514719009399414,
      "logps/chosen": -460.05615234375,
      "logps/rejected": -404.57293701171875,
      "loss": 0.2964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6990850567817688,
      "rewards/margins": 2.3358795642852783,
      "rewards/rejected": -3.0349650382995605,
      "step": 10308
    },
    {
      "epoch": 1.3489981189171505,
      "grad_norm": 7.488485813140869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39042577147483826,
      "logits/rejected": -0.14039762318134308,
      "logps/chosen": -371.2980651855469,
      "logps/rejected": -457.24371337890625,
      "loss": 0.5086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2293188571929932,
      "rewards/margins": 1.2064239978790283,
      "rewards/rejected": -2.4357428550720215,
      "step": 10309
    },
    {
      "epoch": 1.3491289768545023,
      "grad_norm": 5.74315071105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21073684096336365,
      "logits/rejected": -0.22727489471435547,
      "logps/chosen": -380.98907470703125,
      "logps/rejected": -397.35211181640625,
      "loss": 0.3692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8733536005020142,
      "rewards/margins": 1.7887022495269775,
      "rewards/rejected": -2.6620559692382812,
      "step": 10310
    },
    {
      "epoch": 1.349259834791854,
      "grad_norm": 13.07642650604248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1571747362613678,
      "logits/rejected": -0.384636253118515,
      "logps/chosen": -480.58221435546875,
      "logps/rejected": -374.2565002441406,
      "loss": 0.9034,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6223901510238647,
      "rewards/margins": 0.4905385971069336,
      "rewards/rejected": -2.112928867340088,
      "step": 10311
    },
    {
      "epoch": 1.3493906927292059,
      "grad_norm": 12.574387550354004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040303539484739304,
      "logits/rejected": -0.003753975033760071,
      "logps/chosen": -428.25787353515625,
      "logps/rejected": -398.8617248535156,
      "loss": 0.7209,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.640071988105774,
      "rewards/margins": 0.33170586824417114,
      "rewards/rejected": -1.9717776775360107,
      "step": 10312
    },
    {
      "epoch": 1.3495215506665577,
      "grad_norm": 7.1759514808654785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41061124205589294,
      "logits/rejected": -0.6384526491165161,
      "logps/chosen": -412.3023986816406,
      "logps/rejected": -416.2794189453125,
      "loss": 0.2815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.164699912071228,
      "rewards/margins": 1.9290528297424316,
      "rewards/rejected": -3.093752861022949,
      "step": 10313
    },
    {
      "epoch": 1.3496524086039094,
      "grad_norm": 5.935507774353027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34007298946380615,
      "logits/rejected": -0.0963430106639862,
      "logps/chosen": -571.5328979492188,
      "logps/rejected": -513.053955078125,
      "loss": 0.2627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7922395467758179,
      "rewards/margins": 1.57425856590271,
      "rewards/rejected": -2.366497755050659,
      "step": 10314
    },
    {
      "epoch": 1.349783266541261,
      "grad_norm": 3.0734057426452637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04080244526267052,
      "logits/rejected": -0.13598868250846863,
      "logps/chosen": -387.7959289550781,
      "logps/rejected": -382.52728271484375,
      "loss": 0.2588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.992705762386322,
      "rewards/margins": 1.9969383478164673,
      "rewards/rejected": -2.9896440505981445,
      "step": 10315
    },
    {
      "epoch": 1.3499141244786128,
      "grad_norm": 4.919893741607666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2735529839992523,
      "logits/rejected": -0.6494882702827454,
      "logps/chosen": -335.31396484375,
      "logps/rejected": -305.88690185546875,
      "loss": 0.1989,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4838392734527588,
      "rewards/margins": 2.307832717895508,
      "rewards/rejected": -3.7916717529296875,
      "step": 10316
    },
    {
      "epoch": 1.3500449824159646,
      "grad_norm": 5.44962215423584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15869756042957306,
      "logits/rejected": -0.4972738027572632,
      "logps/chosen": -478.7943115234375,
      "logps/rejected": -648.9324951171875,
      "loss": 0.2113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8145660161972046,
      "rewards/margins": 3.134223461151123,
      "rewards/rejected": -3.948789596557617,
      "step": 10317
    },
    {
      "epoch": 1.3501758403533164,
      "grad_norm": 5.474665641784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25271496176719666,
      "logits/rejected": -0.22054129838943481,
      "logps/chosen": -344.02947998046875,
      "logps/rejected": -393.34698486328125,
      "loss": 0.1683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1993968486785889,
      "rewards/margins": 2.2195639610290527,
      "rewards/rejected": -3.4189610481262207,
      "step": 10318
    },
    {
      "epoch": 1.3503066982906682,
      "grad_norm": 2.762349843978882,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2135506421327591,
      "logits/rejected": -0.3138405382633209,
      "logps/chosen": -522.7234497070312,
      "logps/rejected": -517.0021362304688,
      "loss": 0.0742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19871944189071655,
      "rewards/margins": 3.1557435989379883,
      "rewards/rejected": -2.957024097442627,
      "step": 10319
    },
    {
      "epoch": 1.35043755622802,
      "grad_norm": 4.877986907958984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05542563647031784,
      "logits/rejected": -0.045745231211185455,
      "logps/chosen": -661.3179321289062,
      "logps/rejected": -676.0668334960938,
      "loss": 0.2285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28061237931251526,
      "rewards/margins": 2.1573221683502197,
      "rewards/rejected": -2.437934398651123,
      "step": 10320
    },
    {
      "epoch": 1.3505684141653718,
      "grad_norm": 11.528796195983887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5322823524475098,
      "logits/rejected": -0.5802348852157593,
      "logps/chosen": -325.8309326171875,
      "logps/rejected": -279.98846435546875,
      "loss": 0.4566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5083885192871094,
      "rewards/margins": 1.6558760404586792,
      "rewards/rejected": -3.16426420211792,
      "step": 10321
    },
    {
      "epoch": 1.3506992721027236,
      "grad_norm": 11.403264999389648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03484030067920685,
      "logits/rejected": -0.06516972184181213,
      "logps/chosen": -463.2244567871094,
      "logps/rejected": -538.2437133789062,
      "loss": 0.6813,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2283791303634644,
      "rewards/margins": 1.0465290546417236,
      "rewards/rejected": -2.2749080657958984,
      "step": 10322
    },
    {
      "epoch": 1.3508301300400753,
      "grad_norm": 8.034472465515137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040874332189559937,
      "logits/rejected": -0.18844684958457947,
      "logps/chosen": -395.61737060546875,
      "logps/rejected": -375.52142333984375,
      "loss": 0.3714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.39483642578125,
      "rewards/margins": 1.143251895904541,
      "rewards/rejected": -2.538088083267212,
      "step": 10323
    },
    {
      "epoch": 1.3509609879774271,
      "grad_norm": 9.891098976135254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2125343680381775,
      "logits/rejected": -0.08367006480693817,
      "logps/chosen": -571.4970703125,
      "logps/rejected": -742.3094482421875,
      "loss": 0.5184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.229372262954712,
      "rewards/margins": 0.9429904222488403,
      "rewards/rejected": -2.172362804412842,
      "step": 10324
    },
    {
      "epoch": 1.3510918459147787,
      "grad_norm": 12.03345775604248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5104132294654846,
      "logits/rejected": -0.17855092883110046,
      "logps/chosen": -293.4826354980469,
      "logps/rejected": -432.29595947265625,
      "loss": 0.49,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8153470754623413,
      "rewards/margins": 0.81880122423172,
      "rewards/rejected": -2.634148359298706,
      "step": 10325
    },
    {
      "epoch": 1.3512227038521305,
      "grad_norm": 10.582056045532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32491397857666016,
      "logits/rejected": -0.3301429748535156,
      "logps/chosen": -332.80755615234375,
      "logps/rejected": -393.8670654296875,
      "loss": 0.5041,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5238103866577148,
      "rewards/margins": 1.4465970993041992,
      "rewards/rejected": -1.970407485961914,
      "step": 10326
    },
    {
      "epoch": 1.3513535617894823,
      "grad_norm": 13.14612102508545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6579448580741882,
      "logits/rejected": -0.6196617484092712,
      "logps/chosen": -501.2113037109375,
      "logps/rejected": -532.0443115234375,
      "loss": 0.8869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.035834789276123,
      "rewards/margins": 0.4841959774494171,
      "rewards/rejected": -2.5200307369232178,
      "step": 10327
    },
    {
      "epoch": 1.351484419726834,
      "grad_norm": 10.014716148376465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1518985778093338,
      "logits/rejected": -0.19742444157600403,
      "logps/chosen": -487.43170166015625,
      "logps/rejected": -420.864501953125,
      "loss": 0.463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1016881465911865,
      "rewards/margins": 1.1565361022949219,
      "rewards/rejected": -2.2582244873046875,
      "step": 10328
    },
    {
      "epoch": 1.3516152776641859,
      "grad_norm": 10.65766429901123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.477469265460968,
      "logits/rejected": -0.8762383460998535,
      "logps/chosen": -526.95458984375,
      "logps/rejected": -521.5960083007812,
      "loss": 0.4469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1220760345458984,
      "rewards/margins": 1.409924030303955,
      "rewards/rejected": -2.5320000648498535,
      "step": 10329
    },
    {
      "epoch": 1.3517461356015374,
      "grad_norm": 5.4119367599487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16217081248760223,
      "logits/rejected": -0.5307266116142273,
      "logps/chosen": -640.0803833007812,
      "logps/rejected": -358.7218322753906,
      "loss": 0.3416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21660098433494568,
      "rewards/margins": 2.094266414642334,
      "rewards/rejected": -2.3108673095703125,
      "step": 10330
    },
    {
      "epoch": 1.3518769935388892,
      "grad_norm": 7.690139293670654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00998978316783905,
      "logits/rejected": -0.2500917911529541,
      "logps/chosen": -537.6112670898438,
      "logps/rejected": -526.1280517578125,
      "loss": 0.5679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9141403436660767,
      "rewards/margins": 1.2469210624694824,
      "rewards/rejected": -2.1610612869262695,
      "step": 10331
    },
    {
      "epoch": 1.352007851476241,
      "grad_norm": 4.659320831298828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45750734210014343,
      "logits/rejected": -0.4192354381084442,
      "logps/chosen": -646.4373168945312,
      "logps/rejected": -767.6599731445312,
      "loss": 0.225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4479362368583679,
      "rewards/margins": 1.9601017236709595,
      "rewards/rejected": -2.4080381393432617,
      "step": 10332
    },
    {
      "epoch": 1.3521387094135928,
      "grad_norm": 5.983457565307617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18875008821487427,
      "logits/rejected": -0.06897278875112534,
      "logps/chosen": -566.5100708007812,
      "logps/rejected": -483.3291015625,
      "loss": 0.3821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6781517267227173,
      "rewards/margins": 1.3637163639068604,
      "rewards/rejected": -2.041868209838867,
      "step": 10333
    },
    {
      "epoch": 1.3522695673509446,
      "grad_norm": 20.147268295288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2144145965576172,
      "logits/rejected": -0.28685030341148376,
      "logps/chosen": -480.3659973144531,
      "logps/rejected": -401.6890869140625,
      "loss": 0.738,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.180407166481018,
      "rewards/margins": 0.2335059642791748,
      "rewards/rejected": -1.4139132499694824,
      "step": 10334
    },
    {
      "epoch": 1.3524004252882964,
      "grad_norm": 6.5465989112854,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36021891236305237,
      "logits/rejected": -0.3212023973464966,
      "logps/chosen": -468.9513244628906,
      "logps/rejected": -406.939697265625,
      "loss": 0.332,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6335521936416626,
      "rewards/margins": 1.1893844604492188,
      "rewards/rejected": -1.8229365348815918,
      "step": 10335
    },
    {
      "epoch": 1.3525312832256482,
      "grad_norm": 7.688091278076172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3134157359600067,
      "logits/rejected": -0.4632604122161865,
      "logps/chosen": -666.97021484375,
      "logps/rejected": -521.974853515625,
      "loss": 0.4349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.894900381565094,
      "rewards/margins": 1.6184101104736328,
      "rewards/rejected": -2.513310432434082,
      "step": 10336
    },
    {
      "epoch": 1.352662141163,
      "grad_norm": 9.689836502075195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40558353066444397,
      "logits/rejected": -0.3521406650543213,
      "logps/chosen": -411.67620849609375,
      "logps/rejected": -430.270751953125,
      "loss": 0.5317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1622724533081055,
      "rewards/margins": 1.1651042699813843,
      "rewards/rejected": -2.3273766040802,
      "step": 10337
    },
    {
      "epoch": 1.3527929991003518,
      "grad_norm": 5.599440574645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27679672837257385,
      "logits/rejected": -0.3909168243408203,
      "logps/chosen": -458.3480224609375,
      "logps/rejected": -427.642822265625,
      "loss": 0.2345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9424929618835449,
      "rewards/margins": 1.5959422588348389,
      "rewards/rejected": -2.538435220718384,
      "step": 10338
    },
    {
      "epoch": 1.3529238570377036,
      "grad_norm": 8.777918815612793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09659016877412796,
      "logits/rejected": 0.2683205008506775,
      "logps/chosen": -551.3346557617188,
      "logps/rejected": -600.1912841796875,
      "loss": 0.3375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4078330993652344,
      "rewards/margins": 2.106672763824463,
      "rewards/rejected": -2.5145058631896973,
      "step": 10339
    },
    {
      "epoch": 1.3530547149750551,
      "grad_norm": 5.536012172698975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1744714379310608,
      "logits/rejected": -0.2021997720003128,
      "logps/chosen": -470.6375427246094,
      "logps/rejected": -457.96044921875,
      "loss": 0.2551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8909750580787659,
      "rewards/margins": 1.8672003746032715,
      "rewards/rejected": -2.7581753730773926,
      "step": 10340
    },
    {
      "epoch": 1.353185572912407,
      "grad_norm": 8.904264450073242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4397764205932617,
      "logits/rejected": -0.4272153377532959,
      "logps/chosen": -400.1053771972656,
      "logps/rejected": -391.79888916015625,
      "loss": 0.7225,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2566604614257812,
      "rewards/margins": 0.4008467495441437,
      "rewards/rejected": -1.6575074195861816,
      "step": 10341
    },
    {
      "epoch": 1.3533164308497587,
      "grad_norm": 6.302192211151123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03127773478627205,
      "logits/rejected": 0.05842529609799385,
      "logps/chosen": -427.62451171875,
      "logps/rejected": -537.59912109375,
      "loss": 0.3907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3928440809249878,
      "rewards/margins": 1.5814212560653687,
      "rewards/rejected": -2.9742650985717773,
      "step": 10342
    },
    {
      "epoch": 1.3534472887871105,
      "grad_norm": 15.460333824157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3088460862636566,
      "logits/rejected": -0.12003495544195175,
      "logps/chosen": -540.357421875,
      "logps/rejected": -749.7286376953125,
      "loss": 0.7601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4161583185195923,
      "rewards/margins": 1.7752004861831665,
      "rewards/rejected": -3.1913585662841797,
      "step": 10343
    },
    {
      "epoch": 1.3535781467244623,
      "grad_norm": 12.6565523147583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35455283522605896,
      "logits/rejected": -0.07897721230983734,
      "logps/chosen": -389.2892761230469,
      "logps/rejected": -384.1336669921875,
      "loss": 0.837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8748207092285156,
      "rewards/margins": 0.5153049230575562,
      "rewards/rejected": -2.3901257514953613,
      "step": 10344
    },
    {
      "epoch": 1.353709004661814,
      "grad_norm": 10.458977699279785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013465628027915955,
      "logits/rejected": -0.0857602059841156,
      "logps/chosen": -493.770263671875,
      "logps/rejected": -446.4822998046875,
      "loss": 0.5738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.76117342710495,
      "rewards/margins": 0.6448488831520081,
      "rewards/rejected": -1.406022310256958,
      "step": 10345
    },
    {
      "epoch": 1.3538398625991657,
      "grad_norm": 8.60941219329834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23985670506954193,
      "logits/rejected": -0.33145758509635925,
      "logps/chosen": -584.4555053710938,
      "logps/rejected": -685.729248046875,
      "loss": 0.4696,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7716949582099915,
      "rewards/margins": 0.7443622946739197,
      "rewards/rejected": -1.5160573720932007,
      "step": 10346
    },
    {
      "epoch": 1.3539707205365175,
      "grad_norm": 14.811182975769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15699082612991333,
      "logits/rejected": -0.3254375755786896,
      "logps/chosen": -407.6615905761719,
      "logps/rejected": -407.8952331542969,
      "loss": 1.0907,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.660235047340393,
      "rewards/margins": 1.233131766319275,
      "rewards/rejected": -2.893366813659668,
      "step": 10347
    },
    {
      "epoch": 1.3541015784738692,
      "grad_norm": 11.106175422668457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07589153945446014,
      "logits/rejected": 0.04417911544442177,
      "logps/chosen": -568.6044311523438,
      "logps/rejected": -492.0279541015625,
      "loss": 0.5411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3938804566860199,
      "rewards/margins": 0.9664987325668335,
      "rewards/rejected": -1.3603792190551758,
      "step": 10348
    },
    {
      "epoch": 1.354232436411221,
      "grad_norm": 10.514854431152344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2719160318374634,
      "logits/rejected": -0.27051761746406555,
      "logps/chosen": -518.979736328125,
      "logps/rejected": -443.0579528808594,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9720722436904907,
      "rewards/margins": 0.6575160026550293,
      "rewards/rejected": -1.62958824634552,
      "step": 10349
    },
    {
      "epoch": 1.3543632943485728,
      "grad_norm": 6.626041412353516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23706045746803284,
      "logits/rejected": -0.5578228831291199,
      "logps/chosen": -563.4370727539062,
      "logps/rejected": -585.0862426757812,
      "loss": 0.1703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0137317180633545,
      "rewards/margins": 2.8807928562164307,
      "rewards/rejected": -3.8945250511169434,
      "step": 10350
    },
    {
      "epoch": 1.3544941522859246,
      "grad_norm": 8.169699668884277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5946717262268066,
      "logits/rejected": -0.49355924129486084,
      "logps/chosen": -403.2296142578125,
      "logps/rejected": -497.5190124511719,
      "loss": 0.3667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32390692830085754,
      "rewards/margins": 1.8224543333053589,
      "rewards/rejected": -2.1463611125946045,
      "step": 10351
    },
    {
      "epoch": 1.3546250102232764,
      "grad_norm": 10.689056396484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3322126865386963,
      "logits/rejected": -0.20392632484436035,
      "logps/chosen": -508.0408630371094,
      "logps/rejected": -445.6060791015625,
      "loss": 0.6574,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9124702215194702,
      "rewards/margins": 0.8416925668716431,
      "rewards/rejected": -2.7541627883911133,
      "step": 10352
    },
    {
      "epoch": 1.3547558681606282,
      "grad_norm": 9.923689842224121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2900380492210388,
      "logits/rejected": -0.3910452127456665,
      "logps/chosen": -474.0798034667969,
      "logps/rejected": -413.563232421875,
      "loss": 0.5846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.040076844394207,
      "rewards/margins": 0.7455059289932251,
      "rewards/rejected": -0.7855827808380127,
      "step": 10353
    },
    {
      "epoch": 1.35488672609798,
      "grad_norm": 7.696459770202637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010871581733226776,
      "logits/rejected": -0.09851724654436111,
      "logps/chosen": -429.1201171875,
      "logps/rejected": -443.99237060546875,
      "loss": 0.6209,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46740174293518066,
      "rewards/margins": 0.9104510545730591,
      "rewards/rejected": -1.3778527975082397,
      "step": 10354
    },
    {
      "epoch": 1.3550175840353316,
      "grad_norm": 3.662902355194092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026214269921183586,
      "logits/rejected": -0.26846471428871155,
      "logps/chosen": -521.408203125,
      "logps/rejected": -506.064697265625,
      "loss": 0.1792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16546349227428436,
      "rewards/margins": 2.4996957778930664,
      "rewards/rejected": -2.665158987045288,
      "step": 10355
    },
    {
      "epoch": 1.3551484419726834,
      "grad_norm": 8.681324005126953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34168538451194763,
      "logits/rejected": -0.39978548884391785,
      "logps/chosen": -604.4224243164062,
      "logps/rejected": -588.431396484375,
      "loss": 0.3725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5423153638839722,
      "rewards/margins": 1.7127330303192139,
      "rewards/rejected": -2.2550482749938965,
      "step": 10356
    },
    {
      "epoch": 1.3552792999100352,
      "grad_norm": 7.340942859649658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5589606761932373,
      "logits/rejected": -0.4224824905395508,
      "logps/chosen": -404.3214416503906,
      "logps/rejected": -370.7303466796875,
      "loss": 0.4486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3467460870742798,
      "rewards/margins": 1.805181860923767,
      "rewards/rejected": -3.151927947998047,
      "step": 10357
    },
    {
      "epoch": 1.355410157847387,
      "grad_norm": 3.7430107593536377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2022067904472351,
      "logits/rejected": -0.3265940248966217,
      "logps/chosen": -716.565185546875,
      "logps/rejected": -533.5385131835938,
      "loss": 0.1327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.048528119921684265,
      "rewards/margins": 2.527209758758545,
      "rewards/rejected": -2.575737714767456,
      "step": 10358
    },
    {
      "epoch": 1.3555410157847387,
      "grad_norm": 5.108501434326172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35660913586616516,
      "logits/rejected": -0.22132103145122528,
      "logps/chosen": -432.16961669921875,
      "logps/rejected": -414.9422607421875,
      "loss": 0.2569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5221376419067383,
      "rewards/margins": 1.8233088254928589,
      "rewards/rejected": -2.3454465866088867,
      "step": 10359
    },
    {
      "epoch": 1.3556718737220905,
      "grad_norm": 8.125311851501465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4209596812725067,
      "logits/rejected": -0.40690118074417114,
      "logps/chosen": -414.253173828125,
      "logps/rejected": -364.9600830078125,
      "loss": 0.4027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0326132774353027,
      "rewards/margins": 1.4067471027374268,
      "rewards/rejected": -2.4393601417541504,
      "step": 10360
    },
    {
      "epoch": 1.355802731659442,
      "grad_norm": 9.805746078491211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007793210446834564,
      "logits/rejected": -0.10626067221164703,
      "logps/chosen": -674.7717895507812,
      "logps/rejected": -528.0712280273438,
      "loss": 0.582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19679103791713715,
      "rewards/margins": 0.7627550363540649,
      "rewards/rejected": -0.9595460891723633,
      "step": 10361
    },
    {
      "epoch": 1.3559335895967939,
      "grad_norm": 5.195556640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4573884606361389,
      "logits/rejected": -0.5052415728569031,
      "logps/chosen": -272.6767883300781,
      "logps/rejected": -403.0111389160156,
      "loss": 0.293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7173561453819275,
      "rewards/margins": 2.285869598388672,
      "rewards/rejected": -3.003225803375244,
      "step": 10362
    },
    {
      "epoch": 1.3560644475341457,
      "grad_norm": 10.722918510437012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48603400588035583,
      "logits/rejected": -0.30306923389434814,
      "logps/chosen": -447.162109375,
      "logps/rejected": -388.73809814453125,
      "loss": 0.715,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7767634391784668,
      "rewards/margins": 0.6062332391738892,
      "rewards/rejected": -2.3829967975616455,
      "step": 10363
    },
    {
      "epoch": 1.3561953054714975,
      "grad_norm": 5.554601192474365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46346163749694824,
      "logits/rejected": -0.614338755607605,
      "logps/chosen": -290.90338134765625,
      "logps/rejected": -277.064208984375,
      "loss": 0.3464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0258346796035767,
      "rewards/margins": 0.9537665247917175,
      "rewards/rejected": -1.9796011447906494,
      "step": 10364
    },
    {
      "epoch": 1.3563261634088493,
      "grad_norm": 5.99608850479126,
      "learning_rate": 5e-06,
      "logits/chosen": 0.051283106207847595,
      "logits/rejected": -0.006473124027252197,
      "logps/chosen": -549.7941284179688,
      "logps/rejected": -406.2735290527344,
      "loss": 0.3387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0225001573562622,
      "rewards/margins": 1.4526207447052002,
      "rewards/rejected": -2.475121021270752,
      "step": 10365
    },
    {
      "epoch": 1.356457021346201,
      "grad_norm": 5.143435955047607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14169862866401672,
      "logits/rejected": -0.1859988272190094,
      "logps/chosen": -439.10498046875,
      "logps/rejected": -502.12384033203125,
      "loss": 0.3118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05147561430931091,
      "rewards/margins": 1.3771930932998657,
      "rewards/rejected": -1.4286688566207886,
      "step": 10366
    },
    {
      "epoch": 1.3565878792835528,
      "grad_norm": 11.010361671447754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20156781375408173,
      "logits/rejected": -0.39478468894958496,
      "logps/chosen": -455.0328674316406,
      "logps/rejected": -540.9173583984375,
      "loss": 0.5748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4704633951187134,
      "rewards/margins": 1.3627238273620605,
      "rewards/rejected": -2.8331873416900635,
      "step": 10367
    },
    {
      "epoch": 1.3567187372209046,
      "grad_norm": 7.19119119644165,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025930926203727722,
      "logits/rejected": -0.15093958377838135,
      "logps/chosen": -602.2969360351562,
      "logps/rejected": -492.6508483886719,
      "loss": 0.3262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07630538940429688,
      "rewards/margins": 2.4845845699310303,
      "rewards/rejected": -2.4082789421081543,
      "step": 10368
    },
    {
      "epoch": 1.3568495951582564,
      "grad_norm": 9.438164710998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0884324386715889,
      "logits/rejected": -0.1266954094171524,
      "logps/chosen": -524.8323364257812,
      "logps/rejected": -585.1307983398438,
      "loss": 0.3841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8748413920402527,
      "rewards/margins": 1.1995233297348022,
      "rewards/rejected": -2.07436466217041,
      "step": 10369
    },
    {
      "epoch": 1.3569804530956082,
      "grad_norm": 5.849906921386719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06835995614528656,
      "logits/rejected": -0.11563587933778763,
      "logps/chosen": -324.6649475097656,
      "logps/rejected": -334.2634582519531,
      "loss": 0.2909,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6765210628509521,
      "rewards/margins": 1.5675227642059326,
      "rewards/rejected": -2.244044065475464,
      "step": 10370
    },
    {
      "epoch": 1.3571113110329598,
      "grad_norm": 7.71317195892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31704995036125183,
      "logits/rejected": -0.1701001524925232,
      "logps/chosen": -365.4759216308594,
      "logps/rejected": -356.5987243652344,
      "loss": 0.5111,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0578956604003906,
      "rewards/margins": 0.9512832760810852,
      "rewards/rejected": -2.009178876876831,
      "step": 10371
    },
    {
      "epoch": 1.3572421689703116,
      "grad_norm": 8.690986633300781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18216051161289215,
      "logits/rejected": -0.35077375173568726,
      "logps/chosen": -578.7420043945312,
      "logps/rejected": -444.338134765625,
      "loss": 0.4343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2037299871444702,
      "rewards/margins": 1.2957700490951538,
      "rewards/rejected": -2.499500036239624,
      "step": 10372
    },
    {
      "epoch": 1.3573730269076634,
      "grad_norm": 9.257901191711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24047093093395233,
      "logits/rejected": -0.3806818425655365,
      "logps/chosen": -453.48779296875,
      "logps/rejected": -575.1797485351562,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.452447772026062,
      "rewards/margins": 1.5161442756652832,
      "rewards/rejected": -2.9685921669006348,
      "step": 10373
    },
    {
      "epoch": 1.3575038848450152,
      "grad_norm": 7.514254093170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12367065995931625,
      "logits/rejected": -0.19504781067371368,
      "logps/chosen": -404.6639709472656,
      "logps/rejected": -433.0532531738281,
      "loss": 0.3599,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1224298477172852,
      "rewards/margins": 1.9019763469696045,
      "rewards/rejected": -3.0244064331054688,
      "step": 10374
    },
    {
      "epoch": 1.357634742782367,
      "grad_norm": 6.264978885650635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11563944816589355,
      "logits/rejected": -0.35648444294929504,
      "logps/chosen": -566.462158203125,
      "logps/rejected": -504.62823486328125,
      "loss": 0.3466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.219160795211792,
      "rewards/margins": 1.5609548091888428,
      "rewards/rejected": -2.7801156044006348,
      "step": 10375
    },
    {
      "epoch": 1.3577656007197185,
      "grad_norm": 6.5549702644348145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3712538480758667,
      "logits/rejected": -0.33477556705474854,
      "logps/chosen": -549.591064453125,
      "logps/rejected": -679.130859375,
      "loss": 0.2734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9932419061660767,
      "rewards/margins": 2.32905912399292,
      "rewards/rejected": -3.322300910949707,
      "step": 10376
    },
    {
      "epoch": 1.3578964586570703,
      "grad_norm": 7.765439510345459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.296723335981369,
      "logits/rejected": -0.49587392807006836,
      "logps/chosen": -540.8118286132812,
      "logps/rejected": -458.1474914550781,
      "loss": 0.4619,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2327485084533691,
      "rewards/margins": 1.064713954925537,
      "rewards/rejected": -2.297462224960327,
      "step": 10377
    },
    {
      "epoch": 1.358027316594422,
      "grad_norm": 6.7835307121276855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3598767817020416,
      "logits/rejected": -0.23564592003822327,
      "logps/chosen": -600.1636962890625,
      "logps/rejected": -742.9569091796875,
      "loss": 0.3049,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5497126579284668,
      "rewards/margins": 1.7244383096694946,
      "rewards/rejected": -3.2741506099700928,
      "step": 10378
    },
    {
      "epoch": 1.358158174531774,
      "grad_norm": 17.389360427856445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12576201558113098,
      "logits/rejected": 0.05785655975341797,
      "logps/chosen": -720.3246459960938,
      "logps/rejected": -437.1839904785156,
      "loss": 0.8059,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.525368094444275,
      "rewards/margins": 0.7272567749023438,
      "rewards/rejected": -2.252624988555908,
      "step": 10379
    },
    {
      "epoch": 1.3582890324691257,
      "grad_norm": 5.12169075012207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.154811829328537,
      "logits/rejected": -0.23242542147636414,
      "logps/chosen": -469.0670471191406,
      "logps/rejected": -493.69696044921875,
      "loss": 0.202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8276349902153015,
      "rewards/margins": 2.0156259536743164,
      "rewards/rejected": -2.8432610034942627,
      "step": 10380
    },
    {
      "epoch": 1.3584198904064775,
      "grad_norm": 5.028994560241699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12432414293289185,
      "logits/rejected": -0.267300546169281,
      "logps/chosen": -352.89306640625,
      "logps/rejected": -390.31573486328125,
      "loss": 0.3049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9594942331314087,
      "rewards/margins": 1.6427397727966309,
      "rewards/rejected": -2.602234125137329,
      "step": 10381
    },
    {
      "epoch": 1.3585507483438293,
      "grad_norm": 12.59654426574707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2908470034599304,
      "logits/rejected": -0.2848066985607147,
      "logps/chosen": -552.5420532226562,
      "logps/rejected": -511.49603271484375,
      "loss": 0.6972,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8272666931152344,
      "rewards/margins": 0.7430126070976257,
      "rewards/rejected": -1.5702792406082153,
      "step": 10382
    },
    {
      "epoch": 1.358681606281181,
      "grad_norm": 5.829333782196045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2275642454624176,
      "logits/rejected": -0.45694440603256226,
      "logps/chosen": -394.0680847167969,
      "logps/rejected": -345.793701171875,
      "loss": 0.3806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5414431095123291,
      "rewards/margins": 1.1227604150772095,
      "rewards/rejected": -1.6642035245895386,
      "step": 10383
    },
    {
      "epoch": 1.3588124642185329,
      "grad_norm": 4.200738906860352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21916425228118896,
      "logits/rejected": -0.2313353717327118,
      "logps/chosen": -380.3106689453125,
      "logps/rejected": -457.9811706542969,
      "loss": 0.1664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39027139544487,
      "rewards/margins": 2.2666549682617188,
      "rewards/rejected": -2.656926393508911,
      "step": 10384
    },
    {
      "epoch": 1.3589433221558846,
      "grad_norm": 4.9326982498168945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08158119022846222,
      "logits/rejected": -0.3465186357498169,
      "logps/chosen": -416.18994140625,
      "logps/rejected": -440.2203674316406,
      "loss": 0.287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.399759292602539,
      "rewards/margins": 1.9545893669128418,
      "rewards/rejected": -3.3543484210968018,
      "step": 10385
    },
    {
      "epoch": 1.3590741800932362,
      "grad_norm": 9.253077507019043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22230234742164612,
      "logits/rejected": -0.1809975951910019,
      "logps/chosen": -439.43145751953125,
      "logps/rejected": -425.1805725097656,
      "loss": 0.3953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8706592321395874,
      "rewards/margins": 1.5197557210922241,
      "rewards/rejected": -2.3904151916503906,
      "step": 10386
    },
    {
      "epoch": 1.359205038030588,
      "grad_norm": 15.193523406982422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2513158917427063,
      "logits/rejected": -0.3195146918296814,
      "logps/chosen": -578.9646606445312,
      "logps/rejected": -509.6492004394531,
      "loss": 1.1819,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.060769557952881,
      "rewards/margins": -0.3120616674423218,
      "rewards/rejected": -1.7487080097198486,
      "step": 10387
    },
    {
      "epoch": 1.3593358959679398,
      "grad_norm": 10.195211410522461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2415904998779297,
      "logits/rejected": -0.3994131088256836,
      "logps/chosen": -726.9354248046875,
      "logps/rejected": -652.0281982421875,
      "loss": 0.4668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6485874652862549,
      "rewards/margins": 1.6909420490264893,
      "rewards/rejected": -3.339529275894165,
      "step": 10388
    },
    {
      "epoch": 1.3594667539052916,
      "grad_norm": 6.180478572845459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3830171823501587,
      "logits/rejected": -0.3530547618865967,
      "logps/chosen": -411.4407043457031,
      "logps/rejected": -391.40020751953125,
      "loss": 0.3203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.553151547908783,
      "rewards/margins": 1.7125657796859741,
      "rewards/rejected": -2.2657175064086914,
      "step": 10389
    },
    {
      "epoch": 1.3595976118426434,
      "grad_norm": 9.293900489807129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02861969918012619,
      "logits/rejected": -0.15921024978160858,
      "logps/chosen": -501.104736328125,
      "logps/rejected": -338.3192443847656,
      "loss": 0.4599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6260604858398438,
      "rewards/margins": 1.4983890056610107,
      "rewards/rejected": -2.1244494915008545,
      "step": 10390
    },
    {
      "epoch": 1.3597284697799952,
      "grad_norm": 11.034723281860352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5856656432151794,
      "logits/rejected": -0.5472597479820251,
      "logps/chosen": -424.2017822265625,
      "logps/rejected": -478.1575622558594,
      "loss": 0.718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1253446340560913,
      "rewards/margins": 0.15481318533420563,
      "rewards/rejected": -1.2801578044891357,
      "step": 10391
    },
    {
      "epoch": 1.3598593277173467,
      "grad_norm": 6.826179027557373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40442001819610596,
      "logits/rejected": -0.3597574830055237,
      "logps/chosen": -243.0565185546875,
      "logps/rejected": -267.288330078125,
      "loss": 0.4991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6013698577880859,
      "rewards/margins": 0.7881746292114258,
      "rewards/rejected": -1.3895444869995117,
      "step": 10392
    },
    {
      "epoch": 1.3599901856546985,
      "grad_norm": 8.042941093444824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3685542047023773,
      "logits/rejected": -0.4482256770133972,
      "logps/chosen": -481.53125,
      "logps/rejected": -328.5482177734375,
      "loss": 0.4752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0116186141967773,
      "rewards/margins": 1.2399523258209229,
      "rewards/rejected": -2.2515709400177,
      "step": 10393
    },
    {
      "epoch": 1.3601210435920503,
      "grad_norm": 5.459771156311035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16349193453788757,
      "logits/rejected": -0.21496175229549408,
      "logps/chosen": -427.5157470703125,
      "logps/rejected": -384.3993835449219,
      "loss": 0.2693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9099174737930298,
      "rewards/margins": 1.6299011707305908,
      "rewards/rejected": -2.539818525314331,
      "step": 10394
    },
    {
      "epoch": 1.3602519015294021,
      "grad_norm": 5.464932918548584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6774843335151672,
      "logits/rejected": -0.4323008954524994,
      "logps/chosen": -516.212646484375,
      "logps/rejected": -525.6605834960938,
      "loss": 0.2458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08337494730949402,
      "rewards/margins": 1.9072215557098389,
      "rewards/rejected": -1.9905965328216553,
      "step": 10395
    },
    {
      "epoch": 1.360382759466754,
      "grad_norm": 6.498292446136475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30164283514022827,
      "logits/rejected": -0.3593282997608185,
      "logps/chosen": -576.6114501953125,
      "logps/rejected": -546.2980346679688,
      "loss": 0.2691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7314176559448242,
      "rewards/margins": 1.955016851425171,
      "rewards/rejected": -2.686434745788574,
      "step": 10396
    },
    {
      "epoch": 1.3605136174041057,
      "grad_norm": 7.06182861328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08403600007295609,
      "logits/rejected": -0.3113367259502411,
      "logps/chosen": -440.83819580078125,
      "logps/rejected": -566.84521484375,
      "loss": 0.4136,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1767255663871765,
      "rewards/margins": 1.8052983283996582,
      "rewards/rejected": -1.9820237159729004,
      "step": 10397
    },
    {
      "epoch": 1.3606444753414575,
      "grad_norm": 7.5995588302612305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4413358271121979,
      "logits/rejected": -0.35116827487945557,
      "logps/chosen": -495.7651062011719,
      "logps/rejected": -348.3271484375,
      "loss": 0.4061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8145310878753662,
      "rewards/margins": 0.8703249096870422,
      "rewards/rejected": -1.6848561763763428,
      "step": 10398
    },
    {
      "epoch": 1.3607753332788093,
      "grad_norm": 9.157264709472656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3254452347755432,
      "logits/rejected": -0.4544335603713989,
      "logps/chosen": -330.45660400390625,
      "logps/rejected": -321.9444885253906,
      "loss": 0.6186,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.079766273498535,
      "rewards/margins": 0.3718528747558594,
      "rewards/rejected": -2.4516191482543945,
      "step": 10399
    },
    {
      "epoch": 1.360906191216161,
      "grad_norm": 10.335261344909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23169003427028656,
      "logits/rejected": -0.37909263372421265,
      "logps/chosen": -452.5147705078125,
      "logps/rejected": -373.1328125,
      "loss": 0.4787,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.022597074508667,
      "rewards/margins": 1.4925175905227661,
      "rewards/rejected": -2.5151143074035645,
      "step": 10400
    },
    {
      "epoch": 1.3610370491535126,
      "grad_norm": 4.615509510040283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08979569375514984,
      "logits/rejected": 0.09056494385004044,
      "logps/chosen": -419.75,
      "logps/rejected": -524.4473876953125,
      "loss": 0.2093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6240497827529907,
      "rewards/margins": 1.8313989639282227,
      "rewards/rejected": -2.455448865890503,
      "step": 10401
    },
    {
      "epoch": 1.3611679070908644,
      "grad_norm": 4.391969203948975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42326781153678894,
      "logits/rejected": -0.4416094124317169,
      "logps/chosen": -487.8642578125,
      "logps/rejected": -506.108154296875,
      "loss": 0.1714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.012651830911636353,
      "rewards/margins": 2.0098750591278076,
      "rewards/rejected": -1.997223138809204,
      "step": 10402
    },
    {
      "epoch": 1.3612987650282162,
      "grad_norm": 4.913380146026611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1668420135974884,
      "logits/rejected": -0.3932046592235565,
      "logps/chosen": -776.227783203125,
      "logps/rejected": -493.0518493652344,
      "loss": 0.2132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7818401455879211,
      "rewards/margins": 1.9817577600479126,
      "rewards/rejected": -1.1999175548553467,
      "step": 10403
    },
    {
      "epoch": 1.361429622965568,
      "grad_norm": 4.502017498016357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4274470806121826,
      "logits/rejected": -0.4044443368911743,
      "logps/chosen": -439.26385498046875,
      "logps/rejected": -509.0389709472656,
      "loss": 0.1876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0558863878250122,
      "rewards/margins": 2.227064371109009,
      "rewards/rejected": -3.2829508781433105,
      "step": 10404
    },
    {
      "epoch": 1.3615604809029198,
      "grad_norm": 8.01646614074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3985256254673004,
      "logits/rejected": -0.40073415637016296,
      "logps/chosen": -373.6041259765625,
      "logps/rejected": -502.93603515625,
      "loss": 0.3704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2328071594238281,
      "rewards/margins": 1.260235071182251,
      "rewards/rejected": -2.493042230606079,
      "step": 10405
    },
    {
      "epoch": 1.3616913388402716,
      "grad_norm": 14.793329238891602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1320987343788147,
      "logits/rejected": -0.34786471724510193,
      "logps/chosen": -534.56689453125,
      "logps/rejected": -590.5607299804688,
      "loss": 0.7387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2373868227005005,
      "rewards/margins": 0.6853269338607788,
      "rewards/rejected": -1.9227136373519897,
      "step": 10406
    },
    {
      "epoch": 1.3618221967776232,
      "grad_norm": 12.69853401184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2571031451225281,
      "logits/rejected": -0.27248385548591614,
      "logps/chosen": -379.172607421875,
      "logps/rejected": -515.5270385742188,
      "loss": 0.6175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5708881616592407,
      "rewards/margins": 0.6090127825737,
      "rewards/rejected": -2.179900884628296,
      "step": 10407
    },
    {
      "epoch": 1.361953054714975,
      "grad_norm": 9.917228698730469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42278775572776794,
      "logits/rejected": -0.4879690408706665,
      "logps/chosen": -814.9369506835938,
      "logps/rejected": -542.5444946289062,
      "loss": 0.3617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9745768904685974,
      "rewards/margins": 1.4456945657730103,
      "rewards/rejected": -2.420271635055542,
      "step": 10408
    },
    {
      "epoch": 1.3620839126523268,
      "grad_norm": 11.710264205932617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037201888859272,
      "logits/rejected": -0.32635360956192017,
      "logps/chosen": -401.7225646972656,
      "logps/rejected": -402.82550048828125,
      "loss": 0.6548,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9418296813964844,
      "rewards/margins": 1.2776894569396973,
      "rewards/rejected": -2.2195193767547607,
      "step": 10409
    },
    {
      "epoch": 1.3622147705896785,
      "grad_norm": 12.931570053100586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48715856671333313,
      "logits/rejected": -0.48006701469421387,
      "logps/chosen": -511.200927734375,
      "logps/rejected": -358.6949462890625,
      "loss": 0.44,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5261408090591431,
      "rewards/margins": 1.5239181518554688,
      "rewards/rejected": -2.0500590801239014,
      "step": 10410
    },
    {
      "epoch": 1.3623456285270303,
      "grad_norm": 13.973658561706543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024091636762022972,
      "logits/rejected": 0.024619845673441887,
      "logps/chosen": -562.5208740234375,
      "logps/rejected": -419.3008117675781,
      "loss": 0.5741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3971397876739502,
      "rewards/margins": 1.7996701002120972,
      "rewards/rejected": -3.196809768676758,
      "step": 10411
    },
    {
      "epoch": 1.3624764864643821,
      "grad_norm": 16.694259643554688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25989338755607605,
      "logits/rejected": -0.36974555253982544,
      "logps/chosen": -596.1343994140625,
      "logps/rejected": -582.3342895507812,
      "loss": 1.2513,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.265495538711548,
      "rewards/margins": -0.1880115270614624,
      "rewards/rejected": -2.077484130859375,
      "step": 10412
    },
    {
      "epoch": 1.362607344401734,
      "grad_norm": 8.988334655761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5627928376197815,
      "logits/rejected": -0.3164137303829193,
      "logps/chosen": -390.2731628417969,
      "logps/rejected": -537.7804565429688,
      "loss": 0.5014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47808611392974854,
      "rewards/margins": 1.7515232563018799,
      "rewards/rejected": -2.229609251022339,
      "step": 10413
    },
    {
      "epoch": 1.3627382023390857,
      "grad_norm": 7.269331455230713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2271287590265274,
      "logits/rejected": -0.4485563039779663,
      "logps/chosen": -368.259033203125,
      "logps/rejected": -334.27777099609375,
      "loss": 0.4173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9814630150794983,
      "rewards/margins": 0.9352938532829285,
      "rewards/rejected": -1.9167571067810059,
      "step": 10414
    },
    {
      "epoch": 1.3628690602764375,
      "grad_norm": 10.583443641662598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26825928688049316,
      "logits/rejected": -0.4730478525161743,
      "logps/chosen": -541.1685791015625,
      "logps/rejected": -470.4801025390625,
      "loss": 0.6249,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1870617866516113,
      "rewards/margins": 0.5063942074775696,
      "rewards/rejected": -1.6934561729431152,
      "step": 10415
    },
    {
      "epoch": 1.362999918213789,
      "grad_norm": 6.334174156188965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0624987818300724,
      "logits/rejected": -0.2132570743560791,
      "logps/chosen": -494.2528991699219,
      "logps/rejected": -471.834716796875,
      "loss": 0.3242,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6872289776802063,
      "rewards/margins": 1.3555595874786377,
      "rewards/rejected": -2.042788505554199,
      "step": 10416
    },
    {
      "epoch": 1.3631307761511409,
      "grad_norm": 20.17156410217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.295078307390213,
      "logits/rejected": -0.6004307866096497,
      "logps/chosen": -566.1077270507812,
      "logps/rejected": -543.717041015625,
      "loss": 0.8635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4862961769104004,
      "rewards/margins": 0.5068206787109375,
      "rewards/rejected": -1.9931167364120483,
      "step": 10417
    },
    {
      "epoch": 1.3632616340884927,
      "grad_norm": 6.091475486755371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4083884656429291,
      "logits/rejected": -0.2454853653907776,
      "logps/chosen": -265.1059875488281,
      "logps/rejected": -316.49969482421875,
      "loss": 0.4446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0908993482589722,
      "rewards/margins": 1.2100918292999268,
      "rewards/rejected": -2.3009910583496094,
      "step": 10418
    },
    {
      "epoch": 1.3633924920258444,
      "grad_norm": 10.05660343170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12323789298534393,
      "logits/rejected": -0.25811946392059326,
      "logps/chosen": -408.78717041015625,
      "logps/rejected": -453.6824035644531,
      "loss": 0.613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.452549934387207,
      "rewards/margins": 0.86910080909729,
      "rewards/rejected": -2.321650743484497,
      "step": 10419
    },
    {
      "epoch": 1.3635233499631962,
      "grad_norm": 11.729677200317383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.384907066822052,
      "logits/rejected": -0.45699095726013184,
      "logps/chosen": -420.96978759765625,
      "logps/rejected": -369.6034240722656,
      "loss": 0.6601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7445517778396606,
      "rewards/margins": 1.2763817310333252,
      "rewards/rejected": -2.0209333896636963,
      "step": 10420
    },
    {
      "epoch": 1.363654207900548,
      "grad_norm": 6.378741264343262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01742030680179596,
      "logits/rejected": -0.16287827491760254,
      "logps/chosen": -384.0259704589844,
      "logps/rejected": -384.0987548828125,
      "loss": 0.2809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04439011961221695,
      "rewards/margins": 1.6612505912780762,
      "rewards/rejected": -1.7056406736373901,
      "step": 10421
    },
    {
      "epoch": 1.3637850658378996,
      "grad_norm": 6.51513671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4345855116844177,
      "logits/rejected": -0.5642896294593811,
      "logps/chosen": -287.4941711425781,
      "logps/rejected": -375.6659851074219,
      "loss": 0.2618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2752711772918701,
      "rewards/margins": 2.364001750946045,
      "rewards/rejected": -2.6392734050750732,
      "step": 10422
    },
    {
      "epoch": 1.3639159237752514,
      "grad_norm": 7.526738166809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36719924211502075,
      "logits/rejected": -0.2284255474805832,
      "logps/chosen": -413.8395690917969,
      "logps/rejected": -371.3158264160156,
      "loss": 0.4232,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.811202883720398,
      "rewards/margins": 0.8137257099151611,
      "rewards/rejected": -1.6249284744262695,
      "step": 10423
    },
    {
      "epoch": 1.3640467817126032,
      "grad_norm": 7.930348873138428,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30711936950683594,
      "logits/rejected": -0.35209596157073975,
      "logps/chosen": -297.91619873046875,
      "logps/rejected": -500.31640625,
      "loss": 0.5782,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6333707571029663,
      "rewards/margins": 0.8569382429122925,
      "rewards/rejected": -2.490309000015259,
      "step": 10424
    },
    {
      "epoch": 1.364177639649955,
      "grad_norm": 8.182790756225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1644611656665802,
      "logits/rejected": -0.3570079207420349,
      "logps/chosen": -674.7137451171875,
      "logps/rejected": -522.39892578125,
      "loss": 0.4293,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4812791347503662,
      "rewards/margins": 0.9885453581809998,
      "rewards/rejected": -2.4698243141174316,
      "step": 10425
    },
    {
      "epoch": 1.3643084975873068,
      "grad_norm": 7.769591331481934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5966764688491821,
      "logits/rejected": -0.4915844798088074,
      "logps/chosen": -521.3577270507812,
      "logps/rejected": -484.7181091308594,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6816225647926331,
      "rewards/margins": 1.2606325149536133,
      "rewards/rejected": -1.9422551393508911,
      "step": 10426
    },
    {
      "epoch": 1.3644393555246586,
      "grad_norm": 10.206993103027344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24818333983421326,
      "logits/rejected": -0.18899478018283844,
      "logps/chosen": -492.1678466796875,
      "logps/rejected": -523.6446533203125,
      "loss": 0.5893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5791782140731812,
      "rewards/margins": 0.8437995910644531,
      "rewards/rejected": -1.4229779243469238,
      "step": 10427
    },
    {
      "epoch": 1.3645702134620104,
      "grad_norm": 13.492432594299316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10284791141748428,
      "logits/rejected": -0.4147987961769104,
      "logps/chosen": -647.6717529296875,
      "logps/rejected": -615.3442993164062,
      "loss": 0.7974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.872236967086792,
      "rewards/margins": 0.4173128306865692,
      "rewards/rejected": -2.2895500659942627,
      "step": 10428
    },
    {
      "epoch": 1.3647010713993621,
      "grad_norm": 10.721001625061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30560368299484253,
      "logits/rejected": -0.19818520545959473,
      "logps/chosen": -459.3935852050781,
      "logps/rejected": -364.6716003417969,
      "loss": 0.9484,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.445279836654663,
      "rewards/margins": 0.07065559923648834,
      "rewards/rejected": -1.5159354209899902,
      "step": 10429
    },
    {
      "epoch": 1.364831929336714,
      "grad_norm": 10.86825942993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043936096131801605,
      "logits/rejected": -0.2570686638355255,
      "logps/chosen": -661.5889282226562,
      "logps/rejected": -669.7530517578125,
      "loss": 0.5598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8703253269195557,
      "rewards/margins": 1.869762659072876,
      "rewards/rejected": -3.7400879859924316,
      "step": 10430
    },
    {
      "epoch": 1.3649627872740657,
      "grad_norm": 10.664383888244629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43113231658935547,
      "logits/rejected": -0.3723900616168976,
      "logps/chosen": -471.44866943359375,
      "logps/rejected": -670.5604248046875,
      "loss": 0.6841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5635600090026855,
      "rewards/margins": 0.7733544111251831,
      "rewards/rejected": -3.3369147777557373,
      "step": 10431
    },
    {
      "epoch": 1.3650936452114173,
      "grad_norm": 7.882673740386963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24711894989013672,
      "logits/rejected": -0.10665331780910492,
      "logps/chosen": -387.842041015625,
      "logps/rejected": -448.8387145996094,
      "loss": 0.4536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3815323114395142,
      "rewards/margins": 0.9711264967918396,
      "rewards/rejected": -2.352658748626709,
      "step": 10432
    },
    {
      "epoch": 1.365224503148769,
      "grad_norm": 6.502807140350342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3415345847606659,
      "logits/rejected": -0.3114243149757385,
      "logps/chosen": -369.7117004394531,
      "logps/rejected": -358.9775695800781,
      "loss": 0.2973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9521862268447876,
      "rewards/margins": 1.6224265098571777,
      "rewards/rejected": -2.5746123790740967,
      "step": 10433
    },
    {
      "epoch": 1.3653553610861209,
      "grad_norm": 5.757833003997803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3366769850254059,
      "logits/rejected": -0.2507534325122833,
      "logps/chosen": -525.6561279296875,
      "logps/rejected": -398.4527893066406,
      "loss": 0.2691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13201063871383667,
      "rewards/margins": 1.97029447555542,
      "rewards/rejected": -2.1023049354553223,
      "step": 10434
    },
    {
      "epoch": 1.3654862190234727,
      "grad_norm": 5.952395915985107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1940414309501648,
      "logits/rejected": -0.3589147925376892,
      "logps/chosen": -344.66485595703125,
      "logps/rejected": -382.90185546875,
      "loss": 0.277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1018474102020264,
      "rewards/margins": 2.170297622680664,
      "rewards/rejected": -3.2721450328826904,
      "step": 10435
    },
    {
      "epoch": 1.3656170769608245,
      "grad_norm": 5.958482265472412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04627919942140579,
      "logits/rejected": -0.24708497524261475,
      "logps/chosen": -400.38067626953125,
      "logps/rejected": -318.06182861328125,
      "loss": 0.3275,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6683138012886047,
      "rewards/margins": 2.0019943714141846,
      "rewards/rejected": -2.6703083515167236,
      "step": 10436
    },
    {
      "epoch": 1.365747934898176,
      "grad_norm": 12.722142219543457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2709534466266632,
      "logits/rejected": -0.5547629594802856,
      "logps/chosen": -506.0780029296875,
      "logps/rejected": -509.40142822265625,
      "loss": 0.7121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4386966228485107,
      "rewards/margins": 0.5126926898956299,
      "rewards/rejected": -1.9513894319534302,
      "step": 10437
    },
    {
      "epoch": 1.3658787928355278,
      "grad_norm": 8.73979377746582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22535814344882965,
      "logits/rejected": -0.21233177185058594,
      "logps/chosen": -426.88592529296875,
      "logps/rejected": -404.6587829589844,
      "loss": 0.7382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3021330833435059,
      "rewards/margins": 0.4500355124473572,
      "rewards/rejected": -1.7521686553955078,
      "step": 10438
    },
    {
      "epoch": 1.3660096507728796,
      "grad_norm": 10.827362060546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.681522786617279,
      "logits/rejected": -0.2555322051048279,
      "logps/chosen": -362.8313293457031,
      "logps/rejected": -411.08856201171875,
      "loss": 0.5888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1342768669128418,
      "rewards/margins": 1.178538203239441,
      "rewards/rejected": -2.3128151893615723,
      "step": 10439
    },
    {
      "epoch": 1.3661405087102314,
      "grad_norm": 6.775726795196533,
      "learning_rate": 5e-06,
      "logits/chosen": 0.061549000442028046,
      "logits/rejected": -0.5217671990394592,
      "logps/chosen": -382.7020263671875,
      "logps/rejected": -346.03460693359375,
      "loss": 0.3555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4319736957550049,
      "rewards/margins": 1.09037184715271,
      "rewards/rejected": -2.522345542907715,
      "step": 10440
    },
    {
      "epoch": 1.3662713666475832,
      "grad_norm": 12.264688491821289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05204299837350845,
      "logits/rejected": 0.22772836685180664,
      "logps/chosen": -649.5889282226562,
      "logps/rejected": -494.4391174316406,
      "loss": 0.8521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6837654113769531,
      "rewards/margins": 0.1593899130821228,
      "rewards/rejected": -0.8431553244590759,
      "step": 10441
    },
    {
      "epoch": 1.366402224584935,
      "grad_norm": 10.393848419189453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15305668115615845,
      "logits/rejected": -0.0645541399717331,
      "logps/chosen": -567.2313232421875,
      "logps/rejected": -531.25390625,
      "loss": 0.6019,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1243988275527954,
      "rewards/margins": 0.6196065545082092,
      "rewards/rejected": -1.7440054416656494,
      "step": 10442
    },
    {
      "epoch": 1.3665330825222868,
      "grad_norm": 10.168462753295898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44393086433410645,
      "logits/rejected": -0.25828424096107483,
      "logps/chosen": -417.72412109375,
      "logps/rejected": -312.99102783203125,
      "loss": 0.5768,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6471145153045654,
      "rewards/margins": 1.2073726654052734,
      "rewards/rejected": -1.8544871807098389,
      "step": 10443
    },
    {
      "epoch": 1.3666639404596386,
      "grad_norm": 6.197220325469971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10975000262260437,
      "logits/rejected": -0.05905391648411751,
      "logps/chosen": -402.2010192871094,
      "logps/rejected": -413.07806396484375,
      "loss": 0.3829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.462069034576416,
      "rewards/margins": 1.140917420387268,
      "rewards/rejected": -2.6029860973358154,
      "step": 10444
    },
    {
      "epoch": 1.3667947983969904,
      "grad_norm": 5.031139850616455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22317761182785034,
      "logits/rejected": -0.0921424850821495,
      "logps/chosen": -398.4530944824219,
      "logps/rejected": -403.97509765625,
      "loss": 0.289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0593675374984741,
      "rewards/margins": 1.5225962400436401,
      "rewards/rejected": -2.5819637775421143,
      "step": 10445
    },
    {
      "epoch": 1.3669256563343422,
      "grad_norm": 19.072751998901367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05820462107658386,
      "logits/rejected": -0.6626478433609009,
      "logps/chosen": -476.7078857421875,
      "logps/rejected": -520.1536254882812,
      "loss": 0.419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8201367855072021,
      "rewards/margins": 1.3331849575042725,
      "rewards/rejected": -3.1533219814300537,
      "step": 10446
    },
    {
      "epoch": 1.3670565142716937,
      "grad_norm": 5.958086967468262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1995311677455902,
      "logits/rejected": -0.22661952674388885,
      "logps/chosen": -543.6588134765625,
      "logps/rejected": -558.1957397460938,
      "loss": 0.3413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5919445753097534,
      "rewards/margins": 2.034926176071167,
      "rewards/rejected": -2.626870632171631,
      "step": 10447
    },
    {
      "epoch": 1.3671873722090455,
      "grad_norm": 9.005546569824219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09340633451938629,
      "logits/rejected": -0.16637109220027924,
      "logps/chosen": -414.5883483886719,
      "logps/rejected": -419.6481018066406,
      "loss": 0.4645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6722567081451416,
      "rewards/margins": 1.1678080558776855,
      "rewards/rejected": -1.840064525604248,
      "step": 10448
    },
    {
      "epoch": 1.3673182301463973,
      "grad_norm": 12.278395652770996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26582592725753784,
      "logits/rejected": -0.007369250059127808,
      "logps/chosen": -462.71734619140625,
      "logps/rejected": -357.1354675292969,
      "loss": 0.8759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1171308755874634,
      "rewards/margins": 0.18547113239765167,
      "rewards/rejected": -1.3026020526885986,
      "step": 10449
    },
    {
      "epoch": 1.367449088083749,
      "grad_norm": 9.274449348449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5411672592163086,
      "logits/rejected": -0.7000213861465454,
      "logps/chosen": -453.9927062988281,
      "logps/rejected": -436.8479919433594,
      "loss": 0.3276,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8673906326293945,
      "rewards/margins": 1.6640667915344238,
      "rewards/rejected": -2.5314574241638184,
      "step": 10450
    },
    {
      "epoch": 1.367579946021101,
      "grad_norm": 4.917515754699707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28861647844314575,
      "logits/rejected": -0.3071290850639343,
      "logps/chosen": -493.8757019042969,
      "logps/rejected": -574.5918579101562,
      "loss": 0.1805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0393128395080566,
      "rewards/margins": 2.601820468902588,
      "rewards/rejected": -3.6411337852478027,
      "step": 10451
    },
    {
      "epoch": 1.3677108039584527,
      "grad_norm": 5.62127161026001,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2573687732219696,
      "logits/rejected": -0.22184234857559204,
      "logps/chosen": -316.8011474609375,
      "logps/rejected": -323.8427734375,
      "loss": 0.2747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.799464225769043,
      "rewards/margins": 1.4694218635559082,
      "rewards/rejected": -2.268886089324951,
      "step": 10452
    },
    {
      "epoch": 1.3678416618958043,
      "grad_norm": 3.8077359199523926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2322494238615036,
      "logits/rejected": -0.32409435510635376,
      "logps/chosen": -346.47283935546875,
      "logps/rejected": -385.1868591308594,
      "loss": 0.1985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9721537232398987,
      "rewards/margins": 2.580822706222534,
      "rewards/rejected": -3.552976369857788,
      "step": 10453
    },
    {
      "epoch": 1.367972519833156,
      "grad_norm": 1.5404846668243408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4022411108016968,
      "logits/rejected": -0.6433695554733276,
      "logps/chosen": -308.6785888671875,
      "logps/rejected": -530.31982421875,
      "loss": 0.0473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6587693691253662,
      "rewards/margins": 3.4681556224823,
      "rewards/rejected": -4.126925468444824,
      "step": 10454
    },
    {
      "epoch": 1.3681033777705078,
      "grad_norm": 7.03960657119751,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32104387879371643,
      "logits/rejected": -0.5040839314460754,
      "logps/chosen": -604.4117431640625,
      "logps/rejected": -593.2186279296875,
      "loss": 0.2535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6098133325576782,
      "rewards/margins": 2.258474349975586,
      "rewards/rejected": -2.8682875633239746,
      "step": 10455
    },
    {
      "epoch": 1.3682342357078596,
      "grad_norm": 5.534370422363281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21396708488464355,
      "logits/rejected": -0.10549142956733704,
      "logps/chosen": -418.942138671875,
      "logps/rejected": -362.23455810546875,
      "loss": 0.3132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9392313957214355,
      "rewards/margins": 1.3565127849578857,
      "rewards/rejected": -2.2957441806793213,
      "step": 10456
    },
    {
      "epoch": 1.3683650936452114,
      "grad_norm": 8.233260154724121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10509779304265976,
      "logits/rejected": -0.33712393045425415,
      "logps/chosen": -468.162109375,
      "logps/rejected": -511.8578186035156,
      "loss": 0.3611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2260220050811768,
      "rewards/margins": 2.06150484085083,
      "rewards/rejected": -3.287527084350586,
      "step": 10457
    },
    {
      "epoch": 1.3684959515825632,
      "grad_norm": 8.91443157196045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008731663227081299,
      "logits/rejected": -0.14734260737895966,
      "logps/chosen": -490.5884094238281,
      "logps/rejected": -496.4438781738281,
      "loss": 0.4912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3509619235992432,
      "rewards/margins": 1.0217443704605103,
      "rewards/rejected": -2.372706174850464,
      "step": 10458
    },
    {
      "epoch": 1.368626809519915,
      "grad_norm": 8.884357452392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09999877214431763,
      "logits/rejected": 0.0329134464263916,
      "logps/chosen": -407.1862487792969,
      "logps/rejected": -531.825439453125,
      "loss": 0.3829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.460976481437683,
      "rewards/margins": 1.3958064317703247,
      "rewards/rejected": -2.856783151626587,
      "step": 10459
    },
    {
      "epoch": 1.3687576674572668,
      "grad_norm": 10.977022171020508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3415144979953766,
      "logits/rejected": -0.3106828033924103,
      "logps/chosen": -279.02752685546875,
      "logps/rejected": -270.0099182128906,
      "loss": 0.5689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7661109566688538,
      "rewards/margins": 1.3225022554397583,
      "rewards/rejected": -2.088613271713257,
      "step": 10460
    },
    {
      "epoch": 1.3688885253946186,
      "grad_norm": 6.604681015014648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1711277961730957,
      "logits/rejected": -0.2691977024078369,
      "logps/chosen": -736.4061279296875,
      "logps/rejected": -421.6349792480469,
      "loss": 0.3096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45833873748779297,
      "rewards/margins": 1.823560357093811,
      "rewards/rejected": -2.2818994522094727,
      "step": 10461
    },
    {
      "epoch": 1.3690193833319702,
      "grad_norm": 9.393521308898926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0933263823390007,
      "logits/rejected": -0.05597366392612457,
      "logps/chosen": -463.80364990234375,
      "logps/rejected": -483.0384521484375,
      "loss": 0.6014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7296537756919861,
      "rewards/margins": 1.0855046510696411,
      "rewards/rejected": -1.8151583671569824,
      "step": 10462
    },
    {
      "epoch": 1.369150241269322,
      "grad_norm": 6.4097795486450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1881529688835144,
      "logits/rejected": -0.4523301422595978,
      "logps/chosen": -511.847412109375,
      "logps/rejected": -490.5104064941406,
      "loss": 0.3177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34435176849365234,
      "rewards/margins": 2.7572109699249268,
      "rewards/rejected": -3.101562738418579,
      "step": 10463
    },
    {
      "epoch": 1.3692810992066737,
      "grad_norm": 14.5907564163208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21546508371829987,
      "logits/rejected": -0.4190703332424164,
      "logps/chosen": -295.1976013183594,
      "logps/rejected": -298.5552978515625,
      "loss": 0.8517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8354909420013428,
      "rewards/margins": 0.7350988388061523,
      "rewards/rejected": -2.570590019226074,
      "step": 10464
    },
    {
      "epoch": 1.3694119571440255,
      "grad_norm": 6.776937484741211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02134609781205654,
      "logits/rejected": -0.2561914920806885,
      "logps/chosen": -455.7866516113281,
      "logps/rejected": -451.28363037109375,
      "loss": 0.6539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.25061145424842834,
      "rewards/margins": 0.9756591320037842,
      "rewards/rejected": -0.7250477075576782,
      "step": 10465
    },
    {
      "epoch": 1.3695428150813773,
      "grad_norm": 8.63630485534668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18886831402778625,
      "logits/rejected": -0.12056266516447067,
      "logps/chosen": -432.2057189941406,
      "logps/rejected": -327.6192932128906,
      "loss": 0.4399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.076484203338623,
      "rewards/margins": 0.8975521922111511,
      "rewards/rejected": -1.9740363359451294,
      "step": 10466
    },
    {
      "epoch": 1.369673673018729,
      "grad_norm": 9.9882230758667,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1103140264749527,
      "logits/rejected": -0.04669608175754547,
      "logps/chosen": -407.5074157714844,
      "logps/rejected": -743.919921875,
      "loss": 0.5333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9426857233047485,
      "rewards/margins": 0.9995575547218323,
      "rewards/rejected": -1.9422430992126465,
      "step": 10467
    },
    {
      "epoch": 1.3698045309560807,
      "grad_norm": 7.893244743347168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19833335280418396,
      "logits/rejected": -0.4745296537876129,
      "logps/chosen": -393.40948486328125,
      "logps/rejected": -378.2191467285156,
      "loss": 0.5678,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0693233013153076,
      "rewards/margins": 0.49506011605262756,
      "rewards/rejected": -1.5643833875656128,
      "step": 10468
    },
    {
      "epoch": 1.3699353888934325,
      "grad_norm": 6.668851375579834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23546263575553894,
      "logits/rejected": -0.3243641257286072,
      "logps/chosen": -278.90545654296875,
      "logps/rejected": -383.2243347167969,
      "loss": 0.4228,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4283609688282013,
      "rewards/margins": 1.8719871044158936,
      "rewards/rejected": -2.3003480434417725,
      "step": 10469
    },
    {
      "epoch": 1.3700662468307843,
      "grad_norm": 4.4231367111206055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.160307839512825,
      "logits/rejected": -0.12289594113826752,
      "logps/chosen": -487.4425964355469,
      "logps/rejected": -542.3404541015625,
      "loss": 0.243,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1278955936431885,
      "rewards/margins": 2.1060068607330322,
      "rewards/rejected": -3.2339024543762207,
      "step": 10470
    },
    {
      "epoch": 1.370197104768136,
      "grad_norm": 6.941717624664307,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28722473978996277,
      "logits/rejected": -0.24426062405109406,
      "logps/chosen": -470.6944580078125,
      "logps/rejected": -385.08447265625,
      "loss": 0.3807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7366986274719238,
      "rewards/margins": 1.484076976776123,
      "rewards/rejected": -2.2207753658294678,
      "step": 10471
    },
    {
      "epoch": 1.3703279627054878,
      "grad_norm": 8.040960311889648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18300457298755646,
      "logits/rejected": 0.2456793487071991,
      "logps/chosen": -331.806396484375,
      "logps/rejected": -352.9250183105469,
      "loss": 0.4405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6414257287979126,
      "rewards/margins": 0.8293517827987671,
      "rewards/rejected": -2.4707775115966797,
      "step": 10472
    },
    {
      "epoch": 1.3704588206428396,
      "grad_norm": 6.096462726593018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2446153461933136,
      "logits/rejected": -0.25433287024497986,
      "logps/chosen": -553.40234375,
      "logps/rejected": -707.1741943359375,
      "loss": 0.2237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.032280445098877,
      "rewards/margins": 2.5477023124694824,
      "rewards/rejected": -3.5799827575683594,
      "step": 10473
    },
    {
      "epoch": 1.3705896785801914,
      "grad_norm": 6.201657295227051,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4010135233402252,
      "logits/rejected": -0.40138113498687744,
      "logps/chosen": -503.8418884277344,
      "logps/rejected": -458.89703369140625,
      "loss": 0.3744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.474645972251892,
      "rewards/margins": 1.3453494310379028,
      "rewards/rejected": -2.819995403289795,
      "step": 10474
    },
    {
      "epoch": 1.3707205365175432,
      "grad_norm": 13.65198802947998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4196167290210724,
      "logits/rejected": -0.20493483543395996,
      "logps/chosen": -521.8748168945312,
      "logps/rejected": -582.14404296875,
      "loss": 0.5214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4285283088684082,
      "rewards/margins": 0.8550751209259033,
      "rewards/rejected": -2.2836031913757324,
      "step": 10475
    },
    {
      "epoch": 1.370851394454895,
      "grad_norm": 7.366474151611328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05878286808729172,
      "logits/rejected": -0.12882071733474731,
      "logps/chosen": -474.4847717285156,
      "logps/rejected": -424.8869934082031,
      "loss": 0.4656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.489157110452652,
      "rewards/margins": 0.7774204611778259,
      "rewards/rejected": -1.2665774822235107,
      "step": 10476
    },
    {
      "epoch": 1.3709822523922468,
      "grad_norm": 6.613002777099609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29476821422576904,
      "logits/rejected": -0.5271221399307251,
      "logps/chosen": -610.4637451171875,
      "logps/rejected": -432.892333984375,
      "loss": 0.3081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5811092853546143,
      "rewards/margins": 1.5871999263763428,
      "rewards/rejected": -2.168309211730957,
      "step": 10477
    },
    {
      "epoch": 1.3711131103295984,
      "grad_norm": 6.477119445800781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1475248634815216,
      "logits/rejected": -0.36851346492767334,
      "logps/chosen": -351.66778564453125,
      "logps/rejected": -337.6770324707031,
      "loss": 0.4461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2965429425239563,
      "rewards/margins": 0.7469648718833923,
      "rewards/rejected": -1.0435078144073486,
      "step": 10478
    },
    {
      "epoch": 1.3712439682669502,
      "grad_norm": 6.762166976928711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15525421500205994,
      "logits/rejected": -0.1781640201807022,
      "logps/chosen": -312.0960693359375,
      "logps/rejected": -365.5127868652344,
      "loss": 0.4703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4518074989318848,
      "rewards/margins": 0.6893522143363953,
      "rewards/rejected": -2.141160011291504,
      "step": 10479
    },
    {
      "epoch": 1.371374826204302,
      "grad_norm": 11.379217147827148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0862632542848587,
      "logits/rejected": 0.10211540758609772,
      "logps/chosen": -371.8349609375,
      "logps/rejected": -331.5736389160156,
      "loss": 0.6124,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.184154987335205,
      "rewards/margins": 0.6223669052124023,
      "rewards/rejected": -1.8065218925476074,
      "step": 10480
    },
    {
      "epoch": 1.3715056841416537,
      "grad_norm": 8.297191619873047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14092884957790375,
      "logits/rejected": -0.277023583650589,
      "logps/chosen": -493.66656494140625,
      "logps/rejected": -404.49560546875,
      "loss": 0.6948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4494590759277344,
      "rewards/margins": 0.32691842317581177,
      "rewards/rejected": -0.7763774991035461,
      "step": 10481
    },
    {
      "epoch": 1.3716365420790055,
      "grad_norm": 7.009698390960693,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04351755231618881,
      "logits/rejected": -0.017557866871356964,
      "logps/chosen": -474.8609924316406,
      "logps/rejected": -514.240234375,
      "loss": 0.4434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8874070048332214,
      "rewards/margins": 0.8070650100708008,
      "rewards/rejected": -1.6944719552993774,
      "step": 10482
    },
    {
      "epoch": 1.371767400016357,
      "grad_norm": 8.709263801574707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08751778304576874,
      "logits/rejected": -0.21238142251968384,
      "logps/chosen": -398.35870361328125,
      "logps/rejected": -376.8105163574219,
      "loss": 0.5166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09145237505435944,
      "rewards/margins": 1.1404199600219727,
      "rewards/rejected": -1.231872320175171,
      "step": 10483
    },
    {
      "epoch": 1.371898257953709,
      "grad_norm": 8.164539337158203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05554991215467453,
      "logits/rejected": -0.1814412623643875,
      "logps/chosen": -374.99285888671875,
      "logps/rejected": -451.487060546875,
      "loss": 0.4209,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25112801790237427,
      "rewards/margins": 1.7223684787750244,
      "rewards/rejected": -1.973496437072754,
      "step": 10484
    },
    {
      "epoch": 1.3720291158910607,
      "grad_norm": 12.957825660705566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25672903656959534,
      "logits/rejected": -0.32637497782707214,
      "logps/chosen": -525.7137451171875,
      "logps/rejected": -382.296630859375,
      "loss": 0.4786,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.063584566116333,
      "rewards/margins": 1.1761969327926636,
      "rewards/rejected": -2.239781379699707,
      "step": 10485
    },
    {
      "epoch": 1.3721599738284125,
      "grad_norm": 12.59904670715332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19742374122142792,
      "logits/rejected": -0.30829229950904846,
      "logps/chosen": -633.6824951171875,
      "logps/rejected": -553.583740234375,
      "loss": 0.6559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22840212285518646,
      "rewards/margins": 0.7962697744369507,
      "rewards/rejected": -1.0246719121932983,
      "step": 10486
    },
    {
      "epoch": 1.3722908317657643,
      "grad_norm": 8.47891902923584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21878813207149506,
      "logits/rejected": -0.14304251968860626,
      "logps/chosen": -520.08203125,
      "logps/rejected": -338.0706787109375,
      "loss": 0.5297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.065726399421692,
      "rewards/margins": 0.8971168994903564,
      "rewards/rejected": -1.9628431797027588,
      "step": 10487
    },
    {
      "epoch": 1.372421689703116,
      "grad_norm": 8.414229393005371,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10637497901916504,
      "logits/rejected": -0.11015386134386063,
      "logps/chosen": -469.52593994140625,
      "logps/rejected": -404.830078125,
      "loss": 0.5403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.501005232334137,
      "rewards/margins": 0.7391750812530518,
      "rewards/rejected": -1.240180253982544,
      "step": 10488
    },
    {
      "epoch": 1.3725525476404679,
      "grad_norm": 5.930177688598633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025689855217933655,
      "logits/rejected": -0.025289960205554962,
      "logps/chosen": -450.6316833496094,
      "logps/rejected": -578.2980346679688,
      "loss": 0.3281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07519551366567612,
      "rewards/margins": 1.7462704181671143,
      "rewards/rejected": -1.6710749864578247,
      "step": 10489
    },
    {
      "epoch": 1.3726834055778196,
      "grad_norm": 9.447725296020508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0873233899474144,
      "logits/rejected": -0.2873317003250122,
      "logps/chosen": -373.84796142578125,
      "logps/rejected": -278.1058044433594,
      "loss": 0.487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0059540271759033,
      "rewards/margins": 0.6239513754844666,
      "rewards/rejected": -1.6299054622650146,
      "step": 10490
    },
    {
      "epoch": 1.3728142635151714,
      "grad_norm": 8.908039093017578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02110128104686737,
      "logits/rejected": -0.0927184522151947,
      "logps/chosen": -564.2156982421875,
      "logps/rejected": -430.82080078125,
      "loss": 0.4973,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9456579685211182,
      "rewards/margins": 1.016095519065857,
      "rewards/rejected": -1.9617536067962646,
      "step": 10491
    },
    {
      "epoch": 1.3729451214525232,
      "grad_norm": 4.431573390960693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4326229691505432,
      "logits/rejected": -0.7876023054122925,
      "logps/chosen": -366.60198974609375,
      "logps/rejected": -330.0455017089844,
      "loss": 0.3383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3204164505004883,
      "rewards/margins": 1.3775269985198975,
      "rewards/rejected": -2.697943687438965,
      "step": 10492
    },
    {
      "epoch": 1.3730759793898748,
      "grad_norm": 8.030485153198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4004814326763153,
      "logits/rejected": -0.7740214467048645,
      "logps/chosen": -685.4911499023438,
      "logps/rejected": -660.823486328125,
      "loss": 0.3161,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.030495047569275,
      "rewards/margins": 1.8933732509613037,
      "rewards/rejected": -2.923868179321289,
      "step": 10493
    },
    {
      "epoch": 1.3732068373272266,
      "grad_norm": 12.271553039550781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4088282287120819,
      "logits/rejected": -0.41725432872772217,
      "logps/chosen": -414.1759948730469,
      "logps/rejected": -362.7650146484375,
      "loss": 0.4897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5299984216690063,
      "rewards/margins": 1.0599455833435059,
      "rewards/rejected": -2.5899438858032227,
      "step": 10494
    },
    {
      "epoch": 1.3733376952645784,
      "grad_norm": 9.389384269714355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2646954655647278,
      "logits/rejected": -0.13893720507621765,
      "logps/chosen": -500.5760192871094,
      "logps/rejected": -555.4074096679688,
      "loss": 0.5771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8453222513198853,
      "rewards/margins": 0.9949671030044556,
      "rewards/rejected": -1.8402893543243408,
      "step": 10495
    },
    {
      "epoch": 1.3734685532019302,
      "grad_norm": 8.856849670410156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31239551305770874,
      "logits/rejected": -0.2776240408420563,
      "logps/chosen": -358.4605407714844,
      "logps/rejected": -499.1172790527344,
      "loss": 0.6481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1802314519882202,
      "rewards/margins": 0.41859322786331177,
      "rewards/rejected": -1.5988247394561768,
      "step": 10496
    },
    {
      "epoch": 1.373599411139282,
      "grad_norm": 7.869678020477295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15784704685211182,
      "logits/rejected": -0.03146637976169586,
      "logps/chosen": -382.0548400878906,
      "logps/rejected": -444.5782165527344,
      "loss": 0.4696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7310325503349304,
      "rewards/margins": 1.4054818153381348,
      "rewards/rejected": -2.136514663696289,
      "step": 10497
    },
    {
      "epoch": 1.3737302690766338,
      "grad_norm": 4.309054374694824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37696966528892517,
      "logits/rejected": -0.42458921670913696,
      "logps/chosen": -541.5811767578125,
      "logps/rejected": -673.28955078125,
      "loss": 0.1983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3082721531391144,
      "rewards/margins": 3.020021677017212,
      "rewards/rejected": -2.71174955368042,
      "step": 10498
    },
    {
      "epoch": 1.3738611270139853,
      "grad_norm": 5.801474571228027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3649638891220093,
      "logits/rejected": -0.3028610050678253,
      "logps/chosen": -236.32566833496094,
      "logps/rejected": -296.44036865234375,
      "loss": 0.3821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4389747977256775,
      "rewards/margins": 1.6511667966842651,
      "rewards/rejected": -2.090141534805298,
      "step": 10499
    },
    {
      "epoch": 1.3739919849513371,
      "grad_norm": 7.54814338684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1202133521437645,
      "logits/rejected": -0.4170846939086914,
      "logps/chosen": -347.97894287109375,
      "logps/rejected": -429.29180908203125,
      "loss": 0.5956,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.235532283782959,
      "rewards/margins": 1.41209077835083,
      "rewards/rejected": -2.64762282371521,
      "step": 10500
    },
    {
      "epoch": 1.374122842888689,
      "grad_norm": 13.437875747680664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3252139985561371,
      "logits/rejected": -0.31004515290260315,
      "logps/chosen": -454.6953430175781,
      "logps/rejected": -391.5938720703125,
      "loss": 0.7956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2613426446914673,
      "rewards/margins": 0.8595644235610962,
      "rewards/rejected": -2.1209073066711426,
      "step": 10501
    },
    {
      "epoch": 1.3742537008260407,
      "grad_norm": 6.803210735321045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2482949197292328,
      "logits/rejected": -0.4792305529117584,
      "logps/chosen": -526.2839965820312,
      "logps/rejected": -401.0738525390625,
      "loss": 0.395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15194781124591827,
      "rewards/margins": 1.3831684589385986,
      "rewards/rejected": -1.535116195678711,
      "step": 10502
    },
    {
      "epoch": 1.3743845587633925,
      "grad_norm": 6.1129560470581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6299595236778259,
      "logits/rejected": -0.6209463477134705,
      "logps/chosen": -338.8807678222656,
      "logps/rejected": -515.2237548828125,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1483993530273438,
      "rewards/margins": 1.3855676651000977,
      "rewards/rejected": -2.5339672565460205,
      "step": 10503
    },
    {
      "epoch": 1.3745154167007443,
      "grad_norm": 4.192041397094727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3217664062976837,
      "logits/rejected": -0.551853358745575,
      "logps/chosen": -506.64471435546875,
      "logps/rejected": -493.7668151855469,
      "loss": 0.1899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3977454900741577,
      "rewards/margins": 2.0375730991363525,
      "rewards/rejected": -3.435318946838379,
      "step": 10504
    },
    {
      "epoch": 1.374646274638096,
      "grad_norm": 7.3037028312683105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21835197508335114,
      "logits/rejected": -0.18972498178482056,
      "logps/chosen": -569.1005249023438,
      "logps/rejected": -462.48223876953125,
      "loss": 0.2803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12828561663627625,
      "rewards/margins": 2.2383127212524414,
      "rewards/rejected": -2.36659836769104,
      "step": 10505
    },
    {
      "epoch": 1.3747771325754479,
      "grad_norm": 8.585223197937012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5186607241630554,
      "logits/rejected": -0.3107639253139496,
      "logps/chosen": -390.932373046875,
      "logps/rejected": -563.783935546875,
      "loss": 0.4513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1597530841827393,
      "rewards/margins": 1.4677574634552002,
      "rewards/rejected": -2.6275107860565186,
      "step": 10506
    },
    {
      "epoch": 1.3749079905127997,
      "grad_norm": 4.8427958488464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1720978319644928,
      "logits/rejected": 0.11045817285776138,
      "logps/chosen": -391.33599853515625,
      "logps/rejected": -364.167724609375,
      "loss": 0.2555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6000876426696777,
      "rewards/margins": 1.5364959239959717,
      "rewards/rejected": -2.1365835666656494,
      "step": 10507
    },
    {
      "epoch": 1.3750388484501512,
      "grad_norm": 8.374200820922852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24764645099639893,
      "logits/rejected": -0.10613764077425003,
      "logps/chosen": -594.4247436523438,
      "logps/rejected": -443.624755859375,
      "loss": 0.5506,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6648406386375427,
      "rewards/margins": 1.6115443706512451,
      "rewards/rejected": -2.2763850688934326,
      "step": 10508
    },
    {
      "epoch": 1.375169706387503,
      "grad_norm": 6.336542129516602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32625681161880493,
      "logits/rejected": -0.37800273299217224,
      "logps/chosen": -391.0718994140625,
      "logps/rejected": -468.96392822265625,
      "loss": 0.4021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5644669532775879,
      "rewards/margins": 0.8975304365158081,
      "rewards/rejected": -1.4619975090026855,
      "step": 10509
    },
    {
      "epoch": 1.3753005643248548,
      "grad_norm": 9.274660110473633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0500217042863369,
      "logits/rejected": -0.0330355279147625,
      "logps/chosen": -697.7822265625,
      "logps/rejected": -657.619140625,
      "loss": 0.4018,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17951221764087677,
      "rewards/margins": 1.4265577793121338,
      "rewards/rejected": -1.606069803237915,
      "step": 10510
    },
    {
      "epoch": 1.3754314222622066,
      "grad_norm": 12.46082878112793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3950321078300476,
      "logits/rejected": -0.23402953147888184,
      "logps/chosen": -472.7938537597656,
      "logps/rejected": -493.30364990234375,
      "loss": 0.8057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8500456213951111,
      "rewards/margins": 0.39032232761383057,
      "rewards/rejected": -1.2403678894042969,
      "step": 10511
    },
    {
      "epoch": 1.3755622801995584,
      "grad_norm": 6.687821865081787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1893189698457718,
      "logits/rejected": -0.08004458993673325,
      "logps/chosen": -363.94671630859375,
      "logps/rejected": -414.9603576660156,
      "loss": 0.3124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49144211411476135,
      "rewards/margins": 1.5810288190841675,
      "rewards/rejected": -2.0724709033966064,
      "step": 10512
    },
    {
      "epoch": 1.3756931381369102,
      "grad_norm": 7.563689708709717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.304019957780838,
      "logits/rejected": -0.19667203724384308,
      "logps/chosen": -542.0488891601562,
      "logps/rejected": -486.2236022949219,
      "loss": 0.4669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.040233805775642395,
      "rewards/margins": 0.9905603528022766,
      "rewards/rejected": -0.9503265023231506,
      "step": 10513
    },
    {
      "epoch": 1.3758239960742618,
      "grad_norm": 5.614711761474609,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11896608024835587,
      "logits/rejected": 0.004100168123841286,
      "logps/chosen": -592.0436401367188,
      "logps/rejected": -542.8969116210938,
      "loss": 0.1449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3961696922779083,
      "rewards/margins": 2.650623321533203,
      "rewards/rejected": -2.254453659057617,
      "step": 10514
    },
    {
      "epoch": 1.3759548540116135,
      "grad_norm": 9.715315818786621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19913989305496216,
      "logits/rejected": -0.2537683844566345,
      "logps/chosen": -545.4649047851562,
      "logps/rejected": -597.4027099609375,
      "loss": 0.4311,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2981830835342407,
      "rewards/margins": 1.6563993692398071,
      "rewards/rejected": -2.954582452774048,
      "step": 10515
    },
    {
      "epoch": 1.3760857119489653,
      "grad_norm": 12.79353141784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26958945393562317,
      "logits/rejected": 0.05660177767276764,
      "logps/chosen": -385.36553955078125,
      "logps/rejected": -465.972900390625,
      "loss": 0.6784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4450711011886597,
      "rewards/margins": 0.7165452241897583,
      "rewards/rejected": -2.161616086959839,
      "step": 10516
    },
    {
      "epoch": 1.3762165698863171,
      "grad_norm": 4.551272869110107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2250867784023285,
      "logits/rejected": -0.2908034324645996,
      "logps/chosen": -531.1481323242188,
      "logps/rejected": -385.85699462890625,
      "loss": 0.1379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1398431956768036,
      "rewards/margins": 2.3348004817962646,
      "rewards/rejected": -2.4746437072753906,
      "step": 10517
    },
    {
      "epoch": 1.376347427823669,
      "grad_norm": 4.900611400604248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05096166580915451,
      "logits/rejected": -0.05318165943026543,
      "logps/chosen": -570.506103515625,
      "logps/rejected": -477.91064453125,
      "loss": 0.2558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5107731223106384,
      "rewards/margins": 1.7927216291427612,
      "rewards/rejected": -2.303494691848755,
      "step": 10518
    },
    {
      "epoch": 1.3764782857610207,
      "grad_norm": 9.531710624694824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.374754935503006,
      "logits/rejected": -0.45007818937301636,
      "logps/chosen": -522.58056640625,
      "logps/rejected": -469.6246643066406,
      "loss": 0.4308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7355397939682007,
      "rewards/margins": 0.9015038013458252,
      "rewards/rejected": -1.6370435953140259,
      "step": 10519
    },
    {
      "epoch": 1.3766091436983725,
      "grad_norm": 4.941990852355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6554068922996521,
      "logits/rejected": -0.6623731851577759,
      "logps/chosen": -459.6617126464844,
      "logps/rejected": -395.82696533203125,
      "loss": 0.2379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6958974599838257,
      "rewards/margins": 2.0099825859069824,
      "rewards/rejected": -2.7058796882629395,
      "step": 10520
    },
    {
      "epoch": 1.3767400016357243,
      "grad_norm": 4.918827533721924,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03395499289035797,
      "logits/rejected": -0.14627331495285034,
      "logps/chosen": -332.3647766113281,
      "logps/rejected": -334.7855224609375,
      "loss": 0.2565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35969260334968567,
      "rewards/margins": 1.8900115489959717,
      "rewards/rejected": -2.249704360961914,
      "step": 10521
    },
    {
      "epoch": 1.376870859573076,
      "grad_norm": 8.497004508972168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.136700838804245,
      "logits/rejected": -0.004916561767458916,
      "logps/chosen": -358.6595764160156,
      "logps/rejected": -427.087646484375,
      "loss": 0.5872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5981237292289734,
      "rewards/margins": 0.7768192887306213,
      "rewards/rejected": -1.3749428987503052,
      "step": 10522
    },
    {
      "epoch": 1.3770017175104279,
      "grad_norm": 11.178438186645508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14770101010799408,
      "logits/rejected": -0.18415509164333344,
      "logps/chosen": -518.3283081054688,
      "logps/rejected": -546.6551513671875,
      "loss": 0.5039,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2960615158081055,
      "rewards/margins": 1.8347594738006592,
      "rewards/rejected": -3.1308207511901855,
      "step": 10523
    },
    {
      "epoch": 1.3771325754477795,
      "grad_norm": 5.647399425506592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16078601777553558,
      "logits/rejected": -0.3148254454135895,
      "logps/chosen": -351.3879699707031,
      "logps/rejected": -397.40960693359375,
      "loss": 0.2221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5528903007507324,
      "rewards/margins": 2.631068706512451,
      "rewards/rejected": -3.1839587688446045,
      "step": 10524
    },
    {
      "epoch": 1.3772634333851312,
      "grad_norm": 10.231287956237793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019096240401268005,
      "logits/rejected": -0.1638191193342209,
      "logps/chosen": -393.76531982421875,
      "logps/rejected": -376.25439453125,
      "loss": 0.7186,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.24761086702346802,
      "rewards/margins": 0.6435561180114746,
      "rewards/rejected": -0.8911669850349426,
      "step": 10525
    },
    {
      "epoch": 1.377394291322483,
      "grad_norm": 3.1346781253814697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.046743378043174744,
      "logits/rejected": -0.20720884203910828,
      "logps/chosen": -446.9202575683594,
      "logps/rejected": -483.0230712890625,
      "loss": 0.1061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08630862832069397,
      "rewards/margins": 3.039788007736206,
      "rewards/rejected": -3.1260969638824463,
      "step": 10526
    },
    {
      "epoch": 1.3775251492598348,
      "grad_norm": 5.953627586364746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11775514483451843,
      "logits/rejected": -0.13708099722862244,
      "logps/chosen": -510.1291198730469,
      "logps/rejected": -503.4228515625,
      "loss": 0.2837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7350826263427734,
      "rewards/margins": 1.7507719993591309,
      "rewards/rejected": -2.4858546257019043,
      "step": 10527
    },
    {
      "epoch": 1.3776560071971866,
      "grad_norm": 6.195118427276611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2786315679550171,
      "logits/rejected": -0.35866397619247437,
      "logps/chosen": -801.2020263671875,
      "logps/rejected": -570.9448852539062,
      "loss": 0.2733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4594736099243164,
      "rewards/margins": 1.5911716222763062,
      "rewards/rejected": -2.050645112991333,
      "step": 10528
    },
    {
      "epoch": 1.3777868651345382,
      "grad_norm": 11.536008834838867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18120934069156647,
      "logits/rejected": -0.36640292406082153,
      "logps/chosen": -450.62286376953125,
      "logps/rejected": -452.2978210449219,
      "loss": 0.5507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1250752210617065,
      "rewards/margins": 1.0223658084869385,
      "rewards/rejected": -2.1474409103393555,
      "step": 10529
    },
    {
      "epoch": 1.37791772307189,
      "grad_norm": 5.508985996246338,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0654735118150711,
      "logits/rejected": -0.13483262062072754,
      "logps/chosen": -546.34326171875,
      "logps/rejected": -465.15240478515625,
      "loss": 0.2589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6354681253433228,
      "rewards/margins": 2.300257921218872,
      "rewards/rejected": -2.9357261657714844,
      "step": 10530
    },
    {
      "epoch": 1.3780485810092418,
      "grad_norm": 4.06218147277832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27431708574295044,
      "logits/rejected": -0.36826950311660767,
      "logps/chosen": -311.5467834472656,
      "logps/rejected": -325.9585876464844,
      "loss": 0.2335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2592090666294098,
      "rewards/margins": 1.958483099937439,
      "rewards/rejected": -2.2176923751831055,
      "step": 10531
    },
    {
      "epoch": 1.3781794389465936,
      "grad_norm": 5.0167236328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1274184286594391,
      "logits/rejected": -0.35623306035995483,
      "logps/chosen": -518.607177734375,
      "logps/rejected": -518.6198120117188,
      "loss": 0.253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4813595116138458,
      "rewards/margins": 1.7099223136901855,
      "rewards/rejected": -2.191281795501709,
      "step": 10532
    },
    {
      "epoch": 1.3783102968839454,
      "grad_norm": 11.524362564086914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19945356249809265,
      "logits/rejected": -0.4159572720527649,
      "logps/chosen": -697.9923095703125,
      "logps/rejected": -463.54962158203125,
      "loss": 0.5067,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6899465322494507,
      "rewards/margins": 1.288337230682373,
      "rewards/rejected": -1.9782836437225342,
      "step": 10533
    },
    {
      "epoch": 1.3784411548212971,
      "grad_norm": 8.562788009643555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03578084707260132,
      "logits/rejected": 0.025133389979600906,
      "logps/chosen": -374.1824645996094,
      "logps/rejected": -359.8235778808594,
      "loss": 0.3959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48969632387161255,
      "rewards/margins": 0.9519869089126587,
      "rewards/rejected": -1.4416831731796265,
      "step": 10534
    },
    {
      "epoch": 1.378572012758649,
      "grad_norm": 11.858963012695312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1962842047214508,
      "logits/rejected": -0.23024627566337585,
      "logps/chosen": -543.4503784179688,
      "logps/rejected": -498.97186279296875,
      "loss": 0.6915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1154987812042236,
      "rewards/margins": 0.48004382848739624,
      "rewards/rejected": -1.595542550086975,
      "step": 10535
    },
    {
      "epoch": 1.3787028706960007,
      "grad_norm": 6.475682258605957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18885256350040436,
      "logits/rejected": -0.16993406414985657,
      "logps/chosen": -343.09710693359375,
      "logps/rejected": -341.7547912597656,
      "loss": 0.391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5557159781455994,
      "rewards/margins": 1.9161968231201172,
      "rewards/rejected": -2.4719126224517822,
      "step": 10536
    },
    {
      "epoch": 1.3788337286333525,
      "grad_norm": 10.902499198913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2600056529045105,
      "logits/rejected": -0.3585891127586365,
      "logps/chosen": -445.1685485839844,
      "logps/rejected": -411.61346435546875,
      "loss": 0.7869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4028209447860718,
      "rewards/margins": 0.33245059847831726,
      "rewards/rejected": -1.7352715730667114,
      "step": 10537
    },
    {
      "epoch": 1.3789645865707043,
      "grad_norm": 10.805221557617188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5178658962249756,
      "logits/rejected": -0.422923743724823,
      "logps/chosen": -328.5816955566406,
      "logps/rejected": -389.3001708984375,
      "loss": 0.6236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.125651478767395,
      "rewards/margins": 1.0290051698684692,
      "rewards/rejected": -2.1546568870544434,
      "step": 10538
    },
    {
      "epoch": 1.3790954445080559,
      "grad_norm": 13.824049949645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2929631173610687,
      "logits/rejected": -0.181192085146904,
      "logps/chosen": -467.9989318847656,
      "logps/rejected": -521.9932250976562,
      "loss": 0.8284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.394897699356079,
      "rewards/margins": 0.7357844114303589,
      "rewards/rejected": -2.1306819915771484,
      "step": 10539
    },
    {
      "epoch": 1.3792263024454077,
      "grad_norm": 8.652021408081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2592216730117798,
      "logits/rejected": -0.43943461775779724,
      "logps/chosen": -377.6814880371094,
      "logps/rejected": -388.0931701660156,
      "loss": 0.5645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1032074689865112,
      "rewards/margins": 1.277092695236206,
      "rewards/rejected": -2.380300283432007,
      "step": 10540
    },
    {
      "epoch": 1.3793571603827595,
      "grad_norm": 4.743745803833008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0203159861266613,
      "logits/rejected": -0.15661966800689697,
      "logps/chosen": -412.5817565917969,
      "logps/rejected": -404.0234069824219,
      "loss": 0.2995,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9401424527168274,
      "rewards/margins": 1.2943825721740723,
      "rewards/rejected": -2.234524965286255,
      "step": 10541
    },
    {
      "epoch": 1.3794880183201113,
      "grad_norm": 6.90781831741333,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3072260618209839,
      "logits/rejected": -0.37444961071014404,
      "logps/chosen": -501.2597961425781,
      "logps/rejected": -742.0286254882812,
      "loss": 0.3065,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2595853805541992,
      "rewards/margins": 1.6111055612564087,
      "rewards/rejected": -1.870690941810608,
      "step": 10542
    },
    {
      "epoch": 1.379618876257463,
      "grad_norm": 8.65804386138916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1391327977180481,
      "logits/rejected": -0.231511652469635,
      "logps/chosen": -462.37103271484375,
      "logps/rejected": -487.8043518066406,
      "loss": 0.5768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3865318298339844,
      "rewards/margins": 0.9539300203323364,
      "rewards/rejected": -1.3404619693756104,
      "step": 10543
    },
    {
      "epoch": 1.3797497341948148,
      "grad_norm": 6.772054195404053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14674796164035797,
      "logits/rejected": -0.31027913093566895,
      "logps/chosen": -499.34429931640625,
      "logps/rejected": -380.59674072265625,
      "loss": 0.3987,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8864944577217102,
      "rewards/margins": 1.185699701309204,
      "rewards/rejected": -2.0721940994262695,
      "step": 10544
    },
    {
      "epoch": 1.3798805921321664,
      "grad_norm": 9.491503715515137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28936392068862915,
      "logits/rejected": -0.21740677952766418,
      "logps/chosen": -461.9614562988281,
      "logps/rejected": -541.8849487304688,
      "loss": 0.4318,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9551525712013245,
      "rewards/margins": 1.5574091672897339,
      "rewards/rejected": -2.512561798095703,
      "step": 10545
    },
    {
      "epoch": 1.3800114500695182,
      "grad_norm": 7.84584903717041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2523983120918274,
      "logits/rejected": -0.1121724545955658,
      "logps/chosen": -512.12939453125,
      "logps/rejected": -431.9474182128906,
      "loss": 0.5997,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34312668442726135,
      "rewards/margins": 0.9275945425033569,
      "rewards/rejected": -1.2707213163375854,
      "step": 10546
    },
    {
      "epoch": 1.38014230800687,
      "grad_norm": 9.03830623626709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30709192156791687,
      "logits/rejected": -0.4075654149055481,
      "logps/chosen": -234.10243225097656,
      "logps/rejected": -374.93426513671875,
      "loss": 0.634,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2238162755966187,
      "rewards/margins": 0.6010855436325073,
      "rewards/rejected": -1.824901819229126,
      "step": 10547
    },
    {
      "epoch": 1.3802731659442218,
      "grad_norm": 8.588120460510254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016112007200717926,
      "logits/rejected": -0.10280995070934296,
      "logps/chosen": -446.10992431640625,
      "logps/rejected": -355.75555419921875,
      "loss": 0.3213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.90021151304245,
      "rewards/margins": 1.2682948112487793,
      "rewards/rejected": -2.168506145477295,
      "step": 10548
    },
    {
      "epoch": 1.3804040238815736,
      "grad_norm": 10.4304838180542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5627358555793762,
      "logits/rejected": -0.26220592856407166,
      "logps/chosen": -541.2320556640625,
      "logps/rejected": -551.388916015625,
      "loss": 0.4992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6487724781036377,
      "rewards/margins": 0.8858169913291931,
      "rewards/rejected": -2.5345895290374756,
      "step": 10549
    },
    {
      "epoch": 1.3805348818189254,
      "grad_norm": 10.454106330871582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24848756194114685,
      "logits/rejected": 0.25845491886138916,
      "logps/chosen": -650.9991455078125,
      "logps/rejected": -591.8218383789062,
      "loss": 0.7362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18090784549713135,
      "rewards/margins": 0.8679494857788086,
      "rewards/rejected": -1.04885733127594,
      "step": 10550
    },
    {
      "epoch": 1.3806657397562772,
      "grad_norm": 8.026755332946777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38516873121261597,
      "logits/rejected": -0.2973739206790924,
      "logps/chosen": -584.413330078125,
      "logps/rejected": -448.70001220703125,
      "loss": 0.3185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2808980941772461,
      "rewards/margins": 1.439328670501709,
      "rewards/rejected": -1.720226764678955,
      "step": 10551
    },
    {
      "epoch": 1.380796597693629,
      "grad_norm": 16.462556838989258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17856240272521973,
      "logits/rejected": -0.2221827507019043,
      "logps/chosen": -565.2288208007812,
      "logps/rejected": -466.5190124511719,
      "loss": 1.1028,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5867493152618408,
      "rewards/margins": 0.16962355375289917,
      "rewards/rejected": -1.7563728094100952,
      "step": 10552
    },
    {
      "epoch": 1.3809274556309807,
      "grad_norm": 9.327028274536133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06022673100233078,
      "logits/rejected": -0.2594812214374542,
      "logps/chosen": -409.69598388671875,
      "logps/rejected": -378.83990478515625,
      "loss": 0.6153,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6185077428817749,
      "rewards/margins": 0.9714542627334595,
      "rewards/rejected": -1.5899620056152344,
      "step": 10553
    },
    {
      "epoch": 1.3810583135683323,
      "grad_norm": 4.110919952392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0525064542889595,
      "logits/rejected": -0.1386602818965912,
      "logps/chosen": -609.5512084960938,
      "logps/rejected": -423.0252380371094,
      "loss": 0.2204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3819507658481598,
      "rewards/margins": 2.0798163414001465,
      "rewards/rejected": -2.4617671966552734,
      "step": 10554
    },
    {
      "epoch": 1.381189171505684,
      "grad_norm": 6.281531810760498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0194687657058239,
      "logits/rejected": -0.18658041954040527,
      "logps/chosen": -675.7760009765625,
      "logps/rejected": -502.8859558105469,
      "loss": 0.258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1565239131450653,
      "rewards/margins": 2.227841854095459,
      "rewards/rejected": -2.3843657970428467,
      "step": 10555
    },
    {
      "epoch": 1.381320029443036,
      "grad_norm": 6.6286468505859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.036179400980472565,
      "logits/rejected": -0.17081588506698608,
      "logps/chosen": -453.579345703125,
      "logps/rejected": -407.7331237792969,
      "loss": 0.2777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2266302108764648,
      "rewards/margins": 1.4729291200637817,
      "rewards/rejected": -2.699559450149536,
      "step": 10556
    },
    {
      "epoch": 1.3814508873803877,
      "grad_norm": 5.726180076599121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24334293603897095,
      "logits/rejected": -0.26511338353157043,
      "logps/chosen": -607.4391479492188,
      "logps/rejected": -511.51544189453125,
      "loss": 0.182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6012749671936035,
      "rewards/margins": 2.8268392086029053,
      "rewards/rejected": -3.4281139373779297,
      "step": 10557
    },
    {
      "epoch": 1.3815817453177395,
      "grad_norm": 9.929512977600098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4876837432384491,
      "logits/rejected": -0.533363401889801,
      "logps/chosen": -354.12481689453125,
      "logps/rejected": -349.3971862792969,
      "loss": 0.4926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5451582074165344,
      "rewards/margins": 1.4872684478759766,
      "rewards/rejected": -2.032426595687866,
      "step": 10558
    },
    {
      "epoch": 1.3817126032550913,
      "grad_norm": 7.085391998291016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08296048641204834,
      "logits/rejected": -0.3709471523761749,
      "logps/chosen": -356.0892639160156,
      "logps/rejected": -467.3188781738281,
      "loss": 0.4109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1146762371063232,
      "rewards/margins": 1.7986996173858643,
      "rewards/rejected": -2.9133760929107666,
      "step": 10559
    },
    {
      "epoch": 1.3818434611924428,
      "grad_norm": 5.324085235595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25192031264305115,
      "logits/rejected": -0.29269665479660034,
      "logps/chosen": -428.97637939453125,
      "logps/rejected": -439.01263427734375,
      "loss": 0.2771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5444483160972595,
      "rewards/margins": 2.0096378326416016,
      "rewards/rejected": -2.554086208343506,
      "step": 10560
    },
    {
      "epoch": 1.3819743191297946,
      "grad_norm": 7.079156398773193,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019034596160054207,
      "logits/rejected": -0.1404724419116974,
      "logps/chosen": -463.2103576660156,
      "logps/rejected": -476.6785583496094,
      "loss": 0.3439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.300813913345337,
      "rewards/margins": 1.3398245573043823,
      "rewards/rejected": -2.640638589859009,
      "step": 10561
    },
    {
      "epoch": 1.3821051770671464,
      "grad_norm": 18.030405044555664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023081861436367035,
      "logits/rejected": -0.39502987265586853,
      "logps/chosen": -536.5037841796875,
      "logps/rejected": -429.1006164550781,
      "loss": 1.1774,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9575266242027283,
      "rewards/margins": -0.29555827379226685,
      "rewards/rejected": -0.6619682312011719,
      "step": 10562
    },
    {
      "epoch": 1.3822360350044982,
      "grad_norm": 7.3848748207092285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15893670916557312,
      "logits/rejected": 0.06872903555631638,
      "logps/chosen": -491.44659423828125,
      "logps/rejected": -433.27069091796875,
      "loss": 0.3557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7137381434440613,
      "rewards/margins": 1.6439502239227295,
      "rewards/rejected": -2.3576884269714355,
      "step": 10563
    },
    {
      "epoch": 1.38236689294185,
      "grad_norm": 8.239583969116211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40954822301864624,
      "logits/rejected": -0.4377279281616211,
      "logps/chosen": -374.5547790527344,
      "logps/rejected": -352.01446533203125,
      "loss": 0.4196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39277154207229614,
      "rewards/margins": 1.2774819135665894,
      "rewards/rejected": -1.6702535152435303,
      "step": 10564
    },
    {
      "epoch": 1.3824977508792018,
      "grad_norm": 10.259804725646973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33285951614379883,
      "logits/rejected": -0.4241454601287842,
      "logps/chosen": -546.459716796875,
      "logps/rejected": -423.8773498535156,
      "loss": 0.5893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9545975923538208,
      "rewards/margins": 0.9230272769927979,
      "rewards/rejected": -1.8776249885559082,
      "step": 10565
    },
    {
      "epoch": 1.3826286088165536,
      "grad_norm": 4.554393291473389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34619542956352234,
      "logits/rejected": -0.5376706123352051,
      "logps/chosen": -356.7314147949219,
      "logps/rejected": -386.5069885253906,
      "loss": 0.2094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8340633511543274,
      "rewards/margins": 2.3688101768493652,
      "rewards/rejected": -3.202873468399048,
      "step": 10566
    },
    {
      "epoch": 1.3827594667539054,
      "grad_norm": 7.891939163208008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19166894257068634,
      "logits/rejected": -0.33267414569854736,
      "logps/chosen": -332.15631103515625,
      "logps/rejected": -390.94964599609375,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.648186206817627,
      "rewards/margins": 1.7345962524414062,
      "rewards/rejected": -2.382782459259033,
      "step": 10567
    },
    {
      "epoch": 1.3828903246912572,
      "grad_norm": 13.01848316192627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21416455507278442,
      "logits/rejected": -0.2711538076400757,
      "logps/chosen": -351.57769775390625,
      "logps/rejected": -459.09521484375,
      "loss": 0.7752,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.90496826171875,
      "rewards/margins": 0.9169679284095764,
      "rewards/rejected": -1.8219363689422607,
      "step": 10568
    },
    {
      "epoch": 1.3830211826286087,
      "grad_norm": 9.033859252929688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025820929557085037,
      "logits/rejected": -0.06724405288696289,
      "logps/chosen": -420.9335021972656,
      "logps/rejected": -328.0181884765625,
      "loss": 0.4452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7094278931617737,
      "rewards/margins": 1.2080179452896118,
      "rewards/rejected": -1.9174458980560303,
      "step": 10569
    },
    {
      "epoch": 1.3831520405659605,
      "grad_norm": 10.330628395080566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19895802438259125,
      "logits/rejected": -0.03588461875915527,
      "logps/chosen": -388.9530334472656,
      "logps/rejected": -621.2080688476562,
      "loss": 0.4157,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5307130813598633,
      "rewards/margins": 1.0864183902740479,
      "rewards/rejected": -1.6171313524246216,
      "step": 10570
    },
    {
      "epoch": 1.3832828985033123,
      "grad_norm": 9.065839767456055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.038856688886880875,
      "logits/rejected": 0.01261911541223526,
      "logps/chosen": -502.30487060546875,
      "logps/rejected": -556.9336547851562,
      "loss": 0.4118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1322652101516724,
      "rewards/margins": 1.5566802024841309,
      "rewards/rejected": -2.6889455318450928,
      "step": 10571
    },
    {
      "epoch": 1.383413756440664,
      "grad_norm": 4.168014049530029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01992260292172432,
      "logits/rejected": 0.019952699542045593,
      "logps/chosen": -540.4873657226562,
      "logps/rejected": -423.9313049316406,
      "loss": 0.2384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42456018924713135,
      "rewards/margins": 2.005030870437622,
      "rewards/rejected": -1.5804706811904907,
      "step": 10572
    },
    {
      "epoch": 1.383544614378016,
      "grad_norm": 7.031432628631592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16889891028404236,
      "logits/rejected": -0.06745365262031555,
      "logps/chosen": -451.9806213378906,
      "logps/rejected": -432.861572265625,
      "loss": 0.4046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18071918189525604,
      "rewards/margins": 0.9667367339134216,
      "rewards/rejected": -1.1474558115005493,
      "step": 10573
    },
    {
      "epoch": 1.3836754723153677,
      "grad_norm": 18.130617141723633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2568431496620178,
      "logits/rejected": -0.1296844333410263,
      "logps/chosen": -312.8143005371094,
      "logps/rejected": -449.2485046386719,
      "loss": 1.1454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8563318848609924,
      "rewards/margins": 0.7783364057540894,
      "rewards/rejected": -1.6346681118011475,
      "step": 10574
    },
    {
      "epoch": 1.3838063302527193,
      "grad_norm": 12.808440208435059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1777980476617813,
      "logits/rejected": -0.2538108229637146,
      "logps/chosen": -598.55859375,
      "logps/rejected": -571.4854736328125,
      "loss": 0.6164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5022324323654175,
      "rewards/margins": 1.2907416820526123,
      "rewards/rejected": -2.7929739952087402,
      "step": 10575
    },
    {
      "epoch": 1.383937188190071,
      "grad_norm": 9.37105655670166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12708786129951477,
      "logits/rejected": -0.23906438052654266,
      "logps/chosen": -455.239990234375,
      "logps/rejected": -461.1258544921875,
      "loss": 0.5876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.160473346710205,
      "rewards/margins": 1.0140453577041626,
      "rewards/rejected": -2.1745188236236572,
      "step": 10576
    },
    {
      "epoch": 1.3840680461274228,
      "grad_norm": 11.465578079223633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007668346166610718,
      "logits/rejected": -0.01802212744951248,
      "logps/chosen": -453.6231994628906,
      "logps/rejected": -609.9600219726562,
      "loss": 0.7081,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10028115659952164,
      "rewards/margins": 0.9282360076904297,
      "rewards/rejected": -1.028517246246338,
      "step": 10577
    },
    {
      "epoch": 1.3841989040647746,
      "grad_norm": 16.03976058959961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05054199695587158,
      "logits/rejected": -0.03312443196773529,
      "logps/chosen": -408.3599853515625,
      "logps/rejected": -301.911865234375,
      "loss": 1.2397,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.6126782894134521,
      "rewards/margins": -0.5031805634498596,
      "rewards/rejected": -1.1094976663589478,
      "step": 10578
    },
    {
      "epoch": 1.3843297620021264,
      "grad_norm": 3.3700265884399414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05072581768035889,
      "logits/rejected": -0.15496224164962769,
      "logps/chosen": -651.7555541992188,
      "logps/rejected": -475.4496154785156,
      "loss": 0.1525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19916649162769318,
      "rewards/margins": 2.207430839538574,
      "rewards/rejected": -2.0082643032073975,
      "step": 10579
    },
    {
      "epoch": 1.3844606199394782,
      "grad_norm": 6.922564506530762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3437793254852295,
      "logits/rejected": -0.4369906485080719,
      "logps/chosen": -467.6197204589844,
      "logps/rejected": -348.1902770996094,
      "loss": 0.1973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6149102449417114,
      "rewards/margins": 2.1784558296203613,
      "rewards/rejected": -3.7933661937713623,
      "step": 10580
    },
    {
      "epoch": 1.38459147787683,
      "grad_norm": 10.610713005065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2029384970664978,
      "logits/rejected": -0.22109760344028473,
      "logps/chosen": -473.4203796386719,
      "logps/rejected": -429.9774475097656,
      "loss": 0.5321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8750274777412415,
      "rewards/margins": 1.4253044128417969,
      "rewards/rejected": -2.3003318309783936,
      "step": 10581
    },
    {
      "epoch": 1.3847223358141818,
      "grad_norm": 12.511971473693848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07032062113285065,
      "logits/rejected": -0.08972106128931046,
      "logps/chosen": -658.9627075195312,
      "logps/rejected": -442.13116455078125,
      "loss": 0.4934,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5967869162559509,
      "rewards/margins": 1.8669319152832031,
      "rewards/rejected": -2.4637186527252197,
      "step": 10582
    },
    {
      "epoch": 1.3848531937515336,
      "grad_norm": 8.607696533203125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2543072998523712,
      "logits/rejected": -0.10966956615447998,
      "logps/chosen": -328.5901184082031,
      "logps/rejected": -488.8582763671875,
      "loss": 0.5773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.513576865196228,
      "rewards/margins": 0.7536180019378662,
      "rewards/rejected": -2.267194986343384,
      "step": 10583
    },
    {
      "epoch": 1.3849840516888854,
      "grad_norm": 9.671658515930176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014241032302379608,
      "logits/rejected": -0.30498459935188293,
      "logps/chosen": -319.9980163574219,
      "logps/rejected": -281.4956359863281,
      "loss": 0.5894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5155192613601685,
      "rewards/margins": 0.6035428047180176,
      "rewards/rejected": -2.1190621852874756,
      "step": 10584
    },
    {
      "epoch": 1.385114909626237,
      "grad_norm": 8.770623207092285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07500273734331131,
      "logits/rejected": -0.11457459628582001,
      "logps/chosen": -495.33233642578125,
      "logps/rejected": -514.8720703125,
      "loss": 0.4373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1809380054473877,
      "rewards/margins": 1.4483189582824707,
      "rewards/rejected": -2.6292569637298584,
      "step": 10585
    },
    {
      "epoch": 1.3852457675635887,
      "grad_norm": 8.80871868133545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23262903094291687,
      "logits/rejected": -0.18972738087177277,
      "logps/chosen": -596.8026733398438,
      "logps/rejected": -644.0905151367188,
      "loss": 0.4994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.602887749671936,
      "rewards/margins": 0.7880493402481079,
      "rewards/rejected": -1.390937089920044,
      "step": 10586
    },
    {
      "epoch": 1.3853766255009405,
      "grad_norm": 8.079455375671387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11508326232433319,
      "logits/rejected": -0.10268246382474899,
      "logps/chosen": -456.3069152832031,
      "logps/rejected": -505.9688720703125,
      "loss": 0.5953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.549446165561676,
      "rewards/margins": 0.952018141746521,
      "rewards/rejected": -1.5014642477035522,
      "step": 10587
    },
    {
      "epoch": 1.3855074834382923,
      "grad_norm": 18.46080207824707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2781885862350464,
      "logits/rejected": -0.4576220214366913,
      "logps/chosen": -425.962158203125,
      "logps/rejected": -524.7456665039062,
      "loss": 0.4464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7615069150924683,
      "rewards/margins": 1.3073786497116089,
      "rewards/rejected": -3.068885564804077,
      "step": 10588
    },
    {
      "epoch": 1.3856383413756441,
      "grad_norm": 4.949564456939697,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09109970927238464,
      "logits/rejected": -0.236953467130661,
      "logps/chosen": -505.32244873046875,
      "logps/rejected": -533.632568359375,
      "loss": 0.2071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.686522364616394,
      "rewards/margins": 2.154980182647705,
      "rewards/rejected": -2.8415024280548096,
      "step": 10589
    },
    {
      "epoch": 1.3857691993129957,
      "grad_norm": 10.020464897155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23740530014038086,
      "logits/rejected": -0.3114639222621918,
      "logps/chosen": -668.9129638671875,
      "logps/rejected": -501.5006408691406,
      "loss": 0.4673,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5557075142860413,
      "rewards/margins": 1.1108579635620117,
      "rewards/rejected": -1.6665655374526978,
      "step": 10590
    },
    {
      "epoch": 1.3859000572503475,
      "grad_norm": 8.71718978881836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31146740913391113,
      "logits/rejected": -0.2802027463912964,
      "logps/chosen": -566.0592651367188,
      "logps/rejected": -484.6888427734375,
      "loss": 0.3598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22175198793411255,
      "rewards/margins": 2.3732924461364746,
      "rewards/rejected": -2.5950441360473633,
      "step": 10591
    },
    {
      "epoch": 1.3860309151876993,
      "grad_norm": 6.120119094848633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36815375089645386,
      "logits/rejected": -0.5169989466667175,
      "logps/chosen": -464.0895080566406,
      "logps/rejected": -388.9527587890625,
      "loss": 0.3066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5655683875083923,
      "rewards/margins": 1.5408228635787964,
      "rewards/rejected": -2.106391429901123,
      "step": 10592
    },
    {
      "epoch": 1.386161773125051,
      "grad_norm": 7.60831356048584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07826577872037888,
      "logits/rejected": -0.229537233710289,
      "logps/chosen": -460.4226989746094,
      "logps/rejected": -666.5980224609375,
      "loss": 0.3548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6537647247314453,
      "rewards/margins": 2.264925718307495,
      "rewards/rejected": -2.9186904430389404,
      "step": 10593
    },
    {
      "epoch": 1.3862926310624029,
      "grad_norm": 8.454544067382812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2520839273929596,
      "logits/rejected": -0.2954300045967102,
      "logps/chosen": -668.462158203125,
      "logps/rejected": -615.7125854492188,
      "loss": 0.4574,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0041804313659668,
      "rewards/margins": 1.0770220756530762,
      "rewards/rejected": -2.081202507019043,
      "step": 10594
    },
    {
      "epoch": 1.3864234889997546,
      "grad_norm": 9.269085884094238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2537388503551483,
      "logits/rejected": -0.25693008303642273,
      "logps/chosen": -403.2827453613281,
      "logps/rejected": -486.2729187011719,
      "loss": 0.5431,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4312210083007812,
      "rewards/margins": 1.16890549659729,
      "rewards/rejected": -2.600126266479492,
      "step": 10595
    },
    {
      "epoch": 1.3865543469371064,
      "grad_norm": 11.02397632598877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16877484321594238,
      "logits/rejected": -0.18358294665813446,
      "logps/chosen": -683.027099609375,
      "logps/rejected": -455.25341796875,
      "loss": 0.6162,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.849970817565918,
      "rewards/margins": 0.44405311346054077,
      "rewards/rejected": -1.2940239906311035,
      "step": 10596
    },
    {
      "epoch": 1.3866852048744582,
      "grad_norm": 8.930917739868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47432607412338257,
      "logits/rejected": -0.4428030848503113,
      "logps/chosen": -372.60791015625,
      "logps/rejected": -431.8968200683594,
      "loss": 0.4058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8130138516426086,
      "rewards/margins": 1.3809419870376587,
      "rewards/rejected": -2.193955659866333,
      "step": 10597
    },
    {
      "epoch": 1.38681606281181,
      "grad_norm": 14.53151798248291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2337953746318817,
      "logits/rejected": -0.2625223994255066,
      "logps/chosen": -504.3484191894531,
      "logps/rejected": -589.3452758789062,
      "loss": 0.8282,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6322555541992188,
      "rewards/margins": 0.667159378528595,
      "rewards/rejected": -1.299414873123169,
      "step": 10598
    },
    {
      "epoch": 1.3869469207491618,
      "grad_norm": 5.534521102905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3929947018623352,
      "logits/rejected": -0.23462031781673431,
      "logps/chosen": -460.0899658203125,
      "logps/rejected": -403.5872802734375,
      "loss": 0.2992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8020246624946594,
      "rewards/margins": 1.458335041999817,
      "rewards/rejected": -2.260359525680542,
      "step": 10599
    },
    {
      "epoch": 1.3870777786865134,
      "grad_norm": 4.340306282043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.222795769572258,
      "logits/rejected": -0.15652720630168915,
      "logps/chosen": -362.3065185546875,
      "logps/rejected": -325.394775390625,
      "loss": 0.1633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18620872497558594,
      "rewards/margins": 2.3387603759765625,
      "rewards/rejected": -2.5249691009521484,
      "step": 10600
    },
    {
      "epoch": 1.3872086366238652,
      "grad_norm": 8.338278770446777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10989111661911011,
      "logits/rejected": -0.40556296706199646,
      "logps/chosen": -492.5878601074219,
      "logps/rejected": -463.17718505859375,
      "loss": 0.3455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.994986891746521,
      "rewards/margins": 1.765805959701538,
      "rewards/rejected": -2.7607929706573486,
      "step": 10601
    },
    {
      "epoch": 1.387339494561217,
      "grad_norm": 28.222919464111328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04156515747308731,
      "logits/rejected": -0.15264329314231873,
      "logps/chosen": -544.0049438476562,
      "logps/rejected": -571.0072021484375,
      "loss": 0.501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3681572675704956,
      "rewards/margins": 0.8983551263809204,
      "rewards/rejected": -2.266512393951416,
      "step": 10602
    },
    {
      "epoch": 1.3874703524985688,
      "grad_norm": 5.203302383422852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4077285826206207,
      "logits/rejected": -0.6034569144248962,
      "logps/chosen": -471.9051818847656,
      "logps/rejected": -337.07861328125,
      "loss": 0.4106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9030172824859619,
      "rewards/margins": 1.5473346710205078,
      "rewards/rejected": -2.4503517150878906,
      "step": 10603
    },
    {
      "epoch": 1.3876012104359206,
      "grad_norm": 5.859398365020752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33305060863494873,
      "logits/rejected": -0.6339362859725952,
      "logps/chosen": -347.5595397949219,
      "logps/rejected": -448.8782958984375,
      "loss": 0.2629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3385745286941528,
      "rewards/margins": 2.737421751022339,
      "rewards/rejected": -4.075996398925781,
      "step": 10604
    },
    {
      "epoch": 1.3877320683732723,
      "grad_norm": 7.59798002243042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0773230791091919,
      "logits/rejected": -0.09408805519342422,
      "logps/chosen": -497.7508544921875,
      "logps/rejected": -499.0338439941406,
      "loss": 0.3719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18681713938713074,
      "rewards/margins": 2.0856404304504395,
      "rewards/rejected": -2.2724573612213135,
      "step": 10605
    },
    {
      "epoch": 1.387862926310624,
      "grad_norm": 5.1748366355896,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40948235988616943,
      "logits/rejected": -0.563177764415741,
      "logps/chosen": -276.1452941894531,
      "logps/rejected": -359.8829345703125,
      "loss": 0.251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6823168992996216,
      "rewards/margins": 2.1120996475219727,
      "rewards/rejected": -2.7944164276123047,
      "step": 10606
    },
    {
      "epoch": 1.3879937842479757,
      "grad_norm": 5.928308010101318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023485951125621796,
      "logits/rejected": 0.06298898905515671,
      "logps/chosen": -534.4356689453125,
      "logps/rejected": -467.9797668457031,
      "loss": 0.2649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.705826461315155,
      "rewards/margins": 1.5339025259017944,
      "rewards/rejected": -2.2397286891937256,
      "step": 10607
    },
    {
      "epoch": 1.3881246421853275,
      "grad_norm": 11.519742965698242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2161545753479004,
      "logits/rejected": -0.5078280568122864,
      "logps/chosen": -535.7589111328125,
      "logps/rejected": -433.863037109375,
      "loss": 0.5995,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38272416591644287,
      "rewards/margins": 0.4526654779911041,
      "rewards/rejected": -0.8353897333145142,
      "step": 10608
    },
    {
      "epoch": 1.3882555001226793,
      "grad_norm": 10.852380752563477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07710810005664825,
      "logits/rejected": 0.1951608657836914,
      "logps/chosen": -580.6995849609375,
      "logps/rejected": -595.1422119140625,
      "loss": 0.5843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8147304654121399,
      "rewards/margins": 0.535228431224823,
      "rewards/rejected": -1.349958896636963,
      "step": 10609
    },
    {
      "epoch": 1.388386358060031,
      "grad_norm": 8.884583473205566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24093160033226013,
      "logits/rejected": -0.3997044563293457,
      "logps/chosen": -498.99993896484375,
      "logps/rejected": -660.2493286132812,
      "loss": 0.4139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4646821618080139,
      "rewards/margins": 1.82903254032135,
      "rewards/rejected": -2.293714761734009,
      "step": 10610
    },
    {
      "epoch": 1.3885172159973829,
      "grad_norm": 11.280658721923828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18818454444408417,
      "logits/rejected": -0.26369282603263855,
      "logps/chosen": -428.7125549316406,
      "logps/rejected": -353.3045959472656,
      "loss": 0.8115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.001439094543457,
      "rewards/margins": 0.6184509992599487,
      "rewards/rejected": -1.6198902130126953,
      "step": 10611
    },
    {
      "epoch": 1.3886480739347347,
      "grad_norm": 7.133736610412598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40749111771583557,
      "logits/rejected": -0.5623930096626282,
      "logps/chosen": -442.10833740234375,
      "logps/rejected": -568.7181396484375,
      "loss": 0.4159,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3305187225341797,
      "rewards/margins": 1.3349522352218628,
      "rewards/rejected": -2.665471076965332,
      "step": 10612
    },
    {
      "epoch": 1.3887789318720865,
      "grad_norm": 7.593545436859131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20402240753173828,
      "logits/rejected": 0.03701914846897125,
      "logps/chosen": -351.4164733886719,
      "logps/rejected": -464.4879150390625,
      "loss": 0.4325,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3477606177330017,
      "rewards/margins": 1.038602352142334,
      "rewards/rejected": -1.386362910270691,
      "step": 10613
    },
    {
      "epoch": 1.3889097898094382,
      "grad_norm": 12.242450714111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18746739625930786,
      "logits/rejected": -0.1289636641740799,
      "logps/chosen": -379.35882568359375,
      "logps/rejected": -390.5006103515625,
      "loss": 0.9423,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.845632553100586,
      "rewards/margins": 0.036564670503139496,
      "rewards/rejected": -1.8821972608566284,
      "step": 10614
    },
    {
      "epoch": 1.3890406477467898,
      "grad_norm": 9.465070724487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27894124388694763,
      "logits/rejected": -0.27387452125549316,
      "logps/chosen": -460.8950500488281,
      "logps/rejected": -504.228759765625,
      "loss": 0.4292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5077316761016846,
      "rewards/margins": 1.3284598588943481,
      "rewards/rejected": -1.8361914157867432,
      "step": 10615
    },
    {
      "epoch": 1.3891715056841416,
      "grad_norm": 9.818166732788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27585452795028687,
      "logits/rejected": -0.11497445404529572,
      "logps/chosen": -379.1495056152344,
      "logps/rejected": -386.7857666015625,
      "loss": 0.6284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0459699630737305,
      "rewards/margins": 1.0471644401550293,
      "rewards/rejected": -2.093134641647339,
      "step": 10616
    },
    {
      "epoch": 1.3893023636214934,
      "grad_norm": 6.064009189605713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3078443109989166,
      "logits/rejected": -0.31310904026031494,
      "logps/chosen": -399.3122863769531,
      "logps/rejected": -284.8441162109375,
      "loss": 0.3341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.636810302734375,
      "rewards/margins": 1.634735107421875,
      "rewards/rejected": -2.271545171737671,
      "step": 10617
    },
    {
      "epoch": 1.3894332215588452,
      "grad_norm": 5.539676666259766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4086950123310089,
      "logits/rejected": -0.38990962505340576,
      "logps/chosen": -417.82086181640625,
      "logps/rejected": -446.49078369140625,
      "loss": 0.297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37878841161727905,
      "rewards/margins": 1.3904403448104858,
      "rewards/rejected": -1.7692288160324097,
      "step": 10618
    },
    {
      "epoch": 1.389564079496197,
      "grad_norm": 5.8087334632873535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3039611279964447,
      "logits/rejected": -0.49320608377456665,
      "logps/chosen": -840.808837890625,
      "logps/rejected": -685.8533935546875,
      "loss": 0.2964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3440172672271729,
      "rewards/margins": 1.4386382102966309,
      "rewards/rejected": -2.7826554775238037,
      "step": 10619
    },
    {
      "epoch": 1.3896949374335488,
      "grad_norm": 4.128570556640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06052282452583313,
      "logits/rejected": -0.17968535423278809,
      "logps/chosen": -513.8274536132812,
      "logps/rejected": -487.1369934082031,
      "loss": 0.3197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12583959102630615,
      "rewards/margins": 2.3897957801818848,
      "rewards/rejected": -2.5156354904174805,
      "step": 10620
    },
    {
      "epoch": 1.3898257953709003,
      "grad_norm": 5.799055576324463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08063548058271408,
      "logits/rejected": -0.3774881660938263,
      "logps/chosen": -461.4968566894531,
      "logps/rejected": -424.2582092285156,
      "loss": 0.3301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31004735827445984,
      "rewards/margins": 1.1696629524230957,
      "rewards/rejected": -0.8596157431602478,
      "step": 10621
    },
    {
      "epoch": 1.3899566533082521,
      "grad_norm": 11.537071228027344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4202929139137268,
      "logits/rejected": -0.5208210349082947,
      "logps/chosen": -319.791015625,
      "logps/rejected": -331.86004638671875,
      "loss": 0.6183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.884831190109253,
      "rewards/margins": 0.7801021337509155,
      "rewards/rejected": -2.664933443069458,
      "step": 10622
    },
    {
      "epoch": 1.390087511245604,
      "grad_norm": 9.38729476928711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3083076775074005,
      "logits/rejected": -0.25338679552078247,
      "logps/chosen": -440.3139343261719,
      "logps/rejected": -487.5212707519531,
      "loss": 0.6286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3124760389328003,
      "rewards/margins": 0.4181421101093292,
      "rewards/rejected": -1.7306180000305176,
      "step": 10623
    },
    {
      "epoch": 1.3902183691829557,
      "grad_norm": 6.829867839813232,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12403439730405807,
      "logits/rejected": -0.05691046640276909,
      "logps/chosen": -510.3670959472656,
      "logps/rejected": -422.07403564453125,
      "loss": 0.3543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3331329822540283,
      "rewards/margins": 1.2746318578720093,
      "rewards/rejected": -1.607764720916748,
      "step": 10624
    },
    {
      "epoch": 1.3903492271203075,
      "grad_norm": 9.064040184020996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0733591616153717,
      "logits/rejected": -0.14079082012176514,
      "logps/chosen": -571.5374145507812,
      "logps/rejected": -475.7071533203125,
      "loss": 0.9216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3839004039764404,
      "rewards/margins": 0.3976448178291321,
      "rewards/rejected": -1.7815451622009277,
      "step": 10625
    },
    {
      "epoch": 1.3904800850576593,
      "grad_norm": 5.944113254547119,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3449094593524933,
      "logits/rejected": -0.3530142605304718,
      "logps/chosen": -379.65478515625,
      "logps/rejected": -424.509033203125,
      "loss": 0.3888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13956239819526672,
      "rewards/margins": 1.579775094985962,
      "rewards/rejected": -1.440212607383728,
      "step": 10626
    },
    {
      "epoch": 1.390610942995011,
      "grad_norm": 6.041090965270996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07403768599033356,
      "logits/rejected": -0.18051373958587646,
      "logps/chosen": -494.7571716308594,
      "logps/rejected": -525.2509765625,
      "loss": 0.355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5288587808609009,
      "rewards/margins": 1.618536114692688,
      "rewards/rejected": -2.147395133972168,
      "step": 10627
    },
    {
      "epoch": 1.3907418009323629,
      "grad_norm": 3.196228265762329,
      "learning_rate": 5e-06,
      "logits/chosen": -0.463847815990448,
      "logits/rejected": -0.622744083404541,
      "logps/chosen": -409.1406555175781,
      "logps/rejected": -348.0803527832031,
      "loss": 0.2857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28072768449783325,
      "rewards/margins": 1.9463485479354858,
      "rewards/rejected": -2.2270760536193848,
      "step": 10628
    },
    {
      "epoch": 1.3908726588697147,
      "grad_norm": 5.471822738647461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021599851548671722,
      "logits/rejected": -0.0796908438205719,
      "logps/chosen": -482.1878662109375,
      "logps/rejected": -519.197021484375,
      "loss": 0.2346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6434465646743774,
      "rewards/margins": 2.0748376846313477,
      "rewards/rejected": -2.7182841300964355,
      "step": 10629
    },
    {
      "epoch": 1.3910035168070665,
      "grad_norm": 9.984699249267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5101084113121033,
      "logits/rejected": -0.4143112301826477,
      "logps/chosen": -428.34228515625,
      "logps/rejected": -394.2928771972656,
      "loss": 0.5919,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5450997948646545,
      "rewards/margins": 1.306671380996704,
      "rewards/rejected": -1.8517711162567139,
      "step": 10630
    },
    {
      "epoch": 1.391134374744418,
      "grad_norm": 8.199105262756348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2222174108028412,
      "logits/rejected": -0.35633888840675354,
      "logps/chosen": -331.3909912109375,
      "logps/rejected": -269.23486328125,
      "loss": 0.4208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.680177628993988,
      "rewards/margins": 1.500512719154358,
      "rewards/rejected": -2.1806907653808594,
      "step": 10631
    },
    {
      "epoch": 1.3912652326817698,
      "grad_norm": 7.478882789611816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16410931944847107,
      "logits/rejected": -0.4045986533164978,
      "logps/chosen": -455.31072998046875,
      "logps/rejected": -451.05902099609375,
      "loss": 0.3785,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0118002891540527,
      "rewards/margins": 1.247107982635498,
      "rewards/rejected": -2.2589080333709717,
      "step": 10632
    },
    {
      "epoch": 1.3913960906191216,
      "grad_norm": 9.219712257385254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1939798891544342,
      "logits/rejected": -0.10957755893468857,
      "logps/chosen": -457.585693359375,
      "logps/rejected": -430.9093017578125,
      "loss": 0.5916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7859783172607422,
      "rewards/margins": 0.5429046750068665,
      "rewards/rejected": -2.328882932662964,
      "step": 10633
    },
    {
      "epoch": 1.3915269485564734,
      "grad_norm": 14.619237899780273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21938583254814148,
      "logits/rejected": -0.12866777181625366,
      "logps/chosen": -650.474609375,
      "logps/rejected": -451.84722900390625,
      "loss": 0.623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.297883152961731,
      "rewards/margins": 1.2815630435943604,
      "rewards/rejected": -2.5794460773468018,
      "step": 10634
    },
    {
      "epoch": 1.3916578064938252,
      "grad_norm": 9.22262191772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4574645161628723,
      "logits/rejected": -0.5040500164031982,
      "logps/chosen": -410.4895935058594,
      "logps/rejected": -370.13995361328125,
      "loss": 0.4381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7209522128105164,
      "rewards/margins": 1.032918930053711,
      "rewards/rejected": -1.753871202468872,
      "step": 10635
    },
    {
      "epoch": 1.3917886644311768,
      "grad_norm": 7.690557479858398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03990750014781952,
      "logits/rejected": -0.2824234962463379,
      "logps/chosen": -384.9527282714844,
      "logps/rejected": -484.22955322265625,
      "loss": 0.3649,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24577046930789948,
      "rewards/margins": 1.764206886291504,
      "rewards/rejected": -1.5184364318847656,
      "step": 10636
    },
    {
      "epoch": 1.3919195223685286,
      "grad_norm": 10.982610702514648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3565220236778259,
      "logits/rejected": 0.006146758794784546,
      "logps/chosen": -399.9207458496094,
      "logps/rejected": -479.58367919921875,
      "loss": 0.6776,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7242263555526733,
      "rewards/margins": 0.5961025357246399,
      "rewards/rejected": -2.320328950881958,
      "step": 10637
    },
    {
      "epoch": 1.3920503803058804,
      "grad_norm": 5.8531084060668945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16659916937351227,
      "logits/rejected": -0.016436433419585228,
      "logps/chosen": -558.8919067382812,
      "logps/rejected": -432.9097900390625,
      "loss": 0.2675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11605794727802277,
      "rewards/margins": 2.167515993118286,
      "rewards/rejected": -2.0514581203460693,
      "step": 10638
    },
    {
      "epoch": 1.3921812382432321,
      "grad_norm": 9.745348930358887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04934505373239517,
      "logits/rejected": -0.14931486546993256,
      "logps/chosen": -527.7109985351562,
      "logps/rejected": -444.32733154296875,
      "loss": 0.6326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47551044821739197,
      "rewards/margins": 0.5963965654373169,
      "rewards/rejected": -1.0719070434570312,
      "step": 10639
    },
    {
      "epoch": 1.392312096180584,
      "grad_norm": 8.742767333984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37330907583236694,
      "logits/rejected": -0.4515850245952606,
      "logps/chosen": -467.408935546875,
      "logps/rejected": -374.4068603515625,
      "loss": 0.6341,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7461817264556885,
      "rewards/margins": 0.7220459580421448,
      "rewards/rejected": -2.4682278633117676,
      "step": 10640
    },
    {
      "epoch": 1.3924429541179357,
      "grad_norm": 17.32333755493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24519649147987366,
      "logits/rejected": -0.18548305332660675,
      "logps/chosen": -490.3021240234375,
      "logps/rejected": -503.7921447753906,
      "loss": 0.9332,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1676613092422485,
      "rewards/margins": 0.18661773204803467,
      "rewards/rejected": -1.3542789220809937,
      "step": 10641
    },
    {
      "epoch": 1.3925738120552875,
      "grad_norm": 5.906803131103516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01612149178981781,
      "logits/rejected": -0.07348989695310593,
      "logps/chosen": -419.106201171875,
      "logps/rejected": -480.200927734375,
      "loss": 0.3091,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1730348765850067,
      "rewards/margins": 1.4606456756591797,
      "rewards/rejected": -1.2876107692718506,
      "step": 10642
    },
    {
      "epoch": 1.3927046699926393,
      "grad_norm": 5.999605655670166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36924561858177185,
      "logits/rejected": -0.38376814126968384,
      "logps/chosen": -707.9103393554688,
      "logps/rejected": -653.487548828125,
      "loss": 0.2098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3802626132965088,
      "rewards/margins": 2.3194174766540527,
      "rewards/rejected": -2.6996800899505615,
      "step": 10643
    },
    {
      "epoch": 1.392835527929991,
      "grad_norm": 13.351284980773926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38470354676246643,
      "logits/rejected": -0.43228060007095337,
      "logps/chosen": -503.0357360839844,
      "logps/rejected": -382.62493896484375,
      "loss": 0.6469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2640169858932495,
      "rewards/margins": 1.1493549346923828,
      "rewards/rejected": -2.4133718013763428,
      "step": 10644
    },
    {
      "epoch": 1.392966385867343,
      "grad_norm": 8.402316093444824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2866916358470917,
      "logits/rejected": -0.30133509635925293,
      "logps/chosen": -537.137451171875,
      "logps/rejected": -475.230712890625,
      "loss": 0.4594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3347875475883484,
      "rewards/margins": 1.0088512897491455,
      "rewards/rejected": -1.3436388969421387,
      "step": 10645
    },
    {
      "epoch": 1.3930972438046945,
      "grad_norm": 11.739680290222168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22375792264938354,
      "logits/rejected": -0.15703865885734558,
      "logps/chosen": -550.7689208984375,
      "logps/rejected": -465.3318786621094,
      "loss": 0.5215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1512959003448486,
      "rewards/margins": 0.9855523109436035,
      "rewards/rejected": -2.136848211288452,
      "step": 10646
    },
    {
      "epoch": 1.3932281017420463,
      "grad_norm": 10.899064064025879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29850634932518005,
      "logits/rejected": 0.025588855147361755,
      "logps/chosen": -510.3404846191406,
      "logps/rejected": -537.28076171875,
      "loss": 0.5221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6387054324150085,
      "rewards/margins": 1.1903314590454102,
      "rewards/rejected": -1.829037070274353,
      "step": 10647
    },
    {
      "epoch": 1.393358959679398,
      "grad_norm": 8.683537483215332,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10296677052974701,
      "logits/rejected": -0.07841310650110245,
      "logps/chosen": -372.2492980957031,
      "logps/rejected": -417.70172119140625,
      "loss": 0.4821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1491396129131317,
      "rewards/margins": 1.0049775838851929,
      "rewards/rejected": -1.1541171073913574,
      "step": 10648
    },
    {
      "epoch": 1.3934898176167498,
      "grad_norm": 13.64236068725586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10668623447418213,
      "logits/rejected": -0.22998620569705963,
      "logps/chosen": -464.0962829589844,
      "logps/rejected": -347.18701171875,
      "loss": 0.7437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0833584070205688,
      "rewards/margins": 0.6500102877616882,
      "rewards/rejected": -1.7333687543869019,
      "step": 10649
    },
    {
      "epoch": 1.3936206755541016,
      "grad_norm": 4.170919418334961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05895061790943146,
      "logits/rejected": -0.416670024394989,
      "logps/chosen": -452.8970642089844,
      "logps/rejected": -373.5240173339844,
      "loss": 0.1943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7252216935157776,
      "rewards/margins": 2.1788179874420166,
      "rewards/rejected": -2.9040396213531494,
      "step": 10650
    },
    {
      "epoch": 1.3937515334914534,
      "grad_norm": 11.634700775146484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0665685310959816,
      "logits/rejected": -0.164757639169693,
      "logps/chosen": -754.2222290039062,
      "logps/rejected": -443.1181640625,
      "loss": 0.5783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48690909147262573,
      "rewards/margins": 1.0314775705337524,
      "rewards/rejected": -1.5183866024017334,
      "step": 10651
    },
    {
      "epoch": 1.393882391428805,
      "grad_norm": 7.197787284851074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027497664093971252,
      "logits/rejected": -0.06114814430475235,
      "logps/chosen": -565.2510986328125,
      "logps/rejected": -331.3302917480469,
      "loss": 0.3632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.879368782043457,
      "rewards/margins": 1.294663429260254,
      "rewards/rejected": -2.174032211303711,
      "step": 10652
    },
    {
      "epoch": 1.3940132493661568,
      "grad_norm": 6.4839396476745605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35957059264183044,
      "logits/rejected": -0.011339247226715088,
      "logps/chosen": -533.4832763671875,
      "logps/rejected": -455.8399658203125,
      "loss": 0.3949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45679688453674316,
      "rewards/margins": 0.9876102209091187,
      "rewards/rejected": -1.4444069862365723,
      "step": 10653
    },
    {
      "epoch": 1.3941441073035086,
      "grad_norm": 4.991131782531738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3546222746372223,
      "logits/rejected": -0.20015431940555573,
      "logps/chosen": -667.8265380859375,
      "logps/rejected": -648.202392578125,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5313495397567749,
      "rewards/margins": 1.9995120763778687,
      "rewards/rejected": -2.5308618545532227,
      "step": 10654
    },
    {
      "epoch": 1.3942749652408604,
      "grad_norm": 10.067537307739258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3625198006629944,
      "logits/rejected": -0.42759987711906433,
      "logps/chosen": -298.6180725097656,
      "logps/rejected": -357.3888854980469,
      "loss": 0.5377,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.663371205329895,
      "rewards/margins": 1.1417189836502075,
      "rewards/rejected": -2.8050901889801025,
      "step": 10655
    },
    {
      "epoch": 1.3944058231782122,
      "grad_norm": 7.452911376953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08070412278175354,
      "logits/rejected": -0.2880205512046814,
      "logps/chosen": -407.9292297363281,
      "logps/rejected": -379.0996398925781,
      "loss": 0.5412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20723378658294678,
      "rewards/margins": 0.8702572584152222,
      "rewards/rejected": -1.077491044998169,
      "step": 10656
    },
    {
      "epoch": 1.394536681115564,
      "grad_norm": 6.478466987609863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3210560381412506,
      "logits/rejected": -0.3687666058540344,
      "logps/chosen": -430.9834899902344,
      "logps/rejected": -463.1590270996094,
      "loss": 0.3271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35328465700149536,
      "rewards/margins": 1.3233011960983276,
      "rewards/rejected": -1.6765857934951782,
      "step": 10657
    },
    {
      "epoch": 1.3946675390529157,
      "grad_norm": 9.441496849060059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1078534871339798,
      "logits/rejected": -0.18788045644760132,
      "logps/chosen": -383.51470947265625,
      "logps/rejected": -449.989501953125,
      "loss": 0.4918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37684518098831177,
      "rewards/margins": 0.7919105291366577,
      "rewards/rejected": -1.1687557697296143,
      "step": 10658
    },
    {
      "epoch": 1.3947983969902675,
      "grad_norm": 9.122851371765137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014569098129868507,
      "logits/rejected": -0.16501188278198242,
      "logps/chosen": -460.12359619140625,
      "logps/rejected": -608.8471069335938,
      "loss": 0.4272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36572325229644775,
      "rewards/margins": 1.2796655893325806,
      "rewards/rejected": -1.6453888416290283,
      "step": 10659
    },
    {
      "epoch": 1.3949292549276193,
      "grad_norm": 10.325532913208008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32701316475868225,
      "logits/rejected": -0.16637267172336578,
      "logps/chosen": -420.58770751953125,
      "logps/rejected": -365.0294189453125,
      "loss": 0.5957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2815607786178589,
      "rewards/margins": 0.5382595062255859,
      "rewards/rejected": -1.8198202848434448,
      "step": 10660
    },
    {
      "epoch": 1.395060112864971,
      "grad_norm": 5.695621490478516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2217433899641037,
      "logits/rejected": -0.1997746229171753,
      "logps/chosen": -395.64678955078125,
      "logps/rejected": -398.7033386230469,
      "loss": 0.239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17240257561206818,
      "rewards/margins": 1.9395461082458496,
      "rewards/rejected": -2.1119487285614014,
      "step": 10661
    },
    {
      "epoch": 1.3951909708023227,
      "grad_norm": 6.581567287445068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10621316730976105,
      "logits/rejected": -0.16402631998062134,
      "logps/chosen": -317.04644775390625,
      "logps/rejected": -356.3204345703125,
      "loss": 0.4114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0522476434707642,
      "rewards/margins": 1.1079332828521729,
      "rewards/rejected": -2.1601810455322266,
      "step": 10662
    },
    {
      "epoch": 1.3953218287396745,
      "grad_norm": 5.410355567932129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028852984309196472,
      "logits/rejected": -0.24195456504821777,
      "logps/chosen": -365.52716064453125,
      "logps/rejected": -522.095458984375,
      "loss": 0.1845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2020739316940308,
      "rewards/margins": 2.476703405380249,
      "rewards/rejected": -3.6787772178649902,
      "step": 10663
    },
    {
      "epoch": 1.3954526866770263,
      "grad_norm": 11.627910614013672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25894638895988464,
      "logits/rejected": -0.35592320561408997,
      "logps/chosen": -731.3348388671875,
      "logps/rejected": -489.89739990234375,
      "loss": 0.5854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0211886167526245,
      "rewards/margins": 0.903041660785675,
      "rewards/rejected": -1.9242300987243652,
      "step": 10664
    },
    {
      "epoch": 1.395583544614378,
      "grad_norm": 11.671056747436523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.534679651260376,
      "logits/rejected": -0.3868541717529297,
      "logps/chosen": -327.46954345703125,
      "logps/rejected": -265.7371520996094,
      "loss": 0.7121,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2579230070114136,
      "rewards/margins": 0.05082201957702637,
      "rewards/rejected": -1.30874502658844,
      "step": 10665
    },
    {
      "epoch": 1.3957144025517298,
      "grad_norm": 5.01500129699707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22620707750320435,
      "logits/rejected": 0.022397177293896675,
      "logps/chosen": -520.0424194335938,
      "logps/rejected": -447.1145324707031,
      "loss": 0.2964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30292797088623047,
      "rewards/margins": 2.0468525886535645,
      "rewards/rejected": -2.349780321121216,
      "step": 10666
    },
    {
      "epoch": 1.3958452604890814,
      "grad_norm": 11.34829044342041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5320129990577698,
      "logits/rejected": -0.3291861116886139,
      "logps/chosen": -421.5972900390625,
      "logps/rejected": -520.3803100585938,
      "loss": 0.5382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.986782431602478,
      "rewards/margins": 1.4289419651031494,
      "rewards/rejected": -2.415724515914917,
      "step": 10667
    },
    {
      "epoch": 1.3959761184264332,
      "grad_norm": 7.66395902633667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02344752661883831,
      "logits/rejected": 0.05685564875602722,
      "logps/chosen": -366.5838623046875,
      "logps/rejected": -359.3937072753906,
      "loss": 0.4354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4909965991973877,
      "rewards/margins": 1.691523551940918,
      "rewards/rejected": -3.1825201511383057,
      "step": 10668
    },
    {
      "epoch": 1.396106976363785,
      "grad_norm": 7.993969917297363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34602245688438416,
      "logits/rejected": -0.07087734341621399,
      "logps/chosen": -341.2015075683594,
      "logps/rejected": -391.7242126464844,
      "loss": 0.4618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7297079563140869,
      "rewards/margins": 1.5788041353225708,
      "rewards/rejected": -2.308511972427368,
      "step": 10669
    },
    {
      "epoch": 1.3962378343011368,
      "grad_norm": 8.331277847290039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.552222728729248,
      "logits/rejected": -0.47340622544288635,
      "logps/chosen": -502.4139099121094,
      "logps/rejected": -461.1700439453125,
      "loss": 0.3348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21719130873680115,
      "rewards/margins": 1.2561640739440918,
      "rewards/rejected": -1.4733552932739258,
      "step": 10670
    },
    {
      "epoch": 1.3963686922384886,
      "grad_norm": 5.843572616577148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013358633033931255,
      "logits/rejected": -0.012344300746917725,
      "logps/chosen": -556.650146484375,
      "logps/rejected": -533.419921875,
      "loss": 0.2727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30247387290000916,
      "rewards/margins": 1.798596978187561,
      "rewards/rejected": -2.1010708808898926,
      "step": 10671
    },
    {
      "epoch": 1.3964995501758404,
      "grad_norm": 9.750542640686035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31903374195098877,
      "logits/rejected": -0.4454110860824585,
      "logps/chosen": -398.3450927734375,
      "logps/rejected": -335.814453125,
      "loss": 0.6935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1874597072601318,
      "rewards/margins": 0.6436500549316406,
      "rewards/rejected": -1.831109642982483,
      "step": 10672
    },
    {
      "epoch": 1.3966304081131922,
      "grad_norm": 8.616851806640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047593481838703156,
      "logits/rejected": -0.1461670845746994,
      "logps/chosen": -519.7704467773438,
      "logps/rejected": -531.1322631835938,
      "loss": 0.4766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8798355460166931,
      "rewards/margins": 1.1189255714416504,
      "rewards/rejected": -1.9987610578536987,
      "step": 10673
    },
    {
      "epoch": 1.396761266050544,
      "grad_norm": 6.7022786140441895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3593306243419647,
      "logits/rejected": -0.357707142829895,
      "logps/chosen": -405.0710144042969,
      "logps/rejected": -501.0415344238281,
      "loss": 0.2652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6907795071601868,
      "rewards/margins": 1.9779881238937378,
      "rewards/rejected": -2.6687676906585693,
      "step": 10674
    },
    {
      "epoch": 1.3968921239878958,
      "grad_norm": 1.358872413635254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06817913055419922,
      "logits/rejected": 0.031215712428092957,
      "logps/chosen": -442.43011474609375,
      "logps/rejected": -469.9546813964844,
      "loss": 0.0469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19964201748371124,
      "rewards/margins": 3.366672992706299,
      "rewards/rejected": -3.566314458847046,
      "step": 10675
    },
    {
      "epoch": 1.3970229819252473,
      "grad_norm": 5.348496913909912,
      "learning_rate": 5e-06,
      "logits/chosen": -0.036930352449417114,
      "logits/rejected": -0.0023417798802256584,
      "logps/chosen": -411.1445617675781,
      "logps/rejected": -470.36376953125,
      "loss": 0.203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3175218999385834,
      "rewards/margins": 2.3554978370666504,
      "rewards/rejected": -2.6730196475982666,
      "step": 10676
    },
    {
      "epoch": 1.3971538398625991,
      "grad_norm": 6.483372688293457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2696085274219513,
      "logits/rejected": -0.41704919934272766,
      "logps/chosen": -454.7501220703125,
      "logps/rejected": -475.6739196777344,
      "loss": 0.4109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.260056734085083,
      "rewards/margins": 1.1403613090515137,
      "rewards/rejected": -2.400418281555176,
      "step": 10677
    },
    {
      "epoch": 1.397284697799951,
      "grad_norm": 10.381093978881836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2548653781414032,
      "logits/rejected": -0.20766696333885193,
      "logps/chosen": -391.66033935546875,
      "logps/rejected": -402.9636535644531,
      "loss": 0.464,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48755985498428345,
      "rewards/margins": 1.087440848350525,
      "rewards/rejected": -1.5750007629394531,
      "step": 10678
    },
    {
      "epoch": 1.3974155557373027,
      "grad_norm": 14.476351737976074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18799665570259094,
      "logits/rejected": -0.41061151027679443,
      "logps/chosen": -421.39654541015625,
      "logps/rejected": -454.7524108886719,
      "loss": 1.046,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.8462114334106445,
      "rewards/margins": 0.5422905683517456,
      "rewards/rejected": -3.3885016441345215,
      "step": 10679
    },
    {
      "epoch": 1.3975464136746545,
      "grad_norm": 11.241171836853027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3317826986312866,
      "logits/rejected": -0.25699687004089355,
      "logps/chosen": -452.8697814941406,
      "logps/rejected": -385.2390441894531,
      "loss": 0.6852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1115003824234009,
      "rewards/margins": 1.16952383518219,
      "rewards/rejected": -2.281024217605591,
      "step": 10680
    },
    {
      "epoch": 1.3976772716120063,
      "grad_norm": 9.500027656555176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2844677269458771,
      "logits/rejected": -0.32536643743515015,
      "logps/chosen": -304.6856994628906,
      "logps/rejected": -416.877685546875,
      "loss": 0.5894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3996315002441406,
      "rewards/margins": 0.2848958671092987,
      "rewards/rejected": -1.6845273971557617,
      "step": 10681
    },
    {
      "epoch": 1.3978081295493578,
      "grad_norm": 7.270287990570068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7478164434432983,
      "logits/rejected": -0.5730447769165039,
      "logps/chosen": -557.73681640625,
      "logps/rejected": -481.0811462402344,
      "loss": 0.279,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5415759086608887,
      "rewards/margins": 2.5140299797058105,
      "rewards/rejected": -4.055605888366699,
      "step": 10682
    },
    {
      "epoch": 1.3979389874867096,
      "grad_norm": 7.0855255126953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03307759016752243,
      "logits/rejected": -0.18620000779628754,
      "logps/chosen": -367.929931640625,
      "logps/rejected": -341.5360412597656,
      "loss": 0.6349,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.140580415725708,
      "rewards/margins": 0.9579901695251465,
      "rewards/rejected": -2.0985705852508545,
      "step": 10683
    },
    {
      "epoch": 1.3980698454240614,
      "grad_norm": 7.5664520263671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3749852478504181,
      "logits/rejected": -0.2156405746936798,
      "logps/chosen": -628.2784423828125,
      "logps/rejected": -547.5303344726562,
      "loss": 0.3029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32468947768211365,
      "rewards/margins": 1.8382816314697266,
      "rewards/rejected": -2.162971258163452,
      "step": 10684
    },
    {
      "epoch": 1.3982007033614132,
      "grad_norm": 7.461158275604248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027119070291519165,
      "logits/rejected": 0.10405440628528595,
      "logps/chosen": -342.58538818359375,
      "logps/rejected": -393.67523193359375,
      "loss": 0.4055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4758335053920746,
      "rewards/margins": 1.0914299488067627,
      "rewards/rejected": -1.5672634840011597,
      "step": 10685
    },
    {
      "epoch": 1.398331561298765,
      "grad_norm": 10.787294387817383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34452930092811584,
      "logits/rejected": -0.2821834981441498,
      "logps/chosen": -430.35198974609375,
      "logps/rejected": -402.5151062011719,
      "loss": 0.6188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15887364745140076,
      "rewards/margins": 1.007805347442627,
      "rewards/rejected": -1.16667902469635,
      "step": 10686
    },
    {
      "epoch": 1.3984624192361168,
      "grad_norm": 6.890132904052734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37173956632614136,
      "logits/rejected": -0.36525973677635193,
      "logps/chosen": -441.169921875,
      "logps/rejected": -470.9785461425781,
      "loss": 0.231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7633257508277893,
      "rewards/margins": 2.2121057510375977,
      "rewards/rejected": -2.975431203842163,
      "step": 10687
    },
    {
      "epoch": 1.3985932771734686,
      "grad_norm": 13.598620414733887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4072641134262085,
      "logits/rejected": -0.2380567342042923,
      "logps/chosen": -415.2731628417969,
      "logps/rejected": -381.40283203125,
      "loss": 0.9049,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.205491304397583,
      "rewards/margins": 0.11521473526954651,
      "rewards/rejected": -1.3207058906555176,
      "step": 10688
    },
    {
      "epoch": 1.3987241351108204,
      "grad_norm": 11.671547889709473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34011638164520264,
      "logits/rejected": -0.23950578272342682,
      "logps/chosen": -450.2071838378906,
      "logps/rejected": -418.3983459472656,
      "loss": 0.7596,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2068891525268555,
      "rewards/margins": 0.43343356251716614,
      "rewards/rejected": -1.6403226852416992,
      "step": 10689
    },
    {
      "epoch": 1.3988549930481722,
      "grad_norm": 9.263891220092773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1424805074930191,
      "logits/rejected": -0.2147195041179657,
      "logps/chosen": -353.5078125,
      "logps/rejected": -342.95135498046875,
      "loss": 0.5016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10192928463220596,
      "rewards/margins": 0.9996578097343445,
      "rewards/rejected": -1.1015870571136475,
      "step": 10690
    },
    {
      "epoch": 1.398985850985524,
      "grad_norm": 6.9982709884643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01722145825624466,
      "logits/rejected": 0.06722886115312576,
      "logps/chosen": -542.82275390625,
      "logps/rejected": -473.201416015625,
      "loss": 0.368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3732481300830841,
      "rewards/margins": 1.4509981870651245,
      "rewards/rejected": -1.8242462873458862,
      "step": 10691
    },
    {
      "epoch": 1.3991167089228755,
      "grad_norm": 9.326624870300293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06879954040050507,
      "logits/rejected": -0.0958898514509201,
      "logps/chosen": -398.92138671875,
      "logps/rejected": -365.1048889160156,
      "loss": 0.5962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3801190853118896,
      "rewards/margins": 1.4150969982147217,
      "rewards/rejected": -2.7952158451080322,
      "step": 10692
    },
    {
      "epoch": 1.3992475668602273,
      "grad_norm": 8.037967681884766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18802735209465027,
      "logits/rejected": -0.15297985076904297,
      "logps/chosen": -383.9259948730469,
      "logps/rejected": -369.646240234375,
      "loss": 0.4665,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3425116539001465,
      "rewards/margins": 1.0796147584915161,
      "rewards/rejected": -2.422126531600952,
      "step": 10693
    },
    {
      "epoch": 1.3993784247975791,
      "grad_norm": 9.770601272583008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3581114113330841,
      "logits/rejected": -0.39350786805152893,
      "logps/chosen": -419.44775390625,
      "logps/rejected": -358.3205871582031,
      "loss": 0.4814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9848537445068359,
      "rewards/margins": 1.2505249977111816,
      "rewards/rejected": -2.2353787422180176,
      "step": 10694
    },
    {
      "epoch": 1.399509282734931,
      "grad_norm": 15.106749534606934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2555185556411743,
      "logits/rejected": -0.16000914573669434,
      "logps/chosen": -572.1954956054688,
      "logps/rejected": -557.9660034179688,
      "loss": 0.8802,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8916753530502319,
      "rewards/margins": 0.2588924765586853,
      "rewards/rejected": -1.1505677700042725,
      "step": 10695
    },
    {
      "epoch": 1.3996401406722827,
      "grad_norm": 5.790765762329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23677396774291992,
      "logits/rejected": -0.12011155486106873,
      "logps/chosen": -599.8009643554688,
      "logps/rejected": -531.5125732421875,
      "loss": 0.2751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9090899229049683,
      "rewards/margins": 1.6304410696029663,
      "rewards/rejected": -2.5395309925079346,
      "step": 10696
    },
    {
      "epoch": 1.3997709986096345,
      "grad_norm": 9.723886489868164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.054701339453458786,
      "logits/rejected": -0.07993733882904053,
      "logps/chosen": -602.4993896484375,
      "logps/rejected": -616.1803588867188,
      "loss": 0.4694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3305565118789673,
      "rewards/margins": 1.4077316522598267,
      "rewards/rejected": -2.738287925720215,
      "step": 10697
    },
    {
      "epoch": 1.399901856546986,
      "grad_norm": 10.435720443725586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.230284184217453,
      "logits/rejected": -0.15773087739944458,
      "logps/chosen": -612.248046875,
      "logps/rejected": -619.5841064453125,
      "loss": 0.5337,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0654542446136475,
      "rewards/margins": 0.9537112712860107,
      "rewards/rejected": -2.019165515899658,
      "step": 10698
    },
    {
      "epoch": 1.4000327144843379,
      "grad_norm": 3.753936529159546,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05852776765823364,
      "logits/rejected": -0.11878631263971329,
      "logps/chosen": -398.86029052734375,
      "logps/rejected": -349.15777587890625,
      "loss": 0.2658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34939128160476685,
      "rewards/margins": 2.297666072845459,
      "rewards/rejected": -1.948275089263916,
      "step": 10699
    },
    {
      "epoch": 1.4001635724216897,
      "grad_norm": 5.204263687133789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3095758855342865,
      "logits/rejected": -0.2661595046520233,
      "logps/chosen": -478.969482421875,
      "logps/rejected": -432.33087158203125,
      "loss": 0.2377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7227901816368103,
      "rewards/margins": 1.501045823097229,
      "rewards/rejected": -2.2238359451293945,
      "step": 10700
    },
    {
      "epoch": 1.4002944303590414,
      "grad_norm": 12.713362693786621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14731504023075104,
      "logits/rejected": -0.15571819245815277,
      "logps/chosen": -339.7733154296875,
      "logps/rejected": -350.0425109863281,
      "loss": 0.9464,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5861738324165344,
      "rewards/margins": 0.3687082529067993,
      "rewards/rejected": -0.9548820853233337,
      "step": 10701
    },
    {
      "epoch": 1.4004252882963932,
      "grad_norm": 8.837252616882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26512792706489563,
      "logits/rejected": -0.36423420906066895,
      "logps/chosen": -521.16650390625,
      "logps/rejected": -699.1240844726562,
      "loss": 0.4314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.773851990699768,
      "rewards/margins": 0.8746969699859619,
      "rewards/rejected": -2.6485490798950195,
      "step": 10702
    },
    {
      "epoch": 1.400556146233745,
      "grad_norm": 12.238847732543945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42359721660614014,
      "logits/rejected": -0.4251161813735962,
      "logps/chosen": -386.0750732421875,
      "logps/rejected": -377.3901672363281,
      "loss": 0.6487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.588068962097168,
      "rewards/margins": 0.2540792226791382,
      "rewards/rejected": -1.8421480655670166,
      "step": 10703
    },
    {
      "epoch": 1.4006870041710968,
      "grad_norm": 7.090710639953613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10716993361711502,
      "logits/rejected": -0.1994956135749817,
      "logps/chosen": -615.265869140625,
      "logps/rejected": -486.2453918457031,
      "loss": 0.2157,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3121795654296875,
      "rewards/margins": 1.957202672958374,
      "rewards/rejected": -2.2693822383880615,
      "step": 10704
    },
    {
      "epoch": 1.4008178621084486,
      "grad_norm": 8.590736389160156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22748634219169617,
      "logits/rejected": -0.02933967113494873,
      "logps/chosen": -469.4461669921875,
      "logps/rejected": -506.4513244628906,
      "loss": 0.5499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9458152055740356,
      "rewards/margins": 0.8129256963729858,
      "rewards/rejected": -1.758741021156311,
      "step": 10705
    },
    {
      "epoch": 1.4009487200458004,
      "grad_norm": 11.280445098876953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2325194776058197,
      "logits/rejected": -0.4269410967826843,
      "logps/chosen": -405.57891845703125,
      "logps/rejected": -321.31072998046875,
      "loss": 0.6529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8515619039535522,
      "rewards/margins": 0.6235630512237549,
      "rewards/rejected": -2.4751248359680176,
      "step": 10706
    },
    {
      "epoch": 1.401079577983152,
      "grad_norm": 9.113515853881836,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15916886925697327,
      "logits/rejected": -0.07436034083366394,
      "logps/chosen": -648.2181396484375,
      "logps/rejected": -415.6894226074219,
      "loss": 0.3704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8560879230499268,
      "rewards/margins": 1.108185887336731,
      "rewards/rejected": -1.9642736911773682,
      "step": 10707
    },
    {
      "epoch": 1.4012104359205038,
      "grad_norm": 6.597875595092773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20561809837818146,
      "logits/rejected": -0.01703236624598503,
      "logps/chosen": -348.4261779785156,
      "logps/rejected": -456.4591064453125,
      "loss": 0.3906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37395912408828735,
      "rewards/margins": 1.644890308380127,
      "rewards/rejected": -2.0188496112823486,
      "step": 10708
    },
    {
      "epoch": 1.4013412938578556,
      "grad_norm": 10.344231605529785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3998000919818878,
      "logits/rejected": -0.6538919806480408,
      "logps/chosen": -348.4014892578125,
      "logps/rejected": -437.043701171875,
      "loss": 0.5718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8843309879302979,
      "rewards/margins": 1.4194998741149902,
      "rewards/rejected": -3.303830862045288,
      "step": 10709
    },
    {
      "epoch": 1.4014721517952073,
      "grad_norm": 8.31766128540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.196366548538208,
      "logits/rejected": -0.22825585305690765,
      "logps/chosen": -378.3766174316406,
      "logps/rejected": -366.412109375,
      "loss": 0.3687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3812161684036255,
      "rewards/margins": 1.1828099489212036,
      "rewards/rejected": -2.564026117324829,
      "step": 10710
    },
    {
      "epoch": 1.4016030097325591,
      "grad_norm": 9.261363983154297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06698685884475708,
      "logits/rejected": -0.20508535206317902,
      "logps/chosen": -672.9829711914062,
      "logps/rejected": -477.8183288574219,
      "loss": 0.3478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5876586437225342,
      "rewards/margins": 1.695847749710083,
      "rewards/rejected": -3.283506393432617,
      "step": 10711
    },
    {
      "epoch": 1.401733867669911,
      "grad_norm": 12.210302352905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2771233320236206,
      "logits/rejected": -0.427372008562088,
      "logps/chosen": -488.5277099609375,
      "logps/rejected": -469.1409912109375,
      "loss": 0.6629,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3651762008666992,
      "rewards/margins": 1.0930567979812622,
      "rewards/rejected": -2.458232879638672,
      "step": 10712
    },
    {
      "epoch": 1.4018647256072625,
      "grad_norm": 6.8879547119140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11856347322463989,
      "logits/rejected": -0.11640474200248718,
      "logps/chosen": -435.91900634765625,
      "logps/rejected": -384.71466064453125,
      "loss": 0.3178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1143677234649658,
      "rewards/margins": 1.5673904418945312,
      "rewards/rejected": -2.681758165359497,
      "step": 10713
    },
    {
      "epoch": 1.4019955835446143,
      "grad_norm": 7.024469375610352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37574467062950134,
      "logits/rejected": -0.4403940439224243,
      "logps/chosen": -518.6325073242188,
      "logps/rejected": -453.794677734375,
      "loss": 0.4262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7834872007369995,
      "rewards/margins": 1.34255850315094,
      "rewards/rejected": -2.1260454654693604,
      "step": 10714
    },
    {
      "epoch": 1.402126441481966,
      "grad_norm": 8.77401065826416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07608431577682495,
      "logits/rejected": -0.23164446651935577,
      "logps/chosen": -275.8634338378906,
      "logps/rejected": -322.0190124511719,
      "loss": 0.5811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9840834140777588,
      "rewards/margins": 0.4910854697227478,
      "rewards/rejected": -1.4751688241958618,
      "step": 10715
    },
    {
      "epoch": 1.4022572994193179,
      "grad_norm": 7.116159915924072,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46132540702819824,
      "logits/rejected": -0.43620339035987854,
      "logps/chosen": -537.0376586914062,
      "logps/rejected": -757.4443969726562,
      "loss": 0.3221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.444579005241394,
      "rewards/margins": 3.0727548599243164,
      "rewards/rejected": -4.517334461212158,
      "step": 10716
    },
    {
      "epoch": 1.4023881573566697,
      "grad_norm": 10.154582977294922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01980101317167282,
      "logits/rejected": 0.017348110675811768,
      "logps/chosen": -352.26776123046875,
      "logps/rejected": -336.76873779296875,
      "loss": 0.687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9473133087158203,
      "rewards/margins": 0.5629184246063232,
      "rewards/rejected": -1.510231852531433,
      "step": 10717
    },
    {
      "epoch": 1.4025190152940215,
      "grad_norm": 7.135495662689209,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14691904187202454,
      "logits/rejected": -0.13331127166748047,
      "logps/chosen": -426.7640380859375,
      "logps/rejected": -322.34283447265625,
      "loss": 0.3981,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8515552878379822,
      "rewards/margins": 1.1567107439041138,
      "rewards/rejected": -2.008265972137451,
      "step": 10718
    },
    {
      "epoch": 1.4026498732313732,
      "grad_norm": 3.620363712310791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1300714761018753,
      "logits/rejected": -0.10047507286071777,
      "logps/chosen": -457.10919189453125,
      "logps/rejected": -465.93450927734375,
      "loss": 0.1453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22235527634620667,
      "rewards/margins": 2.3289992809295654,
      "rewards/rejected": -2.1066441535949707,
      "step": 10719
    },
    {
      "epoch": 1.402780731168725,
      "grad_norm": 7.7844462394714355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3833644986152649,
      "logits/rejected": -0.17135432362556458,
      "logps/chosen": -376.01043701171875,
      "logps/rejected": -391.32208251953125,
      "loss": 0.4469,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.374749779701233,
      "rewards/margins": 1.10512113571167,
      "rewards/rejected": -2.4798707962036133,
      "step": 10720
    },
    {
      "epoch": 1.4029115891060768,
      "grad_norm": 12.089083671569824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17937706410884857,
      "logits/rejected": -0.3233287036418915,
      "logps/chosen": -416.9356994628906,
      "logps/rejected": -457.394775390625,
      "loss": 0.7528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1253719329833984,
      "rewards/margins": 0.8285814523696899,
      "rewards/rejected": -1.9539533853530884,
      "step": 10721
    },
    {
      "epoch": 1.4030424470434284,
      "grad_norm": 5.426506519317627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39682090282440186,
      "logits/rejected": -0.3154718279838562,
      "logps/chosen": -445.8750305175781,
      "logps/rejected": -469.8540344238281,
      "loss": 0.2225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7712816596031189,
      "rewards/margins": 1.9735379219055176,
      "rewards/rejected": -2.7448196411132812,
      "step": 10722
    },
    {
      "epoch": 1.4031733049807802,
      "grad_norm": 4.851422309875488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16019006073474884,
      "logits/rejected": -0.13876689970493317,
      "logps/chosen": -438.180419921875,
      "logps/rejected": -419.3244323730469,
      "loss": 0.2698,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27983349561691284,
      "rewards/margins": 1.7693281173706055,
      "rewards/rejected": -1.489494800567627,
      "step": 10723
    },
    {
      "epoch": 1.403304162918132,
      "grad_norm": 5.159092426300049,
      "learning_rate": 5e-06,
      "logits/chosen": 0.044680990278720856,
      "logits/rejected": -0.0005543213337659836,
      "logps/chosen": -495.85784912109375,
      "logps/rejected": -427.09002685546875,
      "loss": 0.212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4357006251811981,
      "rewards/margins": 2.492994785308838,
      "rewards/rejected": -2.9286954402923584,
      "step": 10724
    },
    {
      "epoch": 1.4034350208554838,
      "grad_norm": 5.773247241973877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02647925540804863,
      "logits/rejected": -0.029183946549892426,
      "logps/chosen": -288.8743896484375,
      "logps/rejected": -326.4961242675781,
      "loss": 0.4154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40460166335105896,
      "rewards/margins": 1.9686646461486816,
      "rewards/rejected": -2.3732664585113525,
      "step": 10725
    },
    {
      "epoch": 1.4035658787928356,
      "grad_norm": 5.730572700500488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29597294330596924,
      "logits/rejected": -0.40039265155792236,
      "logps/chosen": -431.09423828125,
      "logps/rejected": -553.365966796875,
      "loss": 0.2395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16376131772994995,
      "rewards/margins": 2.6143603324890137,
      "rewards/rejected": -2.7781217098236084,
      "step": 10726
    },
    {
      "epoch": 1.4036967367301874,
      "grad_norm": 4.2405853271484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5182722210884094,
      "logits/rejected": -0.4165056347846985,
      "logps/chosen": -337.9927978515625,
      "logps/rejected": -387.1706237792969,
      "loss": 0.204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5530143976211548,
      "rewards/margins": 1.8441822528839111,
      "rewards/rejected": -2.3971965312957764,
      "step": 10727
    },
    {
      "epoch": 1.403827594667539,
      "grad_norm": 5.898505210876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09226048737764359,
      "logits/rejected": -0.08958069980144501,
      "logps/chosen": -445.1281433105469,
      "logps/rejected": -427.82257080078125,
      "loss": 0.244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7722374200820923,
      "rewards/margins": 1.6721261739730835,
      "rewards/rejected": -2.444363594055176,
      "step": 10728
    },
    {
      "epoch": 1.4039584526048907,
      "grad_norm": 7.2971696853637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34323281049728394,
      "logits/rejected": -0.38790860772132874,
      "logps/chosen": -525.7310180664062,
      "logps/rejected": -570.2762451171875,
      "loss": 0.4316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6647180318832397,
      "rewards/margins": 1.0435811281204224,
      "rewards/rejected": -1.7082992792129517,
      "step": 10729
    },
    {
      "epoch": 1.4040893105422425,
      "grad_norm": 7.122747898101807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19678041338920593,
      "logits/rejected": -0.2747211158275604,
      "logps/chosen": -389.3522033691406,
      "logps/rejected": -419.4404296875,
      "loss": 0.3638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4407238960266113,
      "rewards/margins": 1.7388386726379395,
      "rewards/rejected": -3.179562568664551,
      "step": 10730
    },
    {
      "epoch": 1.4042201684795943,
      "grad_norm": 4.316056728363037,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19426634907722473,
      "logits/rejected": -0.09873276948928833,
      "logps/chosen": -616.3233032226562,
      "logps/rejected": -502.006103515625,
      "loss": 0.1958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5186303853988647,
      "rewards/margins": 1.929153323173523,
      "rewards/rejected": -2.447783946990967,
      "step": 10731
    },
    {
      "epoch": 1.404351026416946,
      "grad_norm": 4.730625629425049,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2911761999130249,
      "logits/rejected": -0.3235512375831604,
      "logps/chosen": -416.0653991699219,
      "logps/rejected": -395.4308166503906,
      "loss": 0.2255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.019746964797377586,
      "rewards/margins": 1.7622435092926025,
      "rewards/rejected": -1.7819904088974,
      "step": 10732
    },
    {
      "epoch": 1.4044818843542979,
      "grad_norm": 7.198605060577393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.227504163980484,
      "logits/rejected": -0.28971463441848755,
      "logps/chosen": -613.552978515625,
      "logps/rejected": -349.3607482910156,
      "loss": 0.5136,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7563978433609009,
      "rewards/margins": 1.0131158828735352,
      "rewards/rejected": -1.7695138454437256,
      "step": 10733
    },
    {
      "epoch": 1.4046127422916497,
      "grad_norm": 10.08149528503418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.142604798078537,
      "logits/rejected": -0.180928573012352,
      "logps/chosen": -475.955322265625,
      "logps/rejected": -450.0519104003906,
      "loss": 0.7081,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8441709280014038,
      "rewards/margins": 0.7613242864608765,
      "rewards/rejected": -1.6054952144622803,
      "step": 10734
    },
    {
      "epoch": 1.4047436002290015,
      "grad_norm": 6.250598907470703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0012281723320484161,
      "logits/rejected": 0.15023669600486755,
      "logps/chosen": -507.39093017578125,
      "logps/rejected": -550.7047119140625,
      "loss": 0.2856,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6315504908561707,
      "rewards/margins": 1.9890787601470947,
      "rewards/rejected": -2.62062931060791,
      "step": 10735
    },
    {
      "epoch": 1.4048744581663533,
      "grad_norm": 6.5280866622924805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2318246066570282,
      "logits/rejected": -0.4131373167037964,
      "logps/chosen": -477.8868713378906,
      "logps/rejected": -368.6976623535156,
      "loss": 0.2957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.470181941986084,
      "rewards/margins": 1.5424596071243286,
      "rewards/rejected": -3.012641668319702,
      "step": 10736
    },
    {
      "epoch": 1.405005316103705,
      "grad_norm": 7.42987585067749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3678210377693176,
      "logits/rejected": -0.384576678276062,
      "logps/chosen": -616.9310302734375,
      "logps/rejected": -455.6481018066406,
      "loss": 0.5376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3881049156188965,
      "rewards/margins": 0.7945203185081482,
      "rewards/rejected": -2.1826255321502686,
      "step": 10737
    },
    {
      "epoch": 1.4051361740410566,
      "grad_norm": 8.559887886047363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31533360481262207,
      "logits/rejected": -0.3714698553085327,
      "logps/chosen": -427.77923583984375,
      "logps/rejected": -516.8198852539062,
      "loss": 0.391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8283945322036743,
      "rewards/margins": 2.507234811782837,
      "rewards/rejected": -3.335629463195801,
      "step": 10738
    },
    {
      "epoch": 1.4052670319784084,
      "grad_norm": 11.659016609191895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08516576886177063,
      "logits/rejected": -0.003320833668112755,
      "logps/chosen": -548.0194091796875,
      "logps/rejected": -467.3811340332031,
      "loss": 0.9839,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.946506142616272,
      "rewards/margins": -0.03049030900001526,
      "rewards/rejected": -1.9160157442092896,
      "step": 10739
    },
    {
      "epoch": 1.4053978899157602,
      "grad_norm": 5.7492547035217285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13215960562229156,
      "logits/rejected": -0.07878025621175766,
      "logps/chosen": -481.50189208984375,
      "logps/rejected": -503.72149658203125,
      "loss": 0.2292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6495791673660278,
      "rewards/margins": 2.025374174118042,
      "rewards/rejected": -3.6749532222747803,
      "step": 10740
    },
    {
      "epoch": 1.405528747853112,
      "grad_norm": 12.224820137023926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43926167488098145,
      "logits/rejected": -0.6919964551925659,
      "logps/chosen": -325.8058776855469,
      "logps/rejected": -302.3337097167969,
      "loss": 0.7445,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4882919788360596,
      "rewards/margins": 0.6501457691192627,
      "rewards/rejected": -2.138437509536743,
      "step": 10741
    },
    {
      "epoch": 1.4056596057904638,
      "grad_norm": 6.955629348754883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11240865290164948,
      "logits/rejected": -0.14926718175411224,
      "logps/chosen": -434.56854248046875,
      "logps/rejected": -338.32427978515625,
      "loss": 0.3268,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8644350171089172,
      "rewards/margins": 1.6806389093399048,
      "rewards/rejected": -2.545073986053467,
      "step": 10742
    },
    {
      "epoch": 1.4057904637278154,
      "grad_norm": 6.928591251373291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2972237765789032,
      "logits/rejected": -0.3099510669708252,
      "logps/chosen": -553.7971801757812,
      "logps/rejected": -638.4426879882812,
      "loss": 0.38,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9100532531738281,
      "rewards/margins": 1.704791784286499,
      "rewards/rejected": -2.6148452758789062,
      "step": 10743
    },
    {
      "epoch": 1.4059213216651671,
      "grad_norm": 9.11123275756836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36115309596061707,
      "logits/rejected": -0.0284750796854496,
      "logps/chosen": -533.2312622070312,
      "logps/rejected": -563.0712890625,
      "loss": 0.4866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06777651607990265,
      "rewards/margins": 1.1521251201629639,
      "rewards/rejected": -1.21990168094635,
      "step": 10744
    },
    {
      "epoch": 1.406052179602519,
      "grad_norm": 6.359846591949463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08549869805574417,
      "logits/rejected": 0.03431783244013786,
      "logps/chosen": -366.040771484375,
      "logps/rejected": -309.2668151855469,
      "loss": 0.3686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2273558378219604,
      "rewards/margins": 0.9860723614692688,
      "rewards/rejected": -2.213428020477295,
      "step": 10745
    },
    {
      "epoch": 1.4061830375398707,
      "grad_norm": 11.906514167785645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03503204137086868,
      "logits/rejected": -0.029624462127685547,
      "logps/chosen": -509.49737548828125,
      "logps/rejected": -474.34112548828125,
      "loss": 0.4989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5009739398956299,
      "rewards/margins": 0.9541871547698975,
      "rewards/rejected": -1.4551610946655273,
      "step": 10746
    },
    {
      "epoch": 1.4063138954772225,
      "grad_norm": 5.947296142578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3081689476966858,
      "logits/rejected": -0.30874869227409363,
      "logps/chosen": -450.6065673828125,
      "logps/rejected": -372.45135498046875,
      "loss": 0.3924,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6393966674804688,
      "rewards/margins": 1.397356629371643,
      "rewards/rejected": -2.0367534160614014,
      "step": 10747
    },
    {
      "epoch": 1.4064447534145743,
      "grad_norm": 12.455649375915527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2139795422554016,
      "logits/rejected": -0.34308820962905884,
      "logps/chosen": -466.0100402832031,
      "logps/rejected": -471.1775207519531,
      "loss": 0.5221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31616532802581787,
      "rewards/margins": 2.1184301376342773,
      "rewards/rejected": -2.4345955848693848,
      "step": 10748
    },
    {
      "epoch": 1.406575611351926,
      "grad_norm": 5.20219087600708,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3277673125267029,
      "logits/rejected": 0.17019855976104736,
      "logps/chosen": -465.4197692871094,
      "logps/rejected": -484.3376770019531,
      "loss": 0.3859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.972796618938446,
      "rewards/margins": 1.5319674015045166,
      "rewards/rejected": -2.5047638416290283,
      "step": 10749
    },
    {
      "epoch": 1.406706469289278,
      "grad_norm": 8.8291597366333,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2335888147354126,
      "logits/rejected": -0.32564079761505127,
      "logps/chosen": -708.971435546875,
      "logps/rejected": -435.122314453125,
      "loss": 0.2564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48044854402542114,
      "rewards/margins": 2.570161819458008,
      "rewards/rejected": -3.050610303878784,
      "step": 10750
    },
    {
      "epoch": 1.4068373272266297,
      "grad_norm": 11.776948928833008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3360458016395569,
      "logits/rejected": -0.2883123755455017,
      "logps/chosen": -539.7018432617188,
      "logps/rejected": -627.36474609375,
      "loss": 0.6655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0639671087265015,
      "rewards/margins": 0.8014005422592163,
      "rewards/rejected": -1.8653677701950073,
      "step": 10751
    },
    {
      "epoch": 1.4069681851639815,
      "grad_norm": 9.943137168884277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14402368664741516,
      "logits/rejected": -0.31116050481796265,
      "logps/chosen": -359.7055969238281,
      "logps/rejected": -511.77337646484375,
      "loss": 0.3627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.22761869430542,
      "rewards/margins": 1.9013780355453491,
      "rewards/rejected": -3.1289968490600586,
      "step": 10752
    },
    {
      "epoch": 1.407099043101333,
      "grad_norm": 6.238604545593262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3179769515991211,
      "logits/rejected": -0.43436184525489807,
      "logps/chosen": -393.6470031738281,
      "logps/rejected": -371.19842529296875,
      "loss": 0.3688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3543269634246826,
      "rewards/margins": 1.517218828201294,
      "rewards/rejected": -2.8715460300445557,
      "step": 10753
    },
    {
      "epoch": 1.4072299010386848,
      "grad_norm": 9.380582809448242,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1351657509803772,
      "logits/rejected": -0.2728947401046753,
      "logps/chosen": -391.2701110839844,
      "logps/rejected": -457.327880859375,
      "loss": 0.4587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20790347456932068,
      "rewards/margins": 1.5018996000289917,
      "rewards/rejected": -1.7098029851913452,
      "step": 10754
    },
    {
      "epoch": 1.4073607589760366,
      "grad_norm": 10.040820121765137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38634902238845825,
      "logits/rejected": -0.5374020934104919,
      "logps/chosen": -433.3105773925781,
      "logps/rejected": -472.52313232421875,
      "loss": 0.7003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2088348865509033,
      "rewards/margins": 0.4851701855659485,
      "rewards/rejected": -1.6940051317214966,
      "step": 10755
    },
    {
      "epoch": 1.4074916169133884,
      "grad_norm": 8.555315971374512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3927125334739685,
      "logits/rejected": -0.45626387000083923,
      "logps/chosen": -463.15789794921875,
      "logps/rejected": -377.5450439453125,
      "loss": 0.4986,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6041243076324463,
      "rewards/margins": 1.0944881439208984,
      "rewards/rejected": -2.698612689971924,
      "step": 10756
    },
    {
      "epoch": 1.4076224748507402,
      "grad_norm": 7.885001182556152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5655776858329773,
      "logits/rejected": -0.5175977945327759,
      "logps/chosen": -424.0032958984375,
      "logps/rejected": -323.7271423339844,
      "loss": 0.3256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.05940580368042,
      "rewards/margins": 1.7642393112182617,
      "rewards/rejected": -2.8236451148986816,
      "step": 10757
    },
    {
      "epoch": 1.407753332788092,
      "grad_norm": 4.000738620758057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1076919436454773,
      "logits/rejected": -0.0006712246686220169,
      "logps/chosen": -556.700439453125,
      "logps/rejected": -480.09393310546875,
      "loss": 0.2135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3232347369194031,
      "rewards/margins": 2.0943987369537354,
      "rewards/rejected": -2.417633295059204,
      "step": 10758
    },
    {
      "epoch": 1.4078841907254436,
      "grad_norm": 5.548004150390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14201267063617706,
      "logits/rejected": -0.2717490792274475,
      "logps/chosen": -421.7281188964844,
      "logps/rejected": -426.6160583496094,
      "loss": 0.3318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0030988454818726,
      "rewards/margins": 1.2104518413543701,
      "rewards/rejected": -2.2135508060455322,
      "step": 10759
    },
    {
      "epoch": 1.4080150486627954,
      "grad_norm": 10.337515830993652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2613513469696045,
      "logits/rejected": -0.4517362415790558,
      "logps/chosen": -539.668701171875,
      "logps/rejected": -515.4843139648438,
      "loss": 0.5941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.1909968852996826,
      "rewards/margins": 0.5265464782714844,
      "rewards/rejected": -2.717543363571167,
      "step": 10760
    },
    {
      "epoch": 1.4081459066001472,
      "grad_norm": 9.141584396362305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29827192425727844,
      "logits/rejected": -0.2696723937988281,
      "logps/chosen": -510.92279052734375,
      "logps/rejected": -455.56695556640625,
      "loss": 0.4842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2382848560810089,
      "rewards/margins": 2.1432077884674072,
      "rewards/rejected": -2.3814926147460938,
      "step": 10761
    },
    {
      "epoch": 1.408276764537499,
      "grad_norm": 5.188080787658691,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23421621322631836,
      "logits/rejected": 0.08229318261146545,
      "logps/chosen": -505.81378173828125,
      "logps/rejected": -365.76043701171875,
      "loss": 0.244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7697857022285461,
      "rewards/margins": 1.4681833982467651,
      "rewards/rejected": -2.237968921661377,
      "step": 10762
    },
    {
      "epoch": 1.4084076224748507,
      "grad_norm": 7.606266975402832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31924986839294434,
      "logits/rejected": -0.2589614689350128,
      "logps/chosen": -327.6714782714844,
      "logps/rejected": -410.3552551269531,
      "loss": 0.6712,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1173553466796875,
      "rewards/margins": 0.32775330543518066,
      "rewards/rejected": -1.4451086521148682,
      "step": 10763
    },
    {
      "epoch": 1.4085384804122025,
      "grad_norm": 5.032647132873535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037435147911310196,
      "logits/rejected": -0.32833945751190186,
      "logps/chosen": -398.7481384277344,
      "logps/rejected": -519.5762329101562,
      "loss": 0.2001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6068696975708008,
      "rewards/margins": 2.6692276000976562,
      "rewards/rejected": -3.276097297668457,
      "step": 10764
    },
    {
      "epoch": 1.4086693383495543,
      "grad_norm": 4.406562328338623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22428864240646362,
      "logits/rejected": -0.374828040599823,
      "logps/chosen": -352.50347900390625,
      "logps/rejected": -395.6573486328125,
      "loss": 0.2132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.73199862241745,
      "rewards/margins": 1.7456133365631104,
      "rewards/rejected": -2.477612257003784,
      "step": 10765
    },
    {
      "epoch": 1.4088001962869061,
      "grad_norm": 7.9487714767456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29001471400260925,
      "logits/rejected": -0.1338086724281311,
      "logps/chosen": -591.2992553710938,
      "logps/rejected": -603.299560546875,
      "loss": 0.4703,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2729625701904297,
      "rewards/margins": 1.0279502868652344,
      "rewards/rejected": -2.300912857055664,
      "step": 10766
    },
    {
      "epoch": 1.408931054224258,
      "grad_norm": 9.587540626525879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2347683161497116,
      "logits/rejected": -0.25236067175865173,
      "logps/chosen": -441.1828308105469,
      "logps/rejected": -478.6571960449219,
      "loss": 0.5062,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2482174634933472,
      "rewards/margins": 1.5918493270874023,
      "rewards/rejected": -2.84006667137146,
      "step": 10767
    },
    {
      "epoch": 1.4090619121616095,
      "grad_norm": 11.3382568359375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10830467939376831,
      "logits/rejected": -0.13439001142978668,
      "logps/chosen": -467.4068603515625,
      "logps/rejected": -363.2869873046875,
      "loss": 0.5073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6811412572860718,
      "rewards/margins": 1.6527315378189087,
      "rewards/rejected": -2.3338727951049805,
      "step": 10768
    },
    {
      "epoch": 1.4091927700989613,
      "grad_norm": 4.604785442352295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02837630733847618,
      "logits/rejected": -0.21014222502708435,
      "logps/chosen": -533.1311645507812,
      "logps/rejected": -452.2808532714844,
      "loss": 0.1736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22028809785842896,
      "rewards/margins": 2.512117385864258,
      "rewards/rejected": -2.732405424118042,
      "step": 10769
    },
    {
      "epoch": 1.409323628036313,
      "grad_norm": 7.054025650024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3435286581516266,
      "logits/rejected": -0.35872018337249756,
      "logps/chosen": -417.37969970703125,
      "logps/rejected": -459.4461669921875,
      "loss": 0.417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9375362992286682,
      "rewards/margins": 0.954948902130127,
      "rewards/rejected": -1.8924853801727295,
      "step": 10770
    },
    {
      "epoch": 1.4094544859736649,
      "grad_norm": 11.342461585998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26226121187210083,
      "logits/rejected": -0.46566832065582275,
      "logps/chosen": -566.6826171875,
      "logps/rejected": -447.6014404296875,
      "loss": 0.6956,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5253831148147583,
      "rewards/margins": 0.8250498175621033,
      "rewards/rejected": -2.350432872772217,
      "step": 10771
    },
    {
      "epoch": 1.4095853439110166,
      "grad_norm": 12.685410499572754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23089554905891418,
      "logits/rejected": -0.31380751729011536,
      "logps/chosen": -643.118408203125,
      "logps/rejected": -574.6312255859375,
      "loss": 0.4503,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6122667789459229,
      "rewards/margins": 1.2120749950408936,
      "rewards/rejected": -2.8243420124053955,
      "step": 10772
    },
    {
      "epoch": 1.4097162018483684,
      "grad_norm": 4.8561296463012695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1756216436624527,
      "logits/rejected": -0.04370015859603882,
      "logps/chosen": -601.1060180664062,
      "logps/rejected": -565.5137939453125,
      "loss": 0.1839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31310081481933594,
      "rewards/margins": 2.2735214233398438,
      "rewards/rejected": -2.5866222381591797,
      "step": 10773
    },
    {
      "epoch": 1.40984705978572,
      "grad_norm": 7.025752544403076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22763602435588837,
      "logits/rejected": -0.26123273372650146,
      "logps/chosen": -412.5317077636719,
      "logps/rejected": -313.3648681640625,
      "loss": 0.363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5129114389419556,
      "rewards/margins": 1.3750486373901367,
      "rewards/rejected": -1.8879601955413818,
      "step": 10774
    },
    {
      "epoch": 1.4099779177230718,
      "grad_norm": 9.915610313415527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32585251331329346,
      "logits/rejected": -0.19821015000343323,
      "logps/chosen": -434.5655212402344,
      "logps/rejected": -518.902587890625,
      "loss": 0.8343,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9442321062088013,
      "rewards/margins": 0.5101064443588257,
      "rewards/rejected": -2.454338550567627,
      "step": 10775
    },
    {
      "epoch": 1.4101087756604236,
      "grad_norm": 7.526863098144531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29451021552085876,
      "logits/rejected": -0.454588919878006,
      "logps/chosen": -587.1675415039062,
      "logps/rejected": -564.75048828125,
      "loss": 0.407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0360490083694458,
      "rewards/margins": 2.099123001098633,
      "rewards/rejected": -3.135172128677368,
      "step": 10776
    },
    {
      "epoch": 1.4102396335977754,
      "grad_norm": 5.125876426696777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13459455966949463,
      "logits/rejected": -0.07724417746067047,
      "logps/chosen": -432.4869384765625,
      "logps/rejected": -411.47784423828125,
      "loss": 0.3215,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3245329260826111,
      "rewards/margins": 1.707720398902893,
      "rewards/rejected": -2.0322532653808594,
      "step": 10777
    },
    {
      "epoch": 1.4103704915351272,
      "grad_norm": 14.29788875579834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060018397867679596,
      "logits/rejected": -0.1559879630804062,
      "logps/chosen": -323.638916015625,
      "logps/rejected": -438.5755310058594,
      "loss": 1.0015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7678983211517334,
      "rewards/margins": -0.08026141673326492,
      "rewards/rejected": -1.6876368522644043,
      "step": 10778
    },
    {
      "epoch": 1.410501349472479,
      "grad_norm": 7.193264007568359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0760829821228981,
      "logits/rejected": -0.36864161491394043,
      "logps/chosen": -434.875,
      "logps/rejected": -368.3902587890625,
      "loss": 0.3772,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3240642547607422,
      "rewards/margins": 1.9586195945739746,
      "rewards/rejected": -3.282684087753296,
      "step": 10779
    },
    {
      "epoch": 1.4106322074098308,
      "grad_norm": 8.82221794128418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43936967849731445,
      "logits/rejected": -0.486687034368515,
      "logps/chosen": -525.6834716796875,
      "logps/rejected": -422.8683166503906,
      "loss": 0.3987,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.222706913948059,
      "rewards/margins": 1.0143462419509888,
      "rewards/rejected": -2.237053155899048,
      "step": 10780
    },
    {
      "epoch": 1.4107630653471825,
      "grad_norm": 11.851028442382812,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0426553413271904,
      "logits/rejected": 0.06835727393627167,
      "logps/chosen": -520.3724365234375,
      "logps/rejected": -667.726806640625,
      "loss": 0.6963,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5299516916275024,
      "rewards/margins": 0.8591175079345703,
      "rewards/rejected": -2.389069080352783,
      "step": 10781
    },
    {
      "epoch": 1.4108939232845343,
      "grad_norm": 9.178836822509766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2600213587284088,
      "logits/rejected": -0.19796772301197052,
      "logps/chosen": -386.57720947265625,
      "logps/rejected": -446.4180603027344,
      "loss": 0.3905,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6550434827804565,
      "rewards/margins": 1.8356928825378418,
      "rewards/rejected": -2.490736484527588,
      "step": 10782
    },
    {
      "epoch": 1.4110247812218861,
      "grad_norm": 10.803606033325195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07668754458427429,
      "logits/rejected": 0.0014569088816642761,
      "logps/chosen": -359.3269348144531,
      "logps/rejected": -349.1375732421875,
      "loss": 0.5941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5313971042633057,
      "rewards/margins": 1.120051383972168,
      "rewards/rejected": -2.6514484882354736,
      "step": 10783
    },
    {
      "epoch": 1.4111556391592377,
      "grad_norm": 4.374509334564209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3142912983894348,
      "logits/rejected": -0.3656465709209442,
      "logps/chosen": -321.5998229980469,
      "logps/rejected": -396.8695373535156,
      "loss": 0.2181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17939692735671997,
      "rewards/margins": 1.768068790435791,
      "rewards/rejected": -1.5886716842651367,
      "step": 10784
    },
    {
      "epoch": 1.4112864970965895,
      "grad_norm": 7.09304666519165,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5493789911270142,
      "logits/rejected": -0.7749196887016296,
      "logps/chosen": -424.02984619140625,
      "logps/rejected": -359.2275390625,
      "loss": 0.531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2613372802734375,
      "rewards/margins": 0.6850274801254272,
      "rewards/rejected": -1.9463648796081543,
      "step": 10785
    },
    {
      "epoch": 1.4114173550339413,
      "grad_norm": 6.681779861450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25909560918807983,
      "logits/rejected": -0.3332796096801758,
      "logps/chosen": -522.8901977539062,
      "logps/rejected": -432.33056640625,
      "loss": 0.4314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7485887408256531,
      "rewards/margins": 2.177760124206543,
      "rewards/rejected": -2.9263486862182617,
      "step": 10786
    },
    {
      "epoch": 1.411548212971293,
      "grad_norm": 10.894735336303711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013868330046534538,
      "logits/rejected": -0.2576034665107727,
      "logps/chosen": -355.777587890625,
      "logps/rejected": -429.03277587890625,
      "loss": 0.6343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3570762872695923,
      "rewards/margins": 1.9597026109695435,
      "rewards/rejected": -3.316779136657715,
      "step": 10787
    },
    {
      "epoch": 1.4116790709086449,
      "grad_norm": 6.462289333343506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1264888346195221,
      "logits/rejected": -0.24611270427703857,
      "logps/chosen": -440.68585205078125,
      "logps/rejected": -472.1652526855469,
      "loss": 0.3144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16314315795898438,
      "rewards/margins": 1.336104154586792,
      "rewards/rejected": -1.4992473125457764,
      "step": 10788
    },
    {
      "epoch": 1.4118099288459964,
      "grad_norm": 6.717790603637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.539796769618988,
      "logits/rejected": -0.5352317690849304,
      "logps/chosen": -292.852294921875,
      "logps/rejected": -261.1549377441406,
      "loss": 0.459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6030789613723755,
      "rewards/margins": 1.1000336408615112,
      "rewards/rejected": -2.7031123638153076,
      "step": 10789
    },
    {
      "epoch": 1.4119407867833482,
      "grad_norm": 7.009624481201172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.318813681602478,
      "logits/rejected": -0.4563312232494354,
      "logps/chosen": -494.7389831542969,
      "logps/rejected": -434.77276611328125,
      "loss": 0.3661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8818823099136353,
      "rewards/margins": 1.8736170530319214,
      "rewards/rejected": -2.7554993629455566,
      "step": 10790
    },
    {
      "epoch": 1.4120716447207,
      "grad_norm": 4.197581768035889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24636386334896088,
      "logits/rejected": -0.0660233348608017,
      "logps/chosen": -396.3197326660156,
      "logps/rejected": -296.22698974609375,
      "loss": 0.2383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.30941906571388245,
      "rewards/margins": 1.5983868837356567,
      "rewards/rejected": -1.9078059196472168,
      "step": 10791
    },
    {
      "epoch": 1.4122025026580518,
      "grad_norm": 13.088241577148438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3219318091869354,
      "logits/rejected": -0.1898912489414215,
      "logps/chosen": -548.2550659179688,
      "logps/rejected": -498.62677001953125,
      "loss": 0.6204,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1276931762695312,
      "rewards/margins": 0.7340080142021179,
      "rewards/rejected": -1.8617011308670044,
      "step": 10792
    },
    {
      "epoch": 1.4123333605954036,
      "grad_norm": 6.9278788566589355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13337214291095734,
      "logits/rejected": -0.05362611263990402,
      "logps/chosen": -435.0721130371094,
      "logps/rejected": -519.0377197265625,
      "loss": 0.4091,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5316669344902039,
      "rewards/margins": 1.1431210041046143,
      "rewards/rejected": -1.674787998199463,
      "step": 10793
    },
    {
      "epoch": 1.4124642185327554,
      "grad_norm": 7.486137390136719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04606000706553459,
      "logits/rejected": -0.22142504155635834,
      "logps/chosen": -404.5711669921875,
      "logps/rejected": -653.975341796875,
      "loss": 0.5309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22186623513698578,
      "rewards/margins": 1.67462158203125,
      "rewards/rejected": -1.896487832069397,
      "step": 10794
    },
    {
      "epoch": 1.4125950764701072,
      "grad_norm": 4.064925670623779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29682064056396484,
      "logits/rejected": -0.2596452534198761,
      "logps/chosen": -376.200927734375,
      "logps/rejected": -483.2361755371094,
      "loss": 0.2314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2304623126983643,
      "rewards/margins": 2.482455015182495,
      "rewards/rejected": -3.7129173278808594,
      "step": 10795
    },
    {
      "epoch": 1.412725934407459,
      "grad_norm": 5.611375331878662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005092675797641277,
      "logits/rejected": -0.22028173506259918,
      "logps/chosen": -598.255615234375,
      "logps/rejected": -470.1038513183594,
      "loss": 0.2997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0067299604415894,
      "rewards/margins": 1.6180955171585083,
      "rewards/rejected": -2.6248257160186768,
      "step": 10796
    },
    {
      "epoch": 1.4128567923448108,
      "grad_norm": 6.69098424911499,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12135440856218338,
      "logits/rejected": -0.29320573806762695,
      "logps/chosen": -365.21575927734375,
      "logps/rejected": -715.2027587890625,
      "loss": 0.6355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5813531875610352,
      "rewards/margins": 2.1396586894989014,
      "rewards/rejected": -3.7210121154785156,
      "step": 10797
    },
    {
      "epoch": 1.4129876502821626,
      "grad_norm": 6.925782680511475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17156721651554108,
      "logits/rejected": 0.0013370905071496964,
      "logps/chosen": -656.440185546875,
      "logps/rejected": -427.8760681152344,
      "loss": 0.305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15193232893943787,
      "rewards/margins": 2.3300304412841797,
      "rewards/rejected": -2.17809796333313,
      "step": 10798
    },
    {
      "epoch": 1.4131185082195141,
      "grad_norm": 14.271431922912598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3784247636795044,
      "logits/rejected": -0.10952626913785934,
      "logps/chosen": -525.7053833007812,
      "logps/rejected": -394.9493713378906,
      "loss": 0.5671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3574469089508057,
      "rewards/margins": 1.6542069911956787,
      "rewards/rejected": -3.0116539001464844,
      "step": 10799
    },
    {
      "epoch": 1.413249366156866,
      "grad_norm": 7.435483932495117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12980589270591736,
      "logits/rejected": -0.06472207605838776,
      "logps/chosen": -605.4517822265625,
      "logps/rejected": -438.7247009277344,
      "loss": 0.3616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2999160885810852,
      "rewards/margins": 1.2811566591262817,
      "rewards/rejected": -1.5810726881027222,
      "step": 10800
    },
    {
      "epoch": 1.4133802240942177,
      "grad_norm": 7.639647483825684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14408744871616364,
      "logits/rejected": -0.4349393844604492,
      "logps/chosen": -462.06988525390625,
      "logps/rejected": -337.5871887207031,
      "loss": 0.4945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3996305465698242,
      "rewards/margins": 1.1405290365219116,
      "rewards/rejected": -2.5401594638824463,
      "step": 10801
    },
    {
      "epoch": 1.4135110820315695,
      "grad_norm": 6.448485374450684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18254578113555908,
      "logits/rejected": -0.2074224054813385,
      "logps/chosen": -379.729248046875,
      "logps/rejected": -415.6080627441406,
      "loss": 0.3302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4179975688457489,
      "rewards/margins": 1.8835554122924805,
      "rewards/rejected": -2.3015530109405518,
      "step": 10802
    },
    {
      "epoch": 1.4136419399689213,
      "grad_norm": 8.701801300048828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32337498664855957,
      "logits/rejected": -0.3811478018760681,
      "logps/chosen": -418.3891906738281,
      "logps/rejected": -397.9567565917969,
      "loss": 0.3917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6799460649490356,
      "rewards/margins": 1.6673657894134521,
      "rewards/rejected": -3.3473122119903564,
      "step": 10803
    },
    {
      "epoch": 1.413772797906273,
      "grad_norm": 8.237286567687988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4146363139152527,
      "logits/rejected": -0.3949999511241913,
      "logps/chosen": -413.3924865722656,
      "logps/rejected": -359.2145080566406,
      "loss": 0.4799,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1965152025222778,
      "rewards/margins": 0.7201310396194458,
      "rewards/rejected": -1.9166464805603027,
      "step": 10804
    },
    {
      "epoch": 1.4139036558436247,
      "grad_norm": 6.1185431480407715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1392124593257904,
      "logits/rejected": -0.182393878698349,
      "logps/chosen": -532.1903076171875,
      "logps/rejected": -485.5703430175781,
      "loss": 0.3664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3366225957870483,
      "rewards/margins": 1.5011037588119507,
      "rewards/rejected": -2.837726354598999,
      "step": 10805
    },
    {
      "epoch": 1.4140345137809764,
      "grad_norm": 11.193754196166992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3417835533618927,
      "logits/rejected": -0.39263975620269775,
      "logps/chosen": -398.8380126953125,
      "logps/rejected": -400.6669006347656,
      "loss": 0.6383,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2010819911956787,
      "rewards/margins": 0.7237753868103027,
      "rewards/rejected": -1.9248573780059814,
      "step": 10806
    },
    {
      "epoch": 1.4141653717183282,
      "grad_norm": 9.33230972290039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15747381746768951,
      "logits/rejected": -0.20768076181411743,
      "logps/chosen": -554.0441284179688,
      "logps/rejected": -475.58648681640625,
      "loss": 0.411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5619352459907532,
      "rewards/margins": 1.161490797996521,
      "rewards/rejected": -1.723426103591919,
      "step": 10807
    },
    {
      "epoch": 1.41429622965568,
      "grad_norm": 4.137363433837891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043197836726903915,
      "logits/rejected": -0.011392354965209961,
      "logps/chosen": -413.896240234375,
      "logps/rejected": -354.9769287109375,
      "loss": 0.1656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05025883764028549,
      "rewards/margins": 2.409080743789673,
      "rewards/rejected": -2.4593396186828613,
      "step": 10808
    },
    {
      "epoch": 1.4144270875930318,
      "grad_norm": 5.484521865844727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10950671136379242,
      "logits/rejected": -0.2558768689632416,
      "logps/chosen": -434.7799377441406,
      "logps/rejected": -397.1455078125,
      "loss": 0.3062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01843203604221344,
      "rewards/margins": 1.3236885070800781,
      "rewards/rejected": -1.3421205282211304,
      "step": 10809
    },
    {
      "epoch": 1.4145579455303836,
      "grad_norm": 9.780664443969727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.027864741161465645,
      "logits/rejected": 0.024500178173184395,
      "logps/chosen": -527.1962890625,
      "logps/rejected": -400.7066955566406,
      "loss": 0.6896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4006147086620331,
      "rewards/margins": 1.1609091758728027,
      "rewards/rejected": -1.5615239143371582,
      "step": 10810
    },
    {
      "epoch": 1.4146888034677354,
      "grad_norm": 8.487149238586426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18212248384952545,
      "logits/rejected": -0.14650878310203552,
      "logps/chosen": -296.6298522949219,
      "logps/rejected": -356.4744873046875,
      "loss": 0.4852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9384317398071289,
      "rewards/margins": 1.6545376777648926,
      "rewards/rejected": -2.5929696559906006,
      "step": 10811
    },
    {
      "epoch": 1.4148196614050872,
      "grad_norm": 11.827613830566406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2252204716205597,
      "logits/rejected": -0.05261824280023575,
      "logps/chosen": -426.01171875,
      "logps/rejected": -399.0052490234375,
      "loss": 0.9647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2323959320783615,
      "rewards/margins": 0.02307089790701866,
      "rewards/rejected": -0.25546684861183167,
      "step": 10812
    },
    {
      "epoch": 1.414950519342439,
      "grad_norm": 9.743877410888672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3183042109012604,
      "logits/rejected": -0.21219997107982635,
      "logps/chosen": -773.462890625,
      "logps/rejected": -519.5574951171875,
      "loss": 0.4532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9229530692100525,
      "rewards/margins": 1.4464256763458252,
      "rewards/rejected": -2.3693785667419434,
      "step": 10813
    },
    {
      "epoch": 1.4150813772797906,
      "grad_norm": 6.933366775512695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1412559300661087,
      "logits/rejected": -0.31023430824279785,
      "logps/chosen": -478.48675537109375,
      "logps/rejected": -350.28125,
      "loss": 0.3154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0125789642333984,
      "rewards/margins": 1.6269123554229736,
      "rewards/rejected": -3.639491081237793,
      "step": 10814
    },
    {
      "epoch": 1.4152122352171423,
      "grad_norm": 6.072840213775635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10281249135732651,
      "logits/rejected": -0.21098564565181732,
      "logps/chosen": -623.95654296875,
      "logps/rejected": -534.913330078125,
      "loss": 0.364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47632086277008057,
      "rewards/margins": 1.1995000839233398,
      "rewards/rejected": -1.6758208274841309,
      "step": 10815
    },
    {
      "epoch": 1.4153430931544941,
      "grad_norm": 8.6951904296875,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16305528581142426,
      "logits/rejected": -0.02030625194311142,
      "logps/chosen": -625.8677978515625,
      "logps/rejected": -530.1465454101562,
      "loss": 0.396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05834425240755081,
      "rewards/margins": 1.8447622060775757,
      "rewards/rejected": -1.7864179611206055,
      "step": 10816
    },
    {
      "epoch": 1.415473951091846,
      "grad_norm": 9.207236289978027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043413903564214706,
      "logits/rejected": -0.15043504536151886,
      "logps/chosen": -359.9398193359375,
      "logps/rejected": -424.6091003417969,
      "loss": 0.8379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0680588483810425,
      "rewards/margins": 0.17121101915836334,
      "rewards/rejected": -1.2392698526382446,
      "step": 10817
    },
    {
      "epoch": 1.4156048090291977,
      "grad_norm": 15.158732414245605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.151426762342453,
      "logits/rejected": -0.405081182718277,
      "logps/chosen": -450.89349365234375,
      "logps/rejected": -468.2257080078125,
      "loss": 0.7445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6562855243682861,
      "rewards/margins": 0.3139610290527344,
      "rewards/rejected": -1.970246434211731,
      "step": 10818
    },
    {
      "epoch": 1.4157356669665495,
      "grad_norm": 7.545509338378906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2910867929458618,
      "logits/rejected": -0.3699742257595062,
      "logps/chosen": -352.1798095703125,
      "logps/rejected": -375.8400573730469,
      "loss": 0.5799,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5248481035232544,
      "rewards/margins": 0.5230171084403992,
      "rewards/rejected": -2.047865152359009,
      "step": 10819
    },
    {
      "epoch": 1.415866524903901,
      "grad_norm": 8.535173416137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34180375933647156,
      "logits/rejected": -0.2387378215789795,
      "logps/chosen": -374.3628234863281,
      "logps/rejected": -510.69317626953125,
      "loss": 0.4659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44591638445854187,
      "rewards/margins": 0.7454242706298828,
      "rewards/rejected": -1.191340684890747,
      "step": 10820
    },
    {
      "epoch": 1.4159973828412529,
      "grad_norm": 6.307479381561279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12492471933364868,
      "logits/rejected": -0.3001517057418823,
      "logps/chosen": -398.65264892578125,
      "logps/rejected": -484.34637451171875,
      "loss": 0.3844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38184911012649536,
      "rewards/margins": 1.3946337699890137,
      "rewards/rejected": -1.7764827013015747,
      "step": 10821
    },
    {
      "epoch": 1.4161282407786047,
      "grad_norm": 6.024744033813477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08804405480623245,
      "logits/rejected": -0.08660415560007095,
      "logps/chosen": -450.32073974609375,
      "logps/rejected": -439.66070556640625,
      "loss": 0.267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4607452154159546,
      "rewards/margins": 1.9109165668487549,
      "rewards/rejected": -2.371661901473999,
      "step": 10822
    },
    {
      "epoch": 1.4162590987159565,
      "grad_norm": 12.58754825592041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05704618990421295,
      "logits/rejected": -0.20622137188911438,
      "logps/chosen": -621.0267944335938,
      "logps/rejected": -509.649169921875,
      "loss": 0.6505,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10835802555084229,
      "rewards/margins": 1.4052441120147705,
      "rewards/rejected": -1.5136020183563232,
      "step": 10823
    },
    {
      "epoch": 1.4163899566533082,
      "grad_norm": 10.713643074035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16028735041618347,
      "logits/rejected": -0.11822585761547089,
      "logps/chosen": -421.5212097167969,
      "logps/rejected": -531.3323974609375,
      "loss": 0.8396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8439838886260986,
      "rewards/margins": 0.7056299448013306,
      "rewards/rejected": -1.5496139526367188,
      "step": 10824
    },
    {
      "epoch": 1.41652081459066,
      "grad_norm": 10.243391036987305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48804453015327454,
      "logits/rejected": -0.25730597972869873,
      "logps/chosen": -456.201171875,
      "logps/rejected": -517.5502319335938,
      "loss": 0.7614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2353497743606567,
      "rewards/margins": 0.23556216061115265,
      "rewards/rejected": -1.4709118604660034,
      "step": 10825
    },
    {
      "epoch": 1.4166516725280118,
      "grad_norm": 7.074717044830322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41539210081100464,
      "logits/rejected": -0.16046305000782013,
      "logps/chosen": -430.72259521484375,
      "logps/rejected": -407.14453125,
      "loss": 0.399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9566528797149658,
      "rewards/margins": 1.0897961854934692,
      "rewards/rejected": -2.0464491844177246,
      "step": 10826
    },
    {
      "epoch": 1.4167825304653636,
      "grad_norm": 15.726207733154297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22349420189857483,
      "logits/rejected": -0.3226422667503357,
      "logps/chosen": -473.5329284667969,
      "logps/rejected": -361.8922424316406,
      "loss": 0.8393,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.139069080352783,
      "rewards/margins": 0.035876452922821045,
      "rewards/rejected": -2.17494535446167,
      "step": 10827
    },
    {
      "epoch": 1.4169133884027154,
      "grad_norm": 7.318475723266602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.431294322013855,
      "logits/rejected": -0.3925870656967163,
      "logps/chosen": -568.9052734375,
      "logps/rejected": -402.89251708984375,
      "loss": 0.4994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9704029560089111,
      "rewards/margins": 1.2490651607513428,
      "rewards/rejected": -2.219468116760254,
      "step": 10828
    },
    {
      "epoch": 1.417044246340067,
      "grad_norm": 9.17719554901123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21283505856990814,
      "logits/rejected": 0.016495874151587486,
      "logps/chosen": -278.4908752441406,
      "logps/rejected": -297.0143737792969,
      "loss": 0.5348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32445037364959717,
      "rewards/margins": 0.7933019399642944,
      "rewards/rejected": -1.1177523136138916,
      "step": 10829
    },
    {
      "epoch": 1.4171751042774188,
      "grad_norm": 8.7122163772583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23646631836891174,
      "logits/rejected": -0.31632256507873535,
      "logps/chosen": -411.35772705078125,
      "logps/rejected": -426.9720458984375,
      "loss": 0.6082,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6788125038146973,
      "rewards/margins": 0.8342002630233765,
      "rewards/rejected": -1.5130126476287842,
      "step": 10830
    },
    {
      "epoch": 1.4173059622147706,
      "grad_norm": 7.702115058898926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07067862153053284,
      "logits/rejected": -0.25829851627349854,
      "logps/chosen": -443.3048400878906,
      "logps/rejected": -414.16265869140625,
      "loss": 0.3585,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0051637887954712,
      "rewards/margins": 1.3885679244995117,
      "rewards/rejected": -2.3937318325042725,
      "step": 10831
    },
    {
      "epoch": 1.4174368201521224,
      "grad_norm": 6.536859512329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3115408420562744,
      "logits/rejected": -0.4772641360759735,
      "logps/chosen": -323.88970947265625,
      "logps/rejected": -381.8763122558594,
      "loss": 0.5216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8611797094345093,
      "rewards/margins": 1.3355242013931274,
      "rewards/rejected": -2.1967039108276367,
      "step": 10832
    },
    {
      "epoch": 1.4175676780894741,
      "grad_norm": 6.1743292808532715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38896048069000244,
      "logits/rejected": -0.3929784893989563,
      "logps/chosen": -331.951171875,
      "logps/rejected": -361.557373046875,
      "loss": 0.4413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2175161838531494,
      "rewards/margins": 0.928864598274231,
      "rewards/rejected": -2.14638090133667,
      "step": 10833
    },
    {
      "epoch": 1.417698536026826,
      "grad_norm": 11.595417022705078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4541287124156952,
      "logits/rejected": -0.5768857598304749,
      "logps/chosen": -359.8494873046875,
      "logps/rejected": -353.9131164550781,
      "loss": 0.7618,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9241085052490234,
      "rewards/margins": 0.03286518156528473,
      "rewards/rejected": -1.9569737911224365,
      "step": 10834
    },
    {
      "epoch": 1.4178293939641775,
      "grad_norm": 7.191890716552734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16173356771469116,
      "logits/rejected": -0.2643951177597046,
      "logps/chosen": -342.30487060546875,
      "logps/rejected": -354.7696838378906,
      "loss": 0.3499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5052667856216431,
      "rewards/margins": 1.009392261505127,
      "rewards/rejected": -1.5146589279174805,
      "step": 10835
    },
    {
      "epoch": 1.4179602519015293,
      "grad_norm": 9.431381225585938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3949262797832489,
      "logits/rejected": -0.4117049276828766,
      "logps/chosen": -402.8277282714844,
      "logps/rejected": -417.4486083984375,
      "loss": 0.3623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6906003952026367,
      "rewards/margins": 1.8772649765014648,
      "rewards/rejected": -2.5678653717041016,
      "step": 10836
    },
    {
      "epoch": 1.418091109838881,
      "grad_norm": 6.412611961364746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1890551745891571,
      "logits/rejected": -0.08084076642990112,
      "logps/chosen": -478.47149658203125,
      "logps/rejected": -376.4630432128906,
      "loss": 0.3404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04909057915210724,
      "rewards/margins": 1.3657835721969604,
      "rewards/rejected": -1.3166931867599487,
      "step": 10837
    },
    {
      "epoch": 1.4182219677762329,
      "grad_norm": 11.414584159851074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2558838427066803,
      "logits/rejected": -0.6297224164009094,
      "logps/chosen": -421.43560791015625,
      "logps/rejected": -358.13232421875,
      "loss": 1.0902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9843580722808838,
      "rewards/margins": 0.9052748084068298,
      "rewards/rejected": -2.8896329402923584,
      "step": 10838
    },
    {
      "epoch": 1.4183528257135847,
      "grad_norm": 8.594880104064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02149854600429535,
      "logits/rejected": -0.09529086947441101,
      "logps/chosen": -407.1647033691406,
      "logps/rejected": -409.5106201171875,
      "loss": 0.5441,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.439695954322815,
      "rewards/margins": 0.5447452664375305,
      "rewards/rejected": -1.9844412803649902,
      "step": 10839
    },
    {
      "epoch": 1.4184836836509365,
      "grad_norm": 7.048346996307373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19682221114635468,
      "logits/rejected": -0.558163046836853,
      "logps/chosen": -325.13604736328125,
      "logps/rejected": -338.38116455078125,
      "loss": 0.4036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8191465735435486,
      "rewards/margins": 1.2455440759658813,
      "rewards/rejected": -2.064690589904785,
      "step": 10840
    },
    {
      "epoch": 1.4186145415882883,
      "grad_norm": 8.751832962036133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11083400249481201,
      "logits/rejected": -0.24042785167694092,
      "logps/chosen": -412.231201171875,
      "logps/rejected": -404.114013671875,
      "loss": 0.5505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8942009210586548,
      "rewards/margins": 0.99669349193573,
      "rewards/rejected": -1.8908942937850952,
      "step": 10841
    },
    {
      "epoch": 1.41874539952564,
      "grad_norm": 5.617868423461914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14559125900268555,
      "logits/rejected": -0.4321553707122803,
      "logps/chosen": -350.35516357421875,
      "logps/rejected": -304.6091003417969,
      "loss": 0.3063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5946312546730042,
      "rewards/margins": 2.3332812786102295,
      "rewards/rejected": -2.927912473678589,
      "step": 10842
    },
    {
      "epoch": 1.4188762574629918,
      "grad_norm": 9.416377067565918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09754204750061035,
      "logits/rejected": -0.3517947494983673,
      "logps/chosen": -525.7686157226562,
      "logps/rejected": -571.375244140625,
      "loss": 0.4609,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36670786142349243,
      "rewards/margins": 1.7078003883361816,
      "rewards/rejected": -2.0745084285736084,
      "step": 10843
    },
    {
      "epoch": 1.4190071154003436,
      "grad_norm": 10.106884002685547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24926626682281494,
      "logits/rejected": -0.31007465720176697,
      "logps/chosen": -423.5714416503906,
      "logps/rejected": -582.5091552734375,
      "loss": 0.6024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7149940133094788,
      "rewards/margins": 1.2677406072616577,
      "rewards/rejected": -1.9827347993850708,
      "step": 10844
    },
    {
      "epoch": 1.4191379733376952,
      "grad_norm": 7.026308536529541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06946258246898651,
      "logits/rejected": 0.050092924386262894,
      "logps/chosen": -468.8563537597656,
      "logps/rejected": -387.54620361328125,
      "loss": 0.5012,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5671989321708679,
      "rewards/margins": 1.1183295249938965,
      "rewards/rejected": -1.6855283975601196,
      "step": 10845
    },
    {
      "epoch": 1.419268831275047,
      "grad_norm": 8.616080284118652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1208379864692688,
      "logits/rejected": -0.16677486896514893,
      "logps/chosen": -487.460205078125,
      "logps/rejected": -458.3158874511719,
      "loss": 0.5026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9984999299049377,
      "rewards/margins": 1.0956103801727295,
      "rewards/rejected": -2.0941102504730225,
      "step": 10846
    },
    {
      "epoch": 1.4193996892123988,
      "grad_norm": 2.632007122039795,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11788316816091537,
      "logits/rejected": 0.009267189539968967,
      "logps/chosen": -376.11431884765625,
      "logps/rejected": -447.6019592285156,
      "loss": 0.0788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05493125319480896,
      "rewards/margins": 3.17193603515625,
      "rewards/rejected": -3.1170051097869873,
      "step": 10847
    },
    {
      "epoch": 1.4195305471497506,
      "grad_norm": 8.070757865905762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4521287679672241,
      "logits/rejected": -0.33125239610671997,
      "logps/chosen": -303.3371887207031,
      "logps/rejected": -374.9484558105469,
      "loss": 0.527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8161272406578064,
      "rewards/margins": 0.8722541332244873,
      "rewards/rejected": -1.6883814334869385,
      "step": 10848
    },
    {
      "epoch": 1.4196614050871024,
      "grad_norm": 10.547843933105469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11959300935268402,
      "logits/rejected": -0.02717137522995472,
      "logps/chosen": -426.8482360839844,
      "logps/rejected": -528.159423828125,
      "loss": 0.5446,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.054857611656189,
      "rewards/margins": 0.796860933303833,
      "rewards/rejected": -1.8517186641693115,
      "step": 10849
    },
    {
      "epoch": 1.4197922630244542,
      "grad_norm": 10.513799667358398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3499930202960968,
      "logits/rejected": -0.5945037603378296,
      "logps/chosen": -377.9899597167969,
      "logps/rejected": -451.08941650390625,
      "loss": 0.6976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3574135303497314,
      "rewards/margins": 1.0874474048614502,
      "rewards/rejected": -2.4448609352111816,
      "step": 10850
    },
    {
      "epoch": 1.4199231209618057,
      "grad_norm": 9.839495658874512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08692718297243118,
      "logits/rejected": -0.07885795086622238,
      "logps/chosen": -283.1142883300781,
      "logps/rejected": -279.9915771484375,
      "loss": 0.5837,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.882224977016449,
      "rewards/margins": 1.0341358184814453,
      "rewards/rejected": -1.916360855102539,
      "step": 10851
    },
    {
      "epoch": 1.4200539788991575,
      "grad_norm": 10.245003700256348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14254158735275269,
      "logits/rejected": -0.10852430760860443,
      "logps/chosen": -473.25506591796875,
      "logps/rejected": -464.2131042480469,
      "loss": 0.5332,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.094741940498352,
      "rewards/margins": 0.7369390726089478,
      "rewards/rejected": -1.8316808938980103,
      "step": 10852
    },
    {
      "epoch": 1.4201848368365093,
      "grad_norm": 6.262737274169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1998036503791809,
      "logits/rejected": -0.03352922946214676,
      "logps/chosen": -867.2210693359375,
      "logps/rejected": -597.5951538085938,
      "loss": 0.23,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44137459993362427,
      "rewards/margins": 2.216890811920166,
      "rewards/rejected": -1.775516152381897,
      "step": 10853
    },
    {
      "epoch": 1.420315694773861,
      "grad_norm": 5.361817836761475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32479894161224365,
      "logits/rejected": -0.4170750081539154,
      "logps/chosen": -504.6649475097656,
      "logps/rejected": -313.6686706542969,
      "loss": 0.2335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6027979254722595,
      "rewards/margins": 2.0763676166534424,
      "rewards/rejected": -2.679165840148926,
      "step": 10854
    },
    {
      "epoch": 1.420446552711213,
      "grad_norm": 11.658051490783691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.167181134223938,
      "logits/rejected": -0.3435754179954529,
      "logps/chosen": -345.6734313964844,
      "logps/rejected": -306.9171142578125,
      "loss": 0.8404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6285715103149414,
      "rewards/margins": 0.07577343285083771,
      "rewards/rejected": -1.7043449878692627,
      "step": 10855
    },
    {
      "epoch": 1.4205774106485647,
      "grad_norm": 5.442473411560059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18636278808116913,
      "logits/rejected": -0.03553595393896103,
      "logps/chosen": -504.81439208984375,
      "logps/rejected": -462.6690979003906,
      "loss": 0.2831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22997531294822693,
      "rewards/margins": 1.7464238405227661,
      "rewards/rejected": -1.9763991832733154,
      "step": 10856
    },
    {
      "epoch": 1.4207082685859165,
      "grad_norm": 12.853255271911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5284053087234497,
      "logits/rejected": -0.3148493468761444,
      "logps/chosen": -480.011962890625,
      "logps/rejected": -518.0330810546875,
      "loss": 0.7013,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.232370376586914,
      "rewards/margins": 0.7602554559707642,
      "rewards/rejected": -1.9926257133483887,
      "step": 10857
    },
    {
      "epoch": 1.4208391265232683,
      "grad_norm": 11.81608772277832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28196704387664795,
      "logits/rejected": -0.2851245403289795,
      "logps/chosen": -332.8515625,
      "logps/rejected": -388.4443359375,
      "loss": 0.6396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5975240468978882,
      "rewards/margins": 0.6502389907836914,
      "rewards/rejected": -2.247762680053711,
      "step": 10858
    },
    {
      "epoch": 1.42096998446062,
      "grad_norm": 5.658997058868408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15183375775814056,
      "logits/rejected": -0.3418191075325012,
      "logps/chosen": -342.5914001464844,
      "logps/rejected": -294.7051086425781,
      "loss": 0.3044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3462986350059509,
      "rewards/margins": 1.3270384073257446,
      "rewards/rejected": -1.6733369827270508,
      "step": 10859
    },
    {
      "epoch": 1.4211008423979716,
      "grad_norm": 11.029053688049316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3015141487121582,
      "logits/rejected": -0.3309163749217987,
      "logps/chosen": -326.46490478515625,
      "logps/rejected": -253.25143432617188,
      "loss": 0.6756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15084238350391388,
      "rewards/margins": 0.522760272026062,
      "rewards/rejected": -0.6736027002334595,
      "step": 10860
    },
    {
      "epoch": 1.4212317003353234,
      "grad_norm": 6.501614570617676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10004851222038269,
      "logits/rejected": -0.292727530002594,
      "logps/chosen": -530.6436767578125,
      "logps/rejected": -471.2337341308594,
      "loss": 0.2686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4038722515106201,
      "rewards/margins": 2.194492816925049,
      "rewards/rejected": -3.598365068435669,
      "step": 10861
    },
    {
      "epoch": 1.4213625582726752,
      "grad_norm": 5.631358623504639,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24888750910758972,
      "logits/rejected": -0.4029492735862732,
      "logps/chosen": -552.5899047851562,
      "logps/rejected": -467.1000671386719,
      "loss": 0.2405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35639896988868713,
      "rewards/margins": 1.9249277114868164,
      "rewards/rejected": -2.2813267707824707,
      "step": 10862
    },
    {
      "epoch": 1.421493416210027,
      "grad_norm": 10.398524284362793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0015456825494766235,
      "logits/rejected": -0.23208369314670563,
      "logps/chosen": -351.5928955078125,
      "logps/rejected": -326.0367736816406,
      "loss": 0.7429,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7082706689834595,
      "rewards/margins": 0.8172483444213867,
      "rewards/rejected": -1.5255188941955566,
      "step": 10863
    },
    {
      "epoch": 1.4216242741473788,
      "grad_norm": 4.660824775695801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5935680866241455,
      "logits/rejected": -0.3900226354598999,
      "logps/chosen": -422.7286682128906,
      "logps/rejected": -507.40875244140625,
      "loss": 0.18,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5879274606704712,
      "rewards/margins": 2.506492853164673,
      "rewards/rejected": -3.0944201946258545,
      "step": 10864
    },
    {
      "epoch": 1.4217551320847306,
      "grad_norm": 11.328339576721191,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0523640513420105,
      "logits/rejected": 0.05358453840017319,
      "logps/chosen": -360.9717102050781,
      "logps/rejected": -301.54302978515625,
      "loss": 0.5903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4519760310649872,
      "rewards/margins": 0.7363073229789734,
      "rewards/rejected": -1.1882833242416382,
      "step": 10865
    },
    {
      "epoch": 1.4218859900220822,
      "grad_norm": 6.988304138183594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4648854732513428,
      "logits/rejected": -0.2581291198730469,
      "logps/chosen": -277.9628601074219,
      "logps/rejected": -330.2999267578125,
      "loss": 0.441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5322126150131226,
      "rewards/margins": 0.8692810535430908,
      "rewards/rejected": -1.4014936685562134,
      "step": 10866
    },
    {
      "epoch": 1.422016847959434,
      "grad_norm": 6.485910415649414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30091404914855957,
      "logits/rejected": -0.47463756799697876,
      "logps/chosen": -551.367919921875,
      "logps/rejected": -444.4990234375,
      "loss": 0.3455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3738204836845398,
      "rewards/margins": 1.3389244079589844,
      "rewards/rejected": -1.712744951248169,
      "step": 10867
    },
    {
      "epoch": 1.4221477058967857,
      "grad_norm": 8.505966186523438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4297512173652649,
      "logits/rejected": -0.39502522349357605,
      "logps/chosen": -399.50390625,
      "logps/rejected": -483.671142578125,
      "loss": 0.6148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8838144540786743,
      "rewards/margins": 0.6823221445083618,
      "rewards/rejected": -1.5661365985870361,
      "step": 10868
    },
    {
      "epoch": 1.4222785638341375,
      "grad_norm": 8.100119590759277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41103875637054443,
      "logits/rejected": -0.32361161708831787,
      "logps/chosen": -447.27520751953125,
      "logps/rejected": -426.76214599609375,
      "loss": 0.3291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0528719425201416,
      "rewards/margins": 1.6751818656921387,
      "rewards/rejected": -2.728053569793701,
      "step": 10869
    },
    {
      "epoch": 1.4224094217714893,
      "grad_norm": 8.990630149841309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3829197883605957,
      "logits/rejected": -0.22229290008544922,
      "logps/chosen": -374.10595703125,
      "logps/rejected": -509.27703857421875,
      "loss": 0.4316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5651472210884094,
      "rewards/margins": 2.431015729904175,
      "rewards/rejected": -2.9961628913879395,
      "step": 10870
    },
    {
      "epoch": 1.4225402797088411,
      "grad_norm": 9.322652816772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18904967606067657,
      "logits/rejected": -0.14001014828681946,
      "logps/chosen": -325.88653564453125,
      "logps/rejected": -469.9811096191406,
      "loss": 0.7462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3707966804504395,
      "rewards/margins": 1.0660125017166138,
      "rewards/rejected": -2.4368090629577637,
      "step": 10871
    },
    {
      "epoch": 1.422671137646193,
      "grad_norm": 5.875663757324219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44989943504333496,
      "logits/rejected": -0.36002570390701294,
      "logps/chosen": -507.3622131347656,
      "logps/rejected": -468.58203125,
      "loss": 0.2835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8911563158035278,
      "rewards/margins": 1.6103854179382324,
      "rewards/rejected": -2.5015416145324707,
      "step": 10872
    },
    {
      "epoch": 1.4228019955835447,
      "grad_norm": 5.149352550506592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14688684046268463,
      "logits/rejected": -0.2961861491203308,
      "logps/chosen": -724.8626098632812,
      "logps/rejected": -481.802978515625,
      "loss": 0.2549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04445004463195801,
      "rewards/margins": 1.6463885307312012,
      "rewards/rejected": -1.6908385753631592,
      "step": 10873
    },
    {
      "epoch": 1.4229328535208965,
      "grad_norm": 5.29992151260376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2579300105571747,
      "logits/rejected": -0.48262473940849304,
      "logps/chosen": -434.4273376464844,
      "logps/rejected": -358.8558044433594,
      "loss": 0.2854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.238181710243225,
      "rewards/margins": 2.0592384338378906,
      "rewards/rejected": -3.2974202632904053,
      "step": 10874
    },
    {
      "epoch": 1.423063711458248,
      "grad_norm": 13.850686073303223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3003876507282257,
      "logits/rejected": -0.02234598807990551,
      "logps/chosen": -463.2738037109375,
      "logps/rejected": -451.34814453125,
      "loss": 0.7034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1348283290863037,
      "rewards/margins": 0.4045483469963074,
      "rewards/rejected": -2.539376735687256,
      "step": 10875
    },
    {
      "epoch": 1.4231945693955999,
      "grad_norm": 11.037078857421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2707257866859436,
      "logits/rejected": -0.21585479378700256,
      "logps/chosen": -538.5548095703125,
      "logps/rejected": -549.96240234375,
      "loss": 0.5891,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3718034029006958,
      "rewards/margins": 1.0739202499389648,
      "rewards/rejected": -2.445723533630371,
      "step": 10876
    },
    {
      "epoch": 1.4233254273329516,
      "grad_norm": 7.236633777618408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05364130437374115,
      "logits/rejected": -0.3579350709915161,
      "logps/chosen": -466.1573181152344,
      "logps/rejected": -523.480224609375,
      "loss": 0.4154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1959117650985718,
      "rewards/margins": 2.1120197772979736,
      "rewards/rejected": -3.307931661605835,
      "step": 10877
    },
    {
      "epoch": 1.4234562852703034,
      "grad_norm": 7.888322353363037,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11597525328397751,
      "logits/rejected": -0.19915030896663666,
      "logps/chosen": -370.65924072265625,
      "logps/rejected": -457.4335021972656,
      "loss": 0.3395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9335506558418274,
      "rewards/margins": 2.3073017597198486,
      "rewards/rejected": -3.240852117538452,
      "step": 10878
    },
    {
      "epoch": 1.4235871432076552,
      "grad_norm": 7.742742538452148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2175389677286148,
      "logits/rejected": -0.22608506679534912,
      "logps/chosen": -592.5709228515625,
      "logps/rejected": -533.927001953125,
      "loss": 0.3276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5568331480026245,
      "rewards/margins": 1.4258133172988892,
      "rewards/rejected": -1.9826464653015137,
      "step": 10879
    },
    {
      "epoch": 1.423718001145007,
      "grad_norm": 6.493178844451904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.151057168841362,
      "logits/rejected": -0.1622716337442398,
      "logps/chosen": -485.3501281738281,
      "logps/rejected": -553.5700073242188,
      "loss": 0.4122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9591916799545288,
      "rewards/margins": 1.5132523775100708,
      "rewards/rejected": -2.4724440574645996,
      "step": 10880
    },
    {
      "epoch": 1.4238488590823586,
      "grad_norm": 8.961599349975586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3512197732925415,
      "logits/rejected": -0.4453701674938202,
      "logps/chosen": -381.9560546875,
      "logps/rejected": -471.3064270019531,
      "loss": 0.3914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1353927850723267,
      "rewards/margins": 1.1500656604766846,
      "rewards/rejected": -2.285458564758301,
      "step": 10881
    },
    {
      "epoch": 1.4239797170197104,
      "grad_norm": 7.996384620666504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33139723539352417,
      "logits/rejected": -0.6085171699523926,
      "logps/chosen": -450.12841796875,
      "logps/rejected": -552.5675659179688,
      "loss": 0.419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.199835181236267,
      "rewards/margins": 1.4169399738311768,
      "rewards/rejected": -2.6167752742767334,
      "step": 10882
    },
    {
      "epoch": 1.4241105749570622,
      "grad_norm": 7.048570156097412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018810123205184937,
      "logits/rejected": -0.15995213389396667,
      "logps/chosen": -352.8565979003906,
      "logps/rejected": -370.2405090332031,
      "loss": 0.4417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9438022971153259,
      "rewards/margins": 1.1002126932144165,
      "rewards/rejected": -2.0440149307250977,
      "step": 10883
    },
    {
      "epoch": 1.424241432894414,
      "grad_norm": 7.3831682205200195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030567530542612076,
      "logits/rejected": -0.09541677683591843,
      "logps/chosen": -516.1485595703125,
      "logps/rejected": -418.6768798828125,
      "loss": 0.636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0543800592422485,
      "rewards/margins": 0.8907150030136108,
      "rewards/rejected": -1.9450950622558594,
      "step": 10884
    },
    {
      "epoch": 1.4243722908317658,
      "grad_norm": 10.630773544311523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10734470188617706,
      "logits/rejected": 0.16049796342849731,
      "logps/chosen": -479.06658935546875,
      "logps/rejected": -406.5375671386719,
      "loss": 0.4758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0357024669647217,
      "rewards/margins": 1.5437755584716797,
      "rewards/rejected": -2.5794780254364014,
      "step": 10885
    },
    {
      "epoch": 1.4245031487691175,
      "grad_norm": 7.781508445739746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18237654864788055,
      "logits/rejected": -0.46747803688049316,
      "logps/chosen": -683.1300659179688,
      "logps/rejected": -451.502197265625,
      "loss": 0.3666,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7474654912948608,
      "rewards/margins": 1.4954241514205933,
      "rewards/rejected": -2.242889881134033,
      "step": 10886
    },
    {
      "epoch": 1.4246340067064693,
      "grad_norm": 10.777608871459961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19515292346477509,
      "logits/rejected": -0.1192958801984787,
      "logps/chosen": -439.779541015625,
      "logps/rejected": -575.211181640625,
      "loss": 0.7645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.242126226425171,
      "rewards/margins": 0.7066783905029297,
      "rewards/rejected": -1.948804497718811,
      "step": 10887
    },
    {
      "epoch": 1.4247648646438211,
      "grad_norm": 6.787720680236816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16912716627120972,
      "logits/rejected": -0.22300170361995697,
      "logps/chosen": -445.60650634765625,
      "logps/rejected": -375.3319091796875,
      "loss": 0.4177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06888123601675034,
      "rewards/margins": 1.3696633577346802,
      "rewards/rejected": -1.438544750213623,
      "step": 10888
    },
    {
      "epoch": 1.424895722581173,
      "grad_norm": 6.878329277038574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15024930238723755,
      "logits/rejected": -0.06814377009868622,
      "logps/chosen": -399.28277587890625,
      "logps/rejected": -389.38458251953125,
      "loss": 0.3071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7677774429321289,
      "rewards/margins": 1.168276071548462,
      "rewards/rejected": -1.9360535144805908,
      "step": 10889
    },
    {
      "epoch": 1.4250265805185247,
      "grad_norm": 8.243024826049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21863242983818054,
      "logits/rejected": -0.148832768201828,
      "logps/chosen": -508.0099182128906,
      "logps/rejected": -435.2181396484375,
      "loss": 0.6404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8206115961074829,
      "rewards/margins": 1.1474926471710205,
      "rewards/rejected": -1.9681041240692139,
      "step": 10890
    },
    {
      "epoch": 1.4251574384558763,
      "grad_norm": 8.216629981994629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19999252259731293,
      "logits/rejected": -0.2128869593143463,
      "logps/chosen": -661.3483276367188,
      "logps/rejected": -569.3118896484375,
      "loss": 0.3355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8839912414550781,
      "rewards/margins": 2.113598108291626,
      "rewards/rejected": -2.997589111328125,
      "step": 10891
    },
    {
      "epoch": 1.425288296393228,
      "grad_norm": 7.431231498718262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018328148871660233,
      "logits/rejected": -0.30098390579223633,
      "logps/chosen": -484.9641418457031,
      "logps/rejected": -493.3721923828125,
      "loss": 0.3635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.337851345539093,
      "rewards/margins": 2.270181179046631,
      "rewards/rejected": -2.608032464981079,
      "step": 10892
    },
    {
      "epoch": 1.4254191543305799,
      "grad_norm": 4.861148357391357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3235410153865814,
      "logits/rejected": -0.19390366971492767,
      "logps/chosen": -491.6292724609375,
      "logps/rejected": -547.8383178710938,
      "loss": 0.2113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5051666498184204,
      "rewards/margins": 1.7907658815383911,
      "rewards/rejected": -2.2959322929382324,
      "step": 10893
    },
    {
      "epoch": 1.4255500122679317,
      "grad_norm": 5.275454998016357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034952208399772644,
      "logits/rejected": -0.08397159725427628,
      "logps/chosen": -796.76220703125,
      "logps/rejected": -530.2260131835938,
      "loss": 0.1906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48767438530921936,
      "rewards/margins": 1.8612074851989746,
      "rewards/rejected": -2.348881959915161,
      "step": 10894
    },
    {
      "epoch": 1.4256808702052834,
      "grad_norm": 14.825043678283691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16902780532836914,
      "logits/rejected": -0.26535382866859436,
      "logps/chosen": -416.32440185546875,
      "logps/rejected": -466.4814453125,
      "loss": 0.731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.905893087387085,
      "rewards/margins": 0.8622057437896729,
      "rewards/rejected": -2.7680985927581787,
      "step": 10895
    },
    {
      "epoch": 1.425811728142635,
      "grad_norm": 8.505637168884277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009099002927541733,
      "logits/rejected": -0.016647610813379288,
      "logps/chosen": -395.1069641113281,
      "logps/rejected": -479.8870849609375,
      "loss": 0.5581,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2111576497554779,
      "rewards/margins": 1.3179497718811035,
      "rewards/rejected": -1.5291074514389038,
      "step": 10896
    },
    {
      "epoch": 1.4259425860799868,
      "grad_norm": 10.18709945678711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3602769076824188,
      "logits/rejected": -0.361279159784317,
      "logps/chosen": -377.91656494140625,
      "logps/rejected": -364.9844665527344,
      "loss": 0.7116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3116220235824585,
      "rewards/margins": 0.22622616589069366,
      "rewards/rejected": -1.5378482341766357,
      "step": 10897
    },
    {
      "epoch": 1.4260734440173386,
      "grad_norm": 7.738738536834717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08115403354167938,
      "logits/rejected": -0.03249968960881233,
      "logps/chosen": -548.9180908203125,
      "logps/rejected": -537.8211059570312,
      "loss": 0.4541,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1644665002822876,
      "rewards/margins": 0.8940815329551697,
      "rewards/rejected": -2.0585479736328125,
      "step": 10898
    },
    {
      "epoch": 1.4262043019546904,
      "grad_norm": 4.657738208770752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20173978805541992,
      "logits/rejected": -0.2059728503227234,
      "logps/chosen": -335.28814697265625,
      "logps/rejected": -379.12359619140625,
      "loss": 0.2253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7542647123336792,
      "rewards/margins": 2.010573625564575,
      "rewards/rejected": -3.764838457107544,
      "step": 10899
    },
    {
      "epoch": 1.4263351598920422,
      "grad_norm": 9.255921363830566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04629615694284439,
      "logits/rejected": -0.0667489767074585,
      "logps/chosen": -801.46484375,
      "logps/rejected": -599.329345703125,
      "loss": 0.4705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47704753279685974,
      "rewards/margins": 1.1359272003173828,
      "rewards/rejected": -1.612974762916565,
      "step": 10900
    },
    {
      "epoch": 1.426466017829394,
      "grad_norm": 9.850668907165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1781243234872818,
      "logits/rejected": -0.27587005496025085,
      "logps/chosen": -509.72979736328125,
      "logps/rejected": -618.096923828125,
      "loss": 0.5693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7299830317497253,
      "rewards/margins": 1.7008168697357178,
      "rewards/rejected": -2.430799961090088,
      "step": 10901
    },
    {
      "epoch": 1.4265968757667458,
      "grad_norm": 5.8964457511901855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38277626037597656,
      "logits/rejected": -0.21576504409313202,
      "logps/chosen": -656.37255859375,
      "logps/rejected": -459.5325012207031,
      "loss": 0.2563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8445488214492798,
      "rewards/margins": 1.5777885913848877,
      "rewards/rejected": -2.422337293624878,
      "step": 10902
    },
    {
      "epoch": 1.4267277337040976,
      "grad_norm": 12.381157875061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04808850586414337,
      "logits/rejected": -0.30527132749557495,
      "logps/chosen": -384.08624267578125,
      "logps/rejected": -255.5495147705078,
      "loss": 0.8925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.1786396503448486,
      "rewards/margins": 0.29777151346206665,
      "rewards/rejected": -2.4764113426208496,
      "step": 10903
    },
    {
      "epoch": 1.4268585916414493,
      "grad_norm": 5.500842094421387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16315466165542603,
      "logits/rejected": -0.473289430141449,
      "logps/chosen": -682.2763671875,
      "logps/rejected": -571.1753540039062,
      "loss": 0.1985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3376884460449219,
      "rewards/margins": 2.111285448074341,
      "rewards/rejected": -2.4489736557006836,
      "step": 10904
    },
    {
      "epoch": 1.4269894495788011,
      "grad_norm": 11.7913236618042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19630056619644165,
      "logits/rejected": -0.3823370933532715,
      "logps/chosen": -651.10791015625,
      "logps/rejected": -547.8319702148438,
      "loss": 0.4412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7517025470733643,
      "rewards/margins": 1.228269338607788,
      "rewards/rejected": -1.9799718856811523,
      "step": 10905
    },
    {
      "epoch": 1.4271203075161527,
      "grad_norm": 8.60498046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4924553632736206,
      "logits/rejected": -0.19384492933750153,
      "logps/chosen": -419.0993347167969,
      "logps/rejected": -476.8269348144531,
      "loss": 0.5322,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3887144327163696,
      "rewards/margins": 1.2529832124710083,
      "rewards/rejected": -2.641697645187378,
      "step": 10906
    },
    {
      "epoch": 1.4272511654535045,
      "grad_norm": 12.214308738708496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09614840149879456,
      "logits/rejected": -0.26441434025764465,
      "logps/chosen": -558.7149047851562,
      "logps/rejected": -420.732421875,
      "loss": 0.7172,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7830849885940552,
      "rewards/margins": 0.573717474937439,
      "rewards/rejected": -1.3568025827407837,
      "step": 10907
    },
    {
      "epoch": 1.4273820233908563,
      "grad_norm": 9.667813301086426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06614547222852707,
      "logits/rejected": -0.21909254789352417,
      "logps/chosen": -456.2501220703125,
      "logps/rejected": -451.4570617675781,
      "loss": 0.5635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3340400457382202,
      "rewards/margins": 0.4489917755126953,
      "rewards/rejected": -1.783031940460205,
      "step": 10908
    },
    {
      "epoch": 1.427512881328208,
      "grad_norm": 6.7190985679626465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08109268546104431,
      "logits/rejected": -0.07765187323093414,
      "logps/chosen": -421.8872375488281,
      "logps/rejected": -436.8692626953125,
      "loss": 0.4869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7061140537261963,
      "rewards/margins": 0.9088268280029297,
      "rewards/rejected": -1.6149407625198364,
      "step": 10909
    },
    {
      "epoch": 1.4276437392655599,
      "grad_norm": 5.676374912261963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2256765067577362,
      "logits/rejected": -0.22772301733493805,
      "logps/chosen": -529.4835205078125,
      "logps/rejected": -417.2372131347656,
      "loss": 0.3341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.081903338432312,
      "rewards/margins": 1.1088379621505737,
      "rewards/rejected": -2.1907413005828857,
      "step": 10910
    },
    {
      "epoch": 1.4277745972029117,
      "grad_norm": 6.519522666931152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12577269971370697,
      "logits/rejected": 0.04127628356218338,
      "logps/chosen": -542.8124389648438,
      "logps/rejected": -416.2303771972656,
      "loss": 0.3431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.012665033340454,
      "rewards/margins": 1.962506890296936,
      "rewards/rejected": -2.975172281265259,
      "step": 10911
    },
    {
      "epoch": 1.4279054551402632,
      "grad_norm": 7.9031548500061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2315244972705841,
      "logits/rejected": -0.027224775403738022,
      "logps/chosen": -412.7430725097656,
      "logps/rejected": -379.4794921875,
      "loss": 0.3698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5058189630508423,
      "rewards/margins": 1.806323766708374,
      "rewards/rejected": -2.312142848968506,
      "step": 10912
    },
    {
      "epoch": 1.428036313077615,
      "grad_norm": 11.365900993347168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06382327526807785,
      "logits/rejected": 0.05194424092769623,
      "logps/chosen": -521.5535278320312,
      "logps/rejected": -645.9667358398438,
      "loss": 0.8178,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.6547327041625977,
      "rewards/margins": -0.044052496552467346,
      "rewards/rejected": -1.6106802225112915,
      "step": 10913
    },
    {
      "epoch": 1.4281671710149668,
      "grad_norm": 9.333065032958984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24309122562408447,
      "logits/rejected": -0.3106956481933594,
      "logps/chosen": -436.9122314453125,
      "logps/rejected": -567.7808227539062,
      "loss": 0.4728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.201538324356079,
      "rewards/margins": 0.917969822883606,
      "rewards/rejected": -2.1195082664489746,
      "step": 10914
    },
    {
      "epoch": 1.4282980289523186,
      "grad_norm": 7.641905307769775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5016763210296631,
      "logits/rejected": -0.44035249948501587,
      "logps/chosen": -352.5743713378906,
      "logps/rejected": -345.5810546875,
      "loss": 0.4365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.972071647644043,
      "rewards/margins": 0.8552771210670471,
      "rewards/rejected": -1.8273487091064453,
      "step": 10915
    },
    {
      "epoch": 1.4284288868896704,
      "grad_norm": 5.9250640869140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010489724576473236,
      "logits/rejected": -0.1647464632987976,
      "logps/chosen": -657.00341796875,
      "logps/rejected": -388.80029296875,
      "loss": 0.3174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6011303067207336,
      "rewards/margins": 1.9280304908752441,
      "rewards/rejected": -2.529160737991333,
      "step": 10916
    },
    {
      "epoch": 1.4285597448270222,
      "grad_norm": 3.5041377544403076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2415575236082077,
      "logits/rejected": -0.21704000234603882,
      "logps/chosen": -389.31976318359375,
      "logps/rejected": -329.1983337402344,
      "loss": 0.1674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9776451587677002,
      "rewards/margins": 2.0564208030700684,
      "rewards/rejected": -3.0340659618377686,
      "step": 10917
    },
    {
      "epoch": 1.428690602764374,
      "grad_norm": 8.984914779663086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23385637998580933,
      "logits/rejected": -0.17219969630241394,
      "logps/chosen": -437.355224609375,
      "logps/rejected": -498.7743225097656,
      "loss": 0.7046,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5554518103599548,
      "rewards/margins": 0.4446357786655426,
      "rewards/rejected": -1.0000876188278198,
      "step": 10918
    },
    {
      "epoch": 1.4288214607017258,
      "grad_norm": 5.941041469573975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02414090186357498,
      "logits/rejected": -0.47293388843536377,
      "logps/chosen": -350.1776123046875,
      "logps/rejected": -363.09454345703125,
      "loss": 0.3662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2973002195358276,
      "rewards/margins": 1.9240608215332031,
      "rewards/rejected": -3.221360921859741,
      "step": 10919
    },
    {
      "epoch": 1.4289523186390776,
      "grad_norm": 14.527227401733398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1484399139881134,
      "logits/rejected": -0.28418633341789246,
      "logps/chosen": -585.6342163085938,
      "logps/rejected": -438.1372375488281,
      "loss": 0.9535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2338117361068726,
      "rewards/margins": 1.6105544567108154,
      "rewards/rejected": -2.8443660736083984,
      "step": 10920
    },
    {
      "epoch": 1.4290831765764291,
      "grad_norm": 5.6124267578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08133485913276672,
      "logits/rejected": -0.2956961989402771,
      "logps/chosen": -400.97479248046875,
      "logps/rejected": -391.3165283203125,
      "loss": 0.3024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3678414821624756,
      "rewards/margins": 1.7672550678253174,
      "rewards/rejected": -3.135096549987793,
      "step": 10921
    },
    {
      "epoch": 1.429214034513781,
      "grad_norm": 11.471075057983398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3081572353839874,
      "logits/rejected": -0.5399911403656006,
      "logps/chosen": -426.70098876953125,
      "logps/rejected": -583.9781494140625,
      "loss": 0.5164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8728704452514648,
      "rewards/margins": 0.6640356183052063,
      "rewards/rejected": -2.5369060039520264,
      "step": 10922
    },
    {
      "epoch": 1.4293448924511327,
      "grad_norm": 6.313479900360107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18603993952274323,
      "logits/rejected": -0.19815762341022491,
      "logps/chosen": -639.5692749023438,
      "logps/rejected": -600.1531982421875,
      "loss": 0.3235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21684035658836365,
      "rewards/margins": 1.5021240711212158,
      "rewards/rejected": -1.7189643383026123,
      "step": 10923
    },
    {
      "epoch": 1.4294757503884845,
      "grad_norm": 6.729891300201416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.512430727481842,
      "logits/rejected": -0.33719268441200256,
      "logps/chosen": -402.0606384277344,
      "logps/rejected": -503.9288330078125,
      "loss": 0.3509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1916425228118896,
      "rewards/margins": 1.289747953414917,
      "rewards/rejected": -2.4813904762268066,
      "step": 10924
    },
    {
      "epoch": 1.4296066083258363,
      "grad_norm": 7.987916469573975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1526380479335785,
      "logits/rejected": -0.20079246163368225,
      "logps/chosen": -419.9967041015625,
      "logps/rejected": -384.3066711425781,
      "loss": 0.4792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3162996768951416,
      "rewards/margins": 0.7455263137817383,
      "rewards/rejected": -2.06182599067688,
      "step": 10925
    },
    {
      "epoch": 1.429737466263188,
      "grad_norm": 6.2390265464782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12252695113420486,
      "logits/rejected": -0.2677193284034729,
      "logps/chosen": -461.8608703613281,
      "logps/rejected": -435.199462890625,
      "loss": 0.4161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20861853659152985,
      "rewards/margins": 1.5443642139434814,
      "rewards/rejected": -1.7529828548431396,
      "step": 10926
    },
    {
      "epoch": 1.4298683242005397,
      "grad_norm": 4.431492328643799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09210366755723953,
      "logits/rejected": -0.032261207699775696,
      "logps/chosen": -610.550537109375,
      "logps/rejected": -535.6788330078125,
      "loss": 0.1979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2010570764541626,
      "rewards/margins": 2.2023279666900635,
      "rewards/rejected": -2.4033849239349365,
      "step": 10927
    },
    {
      "epoch": 1.4299991821378915,
      "grad_norm": 4.471195220947266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2583976089954376,
      "logits/rejected": -0.33336013555526733,
      "logps/chosen": -479.3774108886719,
      "logps/rejected": -520.9905395507812,
      "loss": 0.231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5110084414482117,
      "rewards/margins": 1.994738221168518,
      "rewards/rejected": -2.505746603012085,
      "step": 10928
    },
    {
      "epoch": 1.4301300400752432,
      "grad_norm": 4.650365829467773,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20318728685379028,
      "logits/rejected": -0.24731312692165375,
      "logps/chosen": -537.2776489257812,
      "logps/rejected": -484.0163269042969,
      "loss": 0.1657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5767552852630615,
      "rewards/margins": 1.9924633502960205,
      "rewards/rejected": -2.569218635559082,
      "step": 10929
    },
    {
      "epoch": 1.430260898012595,
      "grad_norm": 13.968354225158691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15074944496154785,
      "logits/rejected": 0.059351883828639984,
      "logps/chosen": -416.7243347167969,
      "logps/rejected": -453.89544677734375,
      "loss": 0.7748,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2665019035339355,
      "rewards/margins": 0.5599806308746338,
      "rewards/rejected": -1.8264822959899902,
      "step": 10930
    },
    {
      "epoch": 1.4303917559499468,
      "grad_norm": 8.682133674621582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044304996728897095,
      "logits/rejected": -0.15280526876449585,
      "logps/chosen": -403.2237548828125,
      "logps/rejected": -393.2762145996094,
      "loss": 0.5714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8050240278244019,
      "rewards/margins": 0.44995254278182983,
      "rewards/rejected": -1.2549763917922974,
      "step": 10931
    },
    {
      "epoch": 1.4305226138872986,
      "grad_norm": 7.247363567352295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11201432347297668,
      "logits/rejected": -0.2819245755672455,
      "logps/chosen": -486.53314208984375,
      "logps/rejected": -610.811767578125,
      "loss": 0.4543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0382165908813477,
      "rewards/margins": 1.1739492416381836,
      "rewards/rejected": -2.212165594100952,
      "step": 10932
    },
    {
      "epoch": 1.4306534718246504,
      "grad_norm": 13.182430267333984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06506363302469254,
      "logits/rejected": -0.24147912859916687,
      "logps/chosen": -652.738525390625,
      "logps/rejected": -503.8091125488281,
      "loss": 0.771,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5819873809814453,
      "rewards/margins": 0.8014501333236694,
      "rewards/rejected": -1.3834375143051147,
      "step": 10933
    },
    {
      "epoch": 1.4307843297620022,
      "grad_norm": 7.795450687408447,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09279093146324158,
      "logits/rejected": -0.14048004150390625,
      "logps/chosen": -476.3282470703125,
      "logps/rejected": -485.03424072265625,
      "loss": 0.4592,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9914560317993164,
      "rewards/margins": 1.1043992042541504,
      "rewards/rejected": -2.0958549976348877,
      "step": 10934
    },
    {
      "epoch": 1.430915187699354,
      "grad_norm": 8.469143867492676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04733097553253174,
      "logits/rejected": -0.13081029057502747,
      "logps/chosen": -414.95452880859375,
      "logps/rejected": -526.615234375,
      "loss": 0.4355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38312721252441406,
      "rewards/margins": 1.201899766921997,
      "rewards/rejected": -1.5850269794464111,
      "step": 10935
    },
    {
      "epoch": 1.4310460456367058,
      "grad_norm": 12.290767669677734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44596490263938904,
      "logits/rejected": -0.33855557441711426,
      "logps/chosen": -491.2952575683594,
      "logps/rejected": -621.100341796875,
      "loss": 0.6859,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6608626842498779,
      "rewards/margins": 0.520878791809082,
      "rewards/rejected": -1.1817415952682495,
      "step": 10936
    },
    {
      "epoch": 1.4311769035740574,
      "grad_norm": 5.8697919845581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3490908145904541,
      "logits/rejected": -0.31374338269233704,
      "logps/chosen": -396.14691162109375,
      "logps/rejected": -382.2288513183594,
      "loss": 0.2887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7613396644592285,
      "rewards/margins": 1.7070655822753906,
      "rewards/rejected": -2.468405246734619,
      "step": 10937
    },
    {
      "epoch": 1.4313077615114091,
      "grad_norm": 4.463902473449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00944141112267971,
      "logits/rejected": -0.238719642162323,
      "logps/chosen": -688.1328125,
      "logps/rejected": -556.2572021484375,
      "loss": 0.1851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6318166851997375,
      "rewards/margins": 2.138744592666626,
      "rewards/rejected": -1.5069279670715332,
      "step": 10938
    },
    {
      "epoch": 1.431438619448761,
      "grad_norm": 10.968546867370605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07004693150520325,
      "logits/rejected": -0.09745226800441742,
      "logps/chosen": -640.310791015625,
      "logps/rejected": -546.2017822265625,
      "loss": 0.6212,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0733805894851685,
      "rewards/margins": 0.6290459036827087,
      "rewards/rejected": -1.702426552772522,
      "step": 10939
    },
    {
      "epoch": 1.4315694773861127,
      "grad_norm": 6.020103931427002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1329878270626068,
      "logits/rejected": -0.40278297662734985,
      "logps/chosen": -368.2373046875,
      "logps/rejected": -376.08502197265625,
      "loss": 0.3719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.863125205039978,
      "rewards/margins": 1.442644476890564,
      "rewards/rejected": -3.305769443511963,
      "step": 10940
    },
    {
      "epoch": 1.4317003353234645,
      "grad_norm": 9.638602256774902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023568904027342796,
      "logits/rejected": -0.07131128013134003,
      "logps/chosen": -449.6363220214844,
      "logps/rejected": -563.91064453125,
      "loss": 0.5764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1819711923599243,
      "rewards/margins": 1.3054462671279907,
      "rewards/rejected": -2.487417459487915,
      "step": 10941
    },
    {
      "epoch": 1.431831193260816,
      "grad_norm": 9.623889923095703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36422237753868103,
      "logits/rejected": -0.40097948908805847,
      "logps/chosen": -361.3111572265625,
      "logps/rejected": -371.87274169921875,
      "loss": 0.5462,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3919036388397217,
      "rewards/margins": 0.8573943376541138,
      "rewards/rejected": -3.249297618865967,
      "step": 10942
    },
    {
      "epoch": 1.4319620511981679,
      "grad_norm": 4.988470554351807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13012801110744476,
      "logits/rejected": -0.08308938145637512,
      "logps/chosen": -390.3596496582031,
      "logps/rejected": -454.9092712402344,
      "loss": 0.2639,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8194568753242493,
      "rewards/margins": 1.6585553884506226,
      "rewards/rejected": -2.4780123233795166,
      "step": 10943
    },
    {
      "epoch": 1.4320929091355197,
      "grad_norm": 7.406429290771484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24073487520217896,
      "logits/rejected": -0.3330167233943939,
      "logps/chosen": -346.9117126464844,
      "logps/rejected": -554.8258056640625,
      "loss": 0.4147,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5034524202346802,
      "rewards/margins": 1.084916591644287,
      "rewards/rejected": -2.588369131088257,
      "step": 10944
    },
    {
      "epoch": 1.4322237670728715,
      "grad_norm": 10.577705383300781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3076189160346985,
      "logits/rejected": -0.33255207538604736,
      "logps/chosen": -495.257568359375,
      "logps/rejected": -563.1326904296875,
      "loss": 0.5948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.443091869354248,
      "rewards/margins": 0.9030780792236328,
      "rewards/rejected": -2.346169948577881,
      "step": 10945
    },
    {
      "epoch": 1.4323546250102233,
      "grad_norm": 3.520721197128296,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3982298970222473,
      "logits/rejected": -0.4814896583557129,
      "logps/chosen": -785.957763671875,
      "logps/rejected": -495.9537353515625,
      "loss": 0.2221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43240588903427124,
      "rewards/margins": 2.103881597518921,
      "rewards/rejected": -2.536287307739258,
      "step": 10946
    },
    {
      "epoch": 1.432485482947575,
      "grad_norm": 7.2822065353393555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.096529521048069,
      "logits/rejected": -0.09590274095535278,
      "logps/chosen": -433.7564392089844,
      "logps/rejected": -347.1058349609375,
      "loss": 0.3772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8602048754692078,
      "rewards/margins": 1.4851112365722656,
      "rewards/rejected": -2.345315933227539,
      "step": 10947
    },
    {
      "epoch": 1.4326163408849268,
      "grad_norm": 8.009839057922363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20039427280426025,
      "logits/rejected": -0.2571379244327545,
      "logps/chosen": -515.0977172851562,
      "logps/rejected": -519.13037109375,
      "loss": 0.4131,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11976791173219681,
      "rewards/margins": 1.2439507246017456,
      "rewards/rejected": -1.1241825819015503,
      "step": 10948
    },
    {
      "epoch": 1.4327471988222786,
      "grad_norm": 8.665292739868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.049668580293655396,
      "logits/rejected": -0.11449006199836731,
      "logps/chosen": -410.4063720703125,
      "logps/rejected": -377.2117919921875,
      "loss": 0.2884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5057499408721924,
      "rewards/margins": 1.2750895023345947,
      "rewards/rejected": -2.780839443206787,
      "step": 10949
    },
    {
      "epoch": 1.4328780567596304,
      "grad_norm": 8.615781784057617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17470496892929077,
      "logits/rejected": -0.13107359409332275,
      "logps/chosen": -564.0496826171875,
      "logps/rejected": -565.3428955078125,
      "loss": 0.4212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4921192228794098,
      "rewards/margins": 1.1493217945098877,
      "rewards/rejected": -1.641440987586975,
      "step": 10950
    },
    {
      "epoch": 1.4330089146969822,
      "grad_norm": 11.87716007232666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05047604441642761,
      "logits/rejected": -0.08228858560323715,
      "logps/chosen": -439.6544189453125,
      "logps/rejected": -402.22247314453125,
      "loss": 0.613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2153048515319824,
      "rewards/margins": 0.794658899307251,
      "rewards/rejected": -2.0099635124206543,
      "step": 10951
    },
    {
      "epoch": 1.4331397726343338,
      "grad_norm": 7.774279594421387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12206781655550003,
      "logits/rejected": -0.26820898056030273,
      "logps/chosen": -438.1490478515625,
      "logps/rejected": -423.8643493652344,
      "loss": 0.4533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.433920979499817,
      "rewards/margins": 1.3691580295562744,
      "rewards/rejected": -2.803079128265381,
      "step": 10952
    },
    {
      "epoch": 1.4332706305716856,
      "grad_norm": 10.783251762390137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0328327938914299,
      "logits/rejected": -0.30894753336906433,
      "logps/chosen": -578.4956665039062,
      "logps/rejected": -481.9361572265625,
      "loss": 0.5472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1041603088378906,
      "rewards/margins": 1.6621434688568115,
      "rewards/rejected": -2.766303777694702,
      "step": 10953
    },
    {
      "epoch": 1.4334014885090374,
      "grad_norm": 8.168212890625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03402315080165863,
      "logits/rejected": -0.12248137593269348,
      "logps/chosen": -522.363037109375,
      "logps/rejected": -438.62298583984375,
      "loss": 0.3581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06054846942424774,
      "rewards/margins": 1.5735983848571777,
      "rewards/rejected": -1.634147047996521,
      "step": 10954
    },
    {
      "epoch": 1.4335323464463892,
      "grad_norm": 8.7533540725708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23425914347171783,
      "logits/rejected": -0.22362735867500305,
      "logps/chosen": -660.6072998046875,
      "logps/rejected": -610.4953002929688,
      "loss": 0.4193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.548549234867096,
      "rewards/margins": 1.4339542388916016,
      "rewards/rejected": -1.9825034141540527,
      "step": 10955
    },
    {
      "epoch": 1.433663204383741,
      "grad_norm": 5.2673187255859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17329935729503632,
      "logits/rejected": -0.06117958202958107,
      "logps/chosen": -441.1568603515625,
      "logps/rejected": -551.3238525390625,
      "loss": 0.298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7803611755371094,
      "rewards/margins": 1.4740833044052124,
      "rewards/rejected": -3.2544445991516113,
      "step": 10956
    },
    {
      "epoch": 1.4337940623210927,
      "grad_norm": 6.490560531616211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10200752317905426,
      "logits/rejected": -0.23700910806655884,
      "logps/chosen": -523.592529296875,
      "logps/rejected": -509.1897277832031,
      "loss": 0.3407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8388856649398804,
      "rewards/margins": 1.3793529272079468,
      "rewards/rejected": -2.218238592147827,
      "step": 10957
    },
    {
      "epoch": 1.4339249202584443,
      "grad_norm": 8.201164245605469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2900703251361847,
      "logits/rejected": -0.30875420570373535,
      "logps/chosen": -345.26422119140625,
      "logps/rejected": -408.66796875,
      "loss": 0.4699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.475642442703247,
      "rewards/margins": 1.0239129066467285,
      "rewards/rejected": -2.4995551109313965,
      "step": 10958
    },
    {
      "epoch": 1.434055778195796,
      "grad_norm": 8.999807357788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15941321849822998,
      "logits/rejected": -0.5118778944015503,
      "logps/chosen": -496.0810852050781,
      "logps/rejected": -408.1003723144531,
      "loss": 0.4926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2904781103134155,
      "rewards/margins": 1.7561290264129639,
      "rewards/rejected": -3.04660701751709,
      "step": 10959
    },
    {
      "epoch": 1.434186636133148,
      "grad_norm": 6.880688667297363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3864184021949768,
      "logits/rejected": -0.3925315737724304,
      "logps/chosen": -525.498779296875,
      "logps/rejected": -602.3976440429688,
      "loss": 0.3619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6988599896430969,
      "rewards/margins": 1.4465036392211914,
      "rewards/rejected": -2.1453635692596436,
      "step": 10960
    },
    {
      "epoch": 1.4343174940704997,
      "grad_norm": 6.490509033203125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03634677827358246,
      "logits/rejected": -0.19738447666168213,
      "logps/chosen": -481.47564697265625,
      "logps/rejected": -423.940673828125,
      "loss": 0.3493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5203754901885986,
      "rewards/margins": 2.1265997886657715,
      "rewards/rejected": -2.64697527885437,
      "step": 10961
    },
    {
      "epoch": 1.4344483520078515,
      "grad_norm": 11.284810066223145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07787168771028519,
      "logits/rejected": -0.11810626834630966,
      "logps/chosen": -938.9242553710938,
      "logps/rejected": -921.771484375,
      "loss": 0.6649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.326069951057434,
      "rewards/margins": 0.7527433633804321,
      "rewards/rejected": -2.0788135528564453,
      "step": 10962
    },
    {
      "epoch": 1.4345792099452033,
      "grad_norm": 8.350937843322754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47746142745018005,
      "logits/rejected": -0.572439432144165,
      "logps/chosen": -373.10906982421875,
      "logps/rejected": -473.1549377441406,
      "loss": 0.3549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.061591863632202,
      "rewards/margins": 1.8081568479537964,
      "rewards/rejected": -3.869748592376709,
      "step": 10963
    },
    {
      "epoch": 1.434710067882555,
      "grad_norm": 11.609940528869629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19615225493907928,
      "logits/rejected": -0.2852911055088043,
      "logps/chosen": -712.6898193359375,
      "logps/rejected": -561.276123046875,
      "loss": 0.6327,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32185250520706177,
      "rewards/margins": 1.6184418201446533,
      "rewards/rejected": -1.9402941465377808,
      "step": 10964
    },
    {
      "epoch": 1.4348409258199069,
      "grad_norm": 6.179343223571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060921262949705124,
      "logits/rejected": -0.1685863733291626,
      "logps/chosen": -329.6778869628906,
      "logps/rejected": -661.458251953125,
      "loss": 0.2324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4018205404281616,
      "rewards/margins": 1.850213646888733,
      "rewards/rejected": -3.2520339488983154,
      "step": 10965
    },
    {
      "epoch": 1.4349717837572586,
      "grad_norm": 6.664234161376953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33957555890083313,
      "logits/rejected": -0.23123303055763245,
      "logps/chosen": -381.914306640625,
      "logps/rejected": -405.37738037109375,
      "loss": 0.2851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1589691638946533,
      "rewards/margins": 1.6257189512252808,
      "rewards/rejected": -2.7846877574920654,
      "step": 10966
    },
    {
      "epoch": 1.4351026416946102,
      "grad_norm": 10.165698051452637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14509405195713043,
      "logits/rejected": -0.10419205576181412,
      "logps/chosen": -455.00543212890625,
      "logps/rejected": -525.0073852539062,
      "loss": 0.6987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7718567848205566,
      "rewards/margins": 0.3708110451698303,
      "rewards/rejected": -2.142667770385742,
      "step": 10967
    },
    {
      "epoch": 1.435233499631962,
      "grad_norm": 16.071372985839844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016037840396165848,
      "logits/rejected": -0.11363497376441956,
      "logps/chosen": -437.99981689453125,
      "logps/rejected": -522.0480346679688,
      "loss": 1.1556,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6455960273742676,
      "rewards/margins": -0.0037807226181030273,
      "rewards/rejected": -1.641815185546875,
      "step": 10968
    },
    {
      "epoch": 1.4353643575693138,
      "grad_norm": 6.6948018074035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24101832509040833,
      "logits/rejected": -0.3679914176464081,
      "logps/chosen": -693.2821655273438,
      "logps/rejected": -710.3195190429688,
      "loss": 0.2911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4307344853878021,
      "rewards/margins": 1.6416717767715454,
      "rewards/rejected": -2.07240629196167,
      "step": 10969
    },
    {
      "epoch": 1.4354952155066656,
      "grad_norm": 6.839423656463623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3782411813735962,
      "logits/rejected": -0.5296960473060608,
      "logps/chosen": -416.98028564453125,
      "logps/rejected": -368.2862243652344,
      "loss": 0.3715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7775686979293823,
      "rewards/margins": 1.161150574684143,
      "rewards/rejected": -1.9387192726135254,
      "step": 10970
    },
    {
      "epoch": 1.4356260734440174,
      "grad_norm": 7.0926642417907715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04883525148034096,
      "logits/rejected": -0.10119936615228653,
      "logps/chosen": -452.8961181640625,
      "logps/rejected": -434.570068359375,
      "loss": 0.4006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2951351404190063,
      "rewards/margins": 1.1444084644317627,
      "rewards/rejected": -2.4395437240600586,
      "step": 10971
    },
    {
      "epoch": 1.4357569313813692,
      "grad_norm": 7.007379531860352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3594527840614319,
      "logits/rejected": -0.2148779332637787,
      "logps/chosen": -477.672119140625,
      "logps/rejected": -634.2180786132812,
      "loss": 0.3426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7038422226905823,
      "rewards/margins": 1.7988651990890503,
      "rewards/rejected": -2.5027074813842773,
      "step": 10972
    },
    {
      "epoch": 1.4358877893187207,
      "grad_norm": 10.044119834899902,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11715171486139297,
      "logits/rejected": -0.11929846554994583,
      "logps/chosen": -595.546630859375,
      "logps/rejected": -468.5587158203125,
      "loss": 0.3912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9402472972869873,
      "rewards/margins": 1.5995492935180664,
      "rewards/rejected": -3.5397965908050537,
      "step": 10973
    },
    {
      "epoch": 1.4360186472560725,
      "grad_norm": 7.179704666137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0663066953420639,
      "logits/rejected": -0.19700857996940613,
      "logps/chosen": -436.41015625,
      "logps/rejected": -527.80517578125,
      "loss": 0.4032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.961092472076416,
      "rewards/margins": 1.2477948665618896,
      "rewards/rejected": -2.2088873386383057,
      "step": 10974
    },
    {
      "epoch": 1.4361495051934243,
      "grad_norm": 9.8366060256958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34860581159591675,
      "logits/rejected": -0.2277967631816864,
      "logps/chosen": -505.51177978515625,
      "logps/rejected": -540.5770874023438,
      "loss": 0.6082,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0849374532699585,
      "rewards/margins": 0.7072392702102661,
      "rewards/rejected": -1.792176604270935,
      "step": 10975
    },
    {
      "epoch": 1.4362803631307761,
      "grad_norm": 5.687441825866699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3968355357646942,
      "logits/rejected": -0.3421742022037506,
      "logps/chosen": -413.35748291015625,
      "logps/rejected": -402.34124755859375,
      "loss": 0.3525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.072077989578247,
      "rewards/margins": 1.1846163272857666,
      "rewards/rejected": -3.2566940784454346,
      "step": 10976
    },
    {
      "epoch": 1.436411221068128,
      "grad_norm": 5.6941046714782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2525467276573181,
      "logits/rejected": -0.37506452202796936,
      "logps/chosen": -327.9752502441406,
      "logps/rejected": -331.989501953125,
      "loss": 0.2562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3378944396972656,
      "rewards/margins": 1.457143783569336,
      "rewards/rejected": -2.7950384616851807,
      "step": 10977
    },
    {
      "epoch": 1.4365420790054797,
      "grad_norm": 9.397997856140137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06136251986026764,
      "logits/rejected": -0.14362941682338715,
      "logps/chosen": -393.4531555175781,
      "logps/rejected": -554.2866821289062,
      "loss": 0.3843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5136882066726685,
      "rewards/margins": 1.758358120918274,
      "rewards/rejected": -3.2720465660095215,
      "step": 10978
    },
    {
      "epoch": 1.4366729369428315,
      "grad_norm": 7.597623825073242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5957009792327881,
      "logits/rejected": -0.6454464197158813,
      "logps/chosen": -361.181884765625,
      "logps/rejected": -335.9032287597656,
      "loss": 0.5407,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3347629308700562,
      "rewards/margins": 0.6886680722236633,
      "rewards/rejected": -2.0234313011169434,
      "step": 10979
    },
    {
      "epoch": 1.4368037948801833,
      "grad_norm": 6.283149242401123,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11727064847946167,
      "logits/rejected": 0.2500990033149719,
      "logps/chosen": -372.7333679199219,
      "logps/rejected": -523.8360595703125,
      "loss": 0.2001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0694621801376343,
      "rewards/margins": 2.8691153526306152,
      "rewards/rejected": -3.938577175140381,
      "step": 10980
    },
    {
      "epoch": 1.436934652817535,
      "grad_norm": 9.900541305541992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08985407650470734,
      "logits/rejected": 0.11773880571126938,
      "logps/chosen": -512.7525024414062,
      "logps/rejected": -497.71710205078125,
      "loss": 0.6338,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46658003330230713,
      "rewards/margins": 1.0607795715332031,
      "rewards/rejected": -1.5273597240447998,
      "step": 10981
    },
    {
      "epoch": 1.4370655107548866,
      "grad_norm": 8.634625434875488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21470117568969727,
      "logits/rejected": -0.3395729660987854,
      "logps/chosen": -359.68780517578125,
      "logps/rejected": -369.9674072265625,
      "loss": 0.4287,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3604580163955688,
      "rewards/margins": 1.2845385074615479,
      "rewards/rejected": -2.644996404647827,
      "step": 10982
    },
    {
      "epoch": 1.4371963686922384,
      "grad_norm": 11.266557693481445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15262147784233093,
      "logits/rejected": -0.20175769925117493,
      "logps/chosen": -585.0979614257812,
      "logps/rejected": -512.7745971679688,
      "loss": 0.422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6388846635818481,
      "rewards/margins": 1.5865005254745483,
      "rewards/rejected": -3.2253851890563965,
      "step": 10983
    },
    {
      "epoch": 1.4373272266295902,
      "grad_norm": 9.732715606689453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17481467127799988,
      "logits/rejected": 0.02725728414952755,
      "logps/chosen": -685.2174072265625,
      "logps/rejected": -529.2530517578125,
      "loss": 0.394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9885464310646057,
      "rewards/margins": 1.6017180681228638,
      "rewards/rejected": -2.5902645587921143,
      "step": 10984
    },
    {
      "epoch": 1.437458084566942,
      "grad_norm": 8.798880577087402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21651291847229004,
      "logits/rejected": -0.23434990644454956,
      "logps/chosen": -403.0558776855469,
      "logps/rejected": -437.48907470703125,
      "loss": 0.5266,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.590273141860962,
      "rewards/margins": 0.8305518627166748,
      "rewards/rejected": -2.4208250045776367,
      "step": 10985
    },
    {
      "epoch": 1.4375889425042938,
      "grad_norm": 17.023393630981445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41637587547302246,
      "logits/rejected": -0.25494852662086487,
      "logps/chosen": -477.88543701171875,
      "logps/rejected": -437.9111328125,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9910770654678345,
      "rewards/margins": 1.5831176042556763,
      "rewards/rejected": -2.57419490814209,
      "step": 10986
    },
    {
      "epoch": 1.4377198004416456,
      "grad_norm": 8.095259666442871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02425253950059414,
      "logits/rejected": 0.0036093872040510178,
      "logps/chosen": -363.44415283203125,
      "logps/rejected": -454.2755126953125,
      "loss": 0.4623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2407526969909668,
      "rewards/margins": 0.8533381819725037,
      "rewards/rejected": -2.0940909385681152,
      "step": 10987
    },
    {
      "epoch": 1.4378506583789972,
      "grad_norm": 5.466098308563232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3631135821342468,
      "logits/rejected": -0.6243227124214172,
      "logps/chosen": -520.90185546875,
      "logps/rejected": -411.23956298828125,
      "loss": 0.2887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5328908562660217,
      "rewards/margins": 1.7596765756607056,
      "rewards/rejected": -2.292567491531372,
      "step": 10988
    },
    {
      "epoch": 1.437981516316349,
      "grad_norm": 12.398324012756348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3284462094306946,
      "logits/rejected": -0.5248066186904907,
      "logps/chosen": -506.1390686035156,
      "logps/rejected": -640.2271118164062,
      "loss": 0.7457,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1387242078781128,
      "rewards/margins": 0.8102564215660095,
      "rewards/rejected": -1.9489805698394775,
      "step": 10989
    },
    {
      "epoch": 1.4381123742537008,
      "grad_norm": 7.071878433227539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30710458755493164,
      "logits/rejected": -0.3015979826450348,
      "logps/chosen": -735.3976440429688,
      "logps/rejected": -455.2337341308594,
      "loss": 0.3089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2560423612594604,
      "rewards/margins": 1.4645705223083496,
      "rewards/rejected": -2.7206125259399414,
      "step": 10990
    },
    {
      "epoch": 1.4382432321910525,
      "grad_norm": 14.341577529907227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20059923827648163,
      "logits/rejected": -0.43391546607017517,
      "logps/chosen": -478.61517333984375,
      "logps/rejected": -506.63946533203125,
      "loss": 0.8557,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.683774709701538,
      "rewards/margins": 0.6819318532943726,
      "rewards/rejected": -2.365706443786621,
      "step": 10991
    },
    {
      "epoch": 1.4383740901284043,
      "grad_norm": 8.444815635681152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40888237953186035,
      "logits/rejected": -0.36200541257858276,
      "logps/chosen": -429.96435546875,
      "logps/rejected": -425.211669921875,
      "loss": 0.5894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6123402118682861,
      "rewards/margins": 0.8431470394134521,
      "rewards/rejected": -2.4554874897003174,
      "step": 10992
    },
    {
      "epoch": 1.4385049480657561,
      "grad_norm": 9.825174331665039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35070788860321045,
      "logits/rejected": -0.1479121744632721,
      "logps/chosen": -360.200927734375,
      "logps/rejected": -447.50030517578125,
      "loss": 0.7086,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1147186756134033,
      "rewards/margins": 0.22276504337787628,
      "rewards/rejected": -2.3374838829040527,
      "step": 10993
    },
    {
      "epoch": 1.438635806003108,
      "grad_norm": 8.911765098571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33574050664901733,
      "logits/rejected": -0.2589333951473236,
      "logps/chosen": -487.71307373046875,
      "logps/rejected": -520.6153564453125,
      "loss": 0.3682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5087755918502808,
      "rewards/margins": 1.4124391078948975,
      "rewards/rejected": -1.9212146997451782,
      "step": 10994
    },
    {
      "epoch": 1.4387666639404597,
      "grad_norm": 7.7107720375061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33878010511398315,
      "logits/rejected": -0.3766142725944519,
      "logps/chosen": -371.0001220703125,
      "logps/rejected": -430.44866943359375,
      "loss": 0.4411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.773985505104065,
      "rewards/margins": 0.9327647089958191,
      "rewards/rejected": -2.7067501544952393,
      "step": 10995
    },
    {
      "epoch": 1.4388975218778115,
      "grad_norm": 8.924443244934082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027192916721105576,
      "logits/rejected": -0.010152701288461685,
      "logps/chosen": -405.1975402832031,
      "logps/rejected": -526.9014892578125,
      "loss": 0.5148,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.193092703819275,
      "rewards/margins": 0.8196951150894165,
      "rewards/rejected": -2.0127875804901123,
      "step": 10996
    },
    {
      "epoch": 1.4390283798151633,
      "grad_norm": 10.032145500183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16793985664844513,
      "logits/rejected": -0.19344133138656616,
      "logps/chosen": -421.2834777832031,
      "logps/rejected": -596.8134765625,
      "loss": 0.5932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5962871313095093,
      "rewards/margins": 1.0115463733673096,
      "rewards/rejected": -2.6078333854675293,
      "step": 10997
    },
    {
      "epoch": 1.4391592377525149,
      "grad_norm": 6.736607074737549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32577642798423767,
      "logits/rejected": -0.3510964810848236,
      "logps/chosen": -433.0517272949219,
      "logps/rejected": -498.58428955078125,
      "loss": 0.3055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0073119401931763,
      "rewards/margins": 1.7189079523086548,
      "rewards/rejected": -2.726219892501831,
      "step": 10998
    },
    {
      "epoch": 1.4392900956898667,
      "grad_norm": 7.040894508361816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03485451638698578,
      "logits/rejected": -0.12098553776741028,
      "logps/chosen": -435.462158203125,
      "logps/rejected": -428.52423095703125,
      "loss": 0.2898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9075939655303955,
      "rewards/margins": 1.6302263736724854,
      "rewards/rejected": -3.53782057762146,
      "step": 10999
    },
    {
      "epoch": 1.4394209536272184,
      "grad_norm": 11.729229927062988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13516589999198914,
      "logits/rejected": -0.1127517819404602,
      "logps/chosen": -384.1330871582031,
      "logps/rejected": -605.3883056640625,
      "loss": 0.6031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2387553453445435,
      "rewards/margins": 1.1762796640396118,
      "rewards/rejected": -2.4150350093841553,
      "step": 11000
    },
    {
      "epoch": 1.4395518115645702,
      "grad_norm": 5.968343734741211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2446732223033905,
      "logits/rejected": -0.2208913117647171,
      "logps/chosen": -468.0822448730469,
      "logps/rejected": -508.83709716796875,
      "loss": 0.3405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5840599536895752,
      "rewards/margins": 1.2200074195861816,
      "rewards/rejected": -2.8040671348571777,
      "step": 11001
    },
    {
      "epoch": 1.439682669501922,
      "grad_norm": 11.149399757385254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2188258320093155,
      "logits/rejected": -0.16025038063526154,
      "logps/chosen": -385.2907409667969,
      "logps/rejected": -568.3570556640625,
      "loss": 0.623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2256805896759033,
      "rewards/margins": 0.3891599476337433,
      "rewards/rejected": -2.6148407459259033,
      "step": 11002
    },
    {
      "epoch": 1.4398135274392738,
      "grad_norm": 6.548014163970947,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2810418903827667,
      "logits/rejected": -0.4985644519329071,
      "logps/chosen": -437.016845703125,
      "logps/rejected": -405.5860900878906,
      "loss": 0.3681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2377837896347046,
      "rewards/margins": 1.3585608005523682,
      "rewards/rejected": -2.596344470977783,
      "step": 11003
    },
    {
      "epoch": 1.4399443853766254,
      "grad_norm": 17.505640029907227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15683317184448242,
      "logits/rejected": -0.32789450883865356,
      "logps/chosen": -608.0807495117188,
      "logps/rejected": -447.3921203613281,
      "loss": 0.5539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1297259330749512,
      "rewards/margins": 2.1097910404205322,
      "rewards/rejected": -3.2395169734954834,
      "step": 11004
    },
    {
      "epoch": 1.4400752433139772,
      "grad_norm": 4.018203258514404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23832082748413086,
      "logits/rejected": -0.32310840487480164,
      "logps/chosen": -492.01153564453125,
      "logps/rejected": -480.3640441894531,
      "loss": 0.2022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2747546434402466,
      "rewards/margins": 2.2994203567504883,
      "rewards/rejected": -3.5741748809814453,
      "step": 11005
    },
    {
      "epoch": 1.440206101251329,
      "grad_norm": 7.4813385009765625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025124792009592056,
      "logits/rejected": -0.05383075401186943,
      "logps/chosen": -420.98138427734375,
      "logps/rejected": -471.5036315917969,
      "loss": 0.3946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8715193867683411,
      "rewards/margins": 1.2146482467651367,
      "rewards/rejected": -2.086167573928833,
      "step": 11006
    },
    {
      "epoch": 1.4403369591886808,
      "grad_norm": 11.579238891601562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009510274976491928,
      "logits/rejected": -0.03882512077689171,
      "logps/chosen": -402.94842529296875,
      "logps/rejected": -510.720458984375,
      "loss": 0.5988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.306915760040283,
      "rewards/margins": 0.9374006986618042,
      "rewards/rejected": -3.244316577911377,
      "step": 11007
    },
    {
      "epoch": 1.4404678171260326,
      "grad_norm": 5.933213233947754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02572409249842167,
      "logits/rejected": -0.0664328932762146,
      "logps/chosen": -573.4705200195312,
      "logps/rejected": -370.29376220703125,
      "loss": 0.252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7707393765449524,
      "rewards/margins": 1.6823747158050537,
      "rewards/rejected": -2.4531140327453613,
      "step": 11008
    },
    {
      "epoch": 1.4405986750633843,
      "grad_norm": 14.041096687316895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14223332703113556,
      "logits/rejected": -0.007380310446023941,
      "logps/chosen": -351.0283508300781,
      "logps/rejected": -422.71295166015625,
      "loss": 1.2643,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.761683464050293,
      "rewards/margins": -0.6267261505126953,
      "rewards/rejected": -2.1349573135375977,
      "step": 11009
    },
    {
      "epoch": 1.4407295330007361,
      "grad_norm": 8.068760871887207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02046060562133789,
      "logits/rejected": -0.41026315093040466,
      "logps/chosen": -344.2618103027344,
      "logps/rejected": -451.0770263671875,
      "loss": 0.5976,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7846801280975342,
      "rewards/margins": 2.6356582641601562,
      "rewards/rejected": -4.4203386306762695,
      "step": 11010
    },
    {
      "epoch": 1.440860390938088,
      "grad_norm": 5.541834831237793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.061998363584280014,
      "logits/rejected": -0.3416230380535126,
      "logps/chosen": -633.2691650390625,
      "logps/rejected": -503.13336181640625,
      "loss": 0.2377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2264676094055176,
      "rewards/margins": 2.0303704738616943,
      "rewards/rejected": -3.256837844848633,
      "step": 11011
    },
    {
      "epoch": 1.4409912488754397,
      "grad_norm": 5.0637006759643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4203455448150635,
      "logits/rejected": -0.35122257471084595,
      "logps/chosen": -499.5662841796875,
      "logps/rejected": -457.90045166015625,
      "loss": 0.3127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.112352728843689,
      "rewards/margins": 1.6908930540084839,
      "rewards/rejected": -2.803246021270752,
      "step": 11012
    },
    {
      "epoch": 1.4411221068127913,
      "grad_norm": 8.196978569030762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057700999081134796,
      "logits/rejected": -0.038914408534765244,
      "logps/chosen": -490.41363525390625,
      "logps/rejected": -390.23211669921875,
      "loss": 0.6181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1773585081100464,
      "rewards/margins": 0.771682858467102,
      "rewards/rejected": -1.949041485786438,
      "step": 11013
    },
    {
      "epoch": 1.441252964750143,
      "grad_norm": 6.233704090118408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22523412108421326,
      "logits/rejected": -0.5036578178405762,
      "logps/chosen": -372.2250061035156,
      "logps/rejected": -412.85662841796875,
      "loss": 0.3449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4275646209716797,
      "rewards/margins": 1.533791422843933,
      "rewards/rejected": -1.9613561630249023,
      "step": 11014
    },
    {
      "epoch": 1.4413838226874949,
      "grad_norm": 14.114714622497559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3597855865955353,
      "logits/rejected": -0.295066773891449,
      "logps/chosen": -544.5978393554688,
      "logps/rejected": -636.5846557617188,
      "loss": 0.7213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1566195487976074,
      "rewards/margins": 0.3865630328655243,
      "rewards/rejected": -2.543182373046875,
      "step": 11015
    },
    {
      "epoch": 1.4415146806248467,
      "grad_norm": 8.344159126281738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1928827464580536,
      "logits/rejected": -0.325031042098999,
      "logps/chosen": -331.50665283203125,
      "logps/rejected": -422.69952392578125,
      "loss": 0.4113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3291678428649902,
      "rewards/margins": 1.51749849319458,
      "rewards/rejected": -2.8466663360595703,
      "step": 11016
    },
    {
      "epoch": 1.4416455385621985,
      "grad_norm": 9.056709289550781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03788696601986885,
      "logits/rejected": -0.044586729258298874,
      "logps/chosen": -443.32373046875,
      "logps/rejected": -451.76361083984375,
      "loss": 0.5257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4932327270507812,
      "rewards/margins": 1.0241843461990356,
      "rewards/rejected": -2.5174169540405273,
      "step": 11017
    },
    {
      "epoch": 1.4417763964995503,
      "grad_norm": 7.255001544952393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05591943860054016,
      "logits/rejected": -0.2272159457206726,
      "logps/chosen": -545.2921752929688,
      "logps/rejected": -457.4606628417969,
      "loss": 0.3664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7800998687744141,
      "rewards/margins": 1.7312333583831787,
      "rewards/rejected": -2.511333465576172,
      "step": 11018
    },
    {
      "epoch": 1.4419072544369018,
      "grad_norm": 6.131764888763428,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23078010976314545,
      "logits/rejected": -0.6816236972808838,
      "logps/chosen": -845.5120239257812,
      "logps/rejected": -491.43914794921875,
      "loss": 0.3916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0948193073272705,
      "rewards/margins": 1.461787223815918,
      "rewards/rejected": -2.5566065311431885,
      "step": 11019
    },
    {
      "epoch": 1.4420381123742536,
      "grad_norm": 13.809393882751465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23156984150409698,
      "logits/rejected": -0.12642592191696167,
      "logps/chosen": -405.4873962402344,
      "logps/rejected": -384.1102600097656,
      "loss": 1.2555,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.233224391937256,
      "rewards/margins": -0.23913484811782837,
      "rewards/rejected": -1.9940896034240723,
      "step": 11020
    },
    {
      "epoch": 1.4421689703116054,
      "grad_norm": 6.222151279449463,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1290014088153839,
      "logits/rejected": -0.35760021209716797,
      "logps/chosen": -334.2346496582031,
      "logps/rejected": -432.4549255371094,
      "loss": 0.2295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9375879764556885,
      "rewards/margins": 2.2152442932128906,
      "rewards/rejected": -3.152832269668579,
      "step": 11021
    },
    {
      "epoch": 1.4422998282489572,
      "grad_norm": 7.49948263168335,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1334928572177887,
      "logits/rejected": -0.22873345017433167,
      "logps/chosen": -631.1845703125,
      "logps/rejected": -507.9922790527344,
      "loss": 0.2508,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3621982336044312,
      "rewards/margins": 1.7137985229492188,
      "rewards/rejected": -3.0759968757629395,
      "step": 11022
    },
    {
      "epoch": 1.442430686186309,
      "grad_norm": 7.906189441680908,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2591339647769928,
      "logits/rejected": -0.4040524661540985,
      "logps/chosen": -510.5491638183594,
      "logps/rejected": -506.96429443359375,
      "loss": 0.44,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47477930784225464,
      "rewards/margins": 1.192145824432373,
      "rewards/rejected": -1.666925072669983,
      "step": 11023
    },
    {
      "epoch": 1.4425615441236608,
      "grad_norm": 6.600374221801758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11868168413639069,
      "logits/rejected": -0.06361930817365646,
      "logps/chosen": -415.11370849609375,
      "logps/rejected": -435.10797119140625,
      "loss": 0.3259,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.794847011566162,
      "rewards/margins": 1.5676708221435547,
      "rewards/rejected": -3.3625175952911377,
      "step": 11024
    },
    {
      "epoch": 1.4426924020610126,
      "grad_norm": 7.615231513977051,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21896149218082428,
      "logits/rejected": -0.12416905164718628,
      "logps/chosen": -589.1981201171875,
      "logps/rejected": -481.89422607421875,
      "loss": 0.4156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2737300395965576,
      "rewards/margins": 1.3852076530456543,
      "rewards/rejected": -2.658937692642212,
      "step": 11025
    },
    {
      "epoch": 1.4428232599983644,
      "grad_norm": 3.705937623977661,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12894083559513092,
      "logits/rejected": -0.226647287607193,
      "logps/chosen": -538.1431274414062,
      "logps/rejected": -519.6636352539062,
      "loss": 0.1253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7917711138725281,
      "rewards/margins": 2.718045473098755,
      "rewards/rejected": -3.5098166465759277,
      "step": 11026
    },
    {
      "epoch": 1.4429541179357162,
      "grad_norm": 4.97381067276001,
      "learning_rate": 5e-06,
      "logits/chosen": -0.474494606256485,
      "logits/rejected": -0.43147772550582886,
      "logps/chosen": -552.262451171875,
      "logps/rejected": -619.2422485351562,
      "loss": 0.2575,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9733884930610657,
      "rewards/margins": 2.2999815940856934,
      "rewards/rejected": -3.2733705043792725,
      "step": 11027
    },
    {
      "epoch": 1.4430849758730677,
      "grad_norm": 16.069007873535156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2745867669582367,
      "logits/rejected": 0.01264260709285736,
      "logps/chosen": -491.2067565917969,
      "logps/rejected": -571.5911254882812,
      "loss": 0.7498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.063162088394165,
      "rewards/margins": 0.46219688653945923,
      "rewards/rejected": -3.5253589153289795,
      "step": 11028
    },
    {
      "epoch": 1.4432158338104195,
      "grad_norm": 4.090205192565918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.437374472618103,
      "logits/rejected": -0.3433891534805298,
      "logps/chosen": -406.14739990234375,
      "logps/rejected": -400.2179870605469,
      "loss": 0.2059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9474729299545288,
      "rewards/margins": 1.7168134450912476,
      "rewards/rejected": -3.6642861366271973,
      "step": 11029
    },
    {
      "epoch": 1.4433466917477713,
      "grad_norm": 12.187214851379395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27181538939476013,
      "logits/rejected": -0.13891476392745972,
      "logps/chosen": -467.4960021972656,
      "logps/rejected": -503.2584533691406,
      "loss": 0.5808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3656097650527954,
      "rewards/margins": 0.9864839315414429,
      "rewards/rejected": -2.3520939350128174,
      "step": 11030
    },
    {
      "epoch": 1.443477549685123,
      "grad_norm": 8.193554878234863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28133097290992737,
      "logits/rejected": -0.026574745774269104,
      "logps/chosen": -485.6400146484375,
      "logps/rejected": -424.5368347167969,
      "loss": 0.4797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.753021001815796,
      "rewards/margins": 0.9330241084098816,
      "rewards/rejected": -2.686044931411743,
      "step": 11031
    },
    {
      "epoch": 1.4436084076224749,
      "grad_norm": 10.003293991088867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14441734552383423,
      "logits/rejected": -0.28569746017456055,
      "logps/chosen": -597.6746215820312,
      "logps/rejected": -502.703125,
      "loss": 0.5138,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6022757291793823,
      "rewards/margins": 1.451967716217041,
      "rewards/rejected": -3.054243564605713,
      "step": 11032
    },
    {
      "epoch": 1.4437392655598267,
      "grad_norm": 4.1093430519104,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1999392956495285,
      "logits/rejected": -0.40076392889022827,
      "logps/chosen": -476.1147155761719,
      "logps/rejected": -545.3731079101562,
      "loss": 0.1498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0624151229858398,
      "rewards/margins": 2.567352771759033,
      "rewards/rejected": -3.629767656326294,
      "step": 11033
    },
    {
      "epoch": 1.4438701234971782,
      "grad_norm": 7.914571285247803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1374332308769226,
      "logits/rejected": -0.34500667452812195,
      "logps/chosen": -598.1651611328125,
      "logps/rejected": -556.0543823242188,
      "loss": 0.3339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2422057390213013,
      "rewards/margins": 1.1218059062957764,
      "rewards/rejected": -2.364011764526367,
      "step": 11034
    },
    {
      "epoch": 1.44400098143453,
      "grad_norm": 9.684856414794922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2885400056838989,
      "logits/rejected": -0.05253560096025467,
      "logps/chosen": -579.6737060546875,
      "logps/rejected": -619.7138061523438,
      "loss": 0.4,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9021363258361816,
      "rewards/margins": 1.7306125164031982,
      "rewards/rejected": -3.632748603820801,
      "step": 11035
    },
    {
      "epoch": 1.4441318393718818,
      "grad_norm": 15.448898315429688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16418108344078064,
      "logits/rejected": -0.010775703936815262,
      "logps/chosen": -451.7943115234375,
      "logps/rejected": -419.13482666015625,
      "loss": 0.7169,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.492220640182495,
      "rewards/margins": 0.2953680157661438,
      "rewards/rejected": -2.7875888347625732,
      "step": 11036
    },
    {
      "epoch": 1.4442626973092336,
      "grad_norm": 8.593110084533691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.127315491437912,
      "logits/rejected": -0.11407163739204407,
      "logps/chosen": -523.1494750976562,
      "logps/rejected": -467.57501220703125,
      "loss": 0.5023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5413986444473267,
      "rewards/margins": 1.2535791397094727,
      "rewards/rejected": -2.7949776649475098,
      "step": 11037
    },
    {
      "epoch": 1.4443935552465854,
      "grad_norm": 8.406770706176758,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03401591628789902,
      "logits/rejected": -0.04773882031440735,
      "logps/chosen": -441.0198059082031,
      "logps/rejected": -455.1763000488281,
      "loss": 0.4726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.479172945022583,
      "rewards/margins": 0.74833744764328,
      "rewards/rejected": -2.2275102138519287,
      "step": 11038
    },
    {
      "epoch": 1.4445244131839372,
      "grad_norm": 6.170456409454346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47688451409339905,
      "logits/rejected": -0.6039207577705383,
      "logps/chosen": -368.0357971191406,
      "logps/rejected": -455.53662109375,
      "loss": 0.3009,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4268814325332642,
      "rewards/margins": 1.5948123931884766,
      "rewards/rejected": -3.0216939449310303,
      "step": 11039
    },
    {
      "epoch": 1.444655271121289,
      "grad_norm": 11.01243782043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15872812271118164,
      "logits/rejected": -0.16958780586719513,
      "logps/chosen": -471.45062255859375,
      "logps/rejected": -505.3822937011719,
      "loss": 0.915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4237128496170044,
      "rewards/margins": 0.856248676776886,
      "rewards/rejected": -2.279961585998535,
      "step": 11040
    },
    {
      "epoch": 1.4447861290586408,
      "grad_norm": 4.817502975463867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24299301207065582,
      "logits/rejected": -0.23297782242298126,
      "logps/chosen": -369.55169677734375,
      "logps/rejected": -339.9588928222656,
      "loss": 0.3548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.180847644805908,
      "rewards/margins": 1.357202410697937,
      "rewards/rejected": -3.5380501747131348,
      "step": 11041
    },
    {
      "epoch": 1.4449169869959926,
      "grad_norm": 4.863348484039307,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0358683317899704,
      "logits/rejected": -0.19995267689228058,
      "logps/chosen": -663.3775024414062,
      "logps/rejected": -451.9499206542969,
      "loss": 0.2117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8594050407409668,
      "rewards/margins": 2.6896021366119385,
      "rewards/rejected": -3.5490071773529053,
      "step": 11042
    },
    {
      "epoch": 1.4450478449333444,
      "grad_norm": 7.5406813621521,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07023343443870544,
      "logits/rejected": 0.08104267716407776,
      "logps/chosen": -489.97650146484375,
      "logps/rejected": -521.8567504882812,
      "loss": 0.5026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.11496639251709,
      "rewards/margins": 0.8849369287490845,
      "rewards/rejected": -2.999903440475464,
      "step": 11043
    },
    {
      "epoch": 1.445178702870696,
      "grad_norm": 5.948654651641846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3713020384311676,
      "logits/rejected": -0.41386178135871887,
      "logps/chosen": -496.96929931640625,
      "logps/rejected": -433.43304443359375,
      "loss": 0.2735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6690045595169067,
      "rewards/margins": 1.616083025932312,
      "rewards/rejected": -3.2850875854492188,
      "step": 11044
    },
    {
      "epoch": 1.4453095608080477,
      "grad_norm": 9.981598854064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09961868077516556,
      "logits/rejected": -0.1937873512506485,
      "logps/chosen": -631.8094482421875,
      "logps/rejected": -506.9283752441406,
      "loss": 0.3516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5289912223815918,
      "rewards/margins": 1.31150484085083,
      "rewards/rejected": -2.840496063232422,
      "step": 11045
    },
    {
      "epoch": 1.4454404187453995,
      "grad_norm": 4.987728595733643,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20622950792312622,
      "logits/rejected": -0.0638921931385994,
      "logps/chosen": -362.692138671875,
      "logps/rejected": -455.6733093261719,
      "loss": 0.1994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.266499638557434,
      "rewards/margins": 1.7527680397033691,
      "rewards/rejected": -3.0192677974700928,
      "step": 11046
    },
    {
      "epoch": 1.4455712766827513,
      "grad_norm": 18.47574806213379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009278162382543087,
      "logits/rejected": -0.1176024079322815,
      "logps/chosen": -495.07867431640625,
      "logps/rejected": -417.8643798828125,
      "loss": 0.9547,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3371379375457764,
      "rewards/margins": 0.3470730483531952,
      "rewards/rejected": -1.6842111349105835,
      "step": 11047
    },
    {
      "epoch": 1.445702134620103,
      "grad_norm": 11.128376960754395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13865631818771362,
      "logits/rejected": -0.17849747836589813,
      "logps/chosen": -465.33392333984375,
      "logps/rejected": -464.23974609375,
      "loss": 0.5228,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.56203293800354,
      "rewards/margins": 1.3091588020324707,
      "rewards/rejected": -3.87119197845459,
      "step": 11048
    },
    {
      "epoch": 1.4458329925574547,
      "grad_norm": 9.834090232849121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06333206593990326,
      "logits/rejected": 0.24188868701457977,
      "logps/chosen": -421.3215026855469,
      "logps/rejected": -488.958740234375,
      "loss": 0.6471,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.07147479057312,
      "rewards/margins": 0.9431235790252686,
      "rewards/rejected": -3.0145983695983887,
      "step": 11049
    },
    {
      "epoch": 1.4459638504948065,
      "grad_norm": 5.042048931121826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2707843482494354,
      "logits/rejected": -0.1305919885635376,
      "logps/chosen": -340.61114501953125,
      "logps/rejected": -525.1297607421875,
      "loss": 0.2065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.014385938644409,
      "rewards/margins": 2.952399730682373,
      "rewards/rejected": -4.966785430908203,
      "step": 11050
    },
    {
      "epoch": 1.4460947084321583,
      "grad_norm": 9.137521743774414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5681013464927673,
      "logits/rejected": -0.6601570844650269,
      "logps/chosen": -455.0188293457031,
      "logps/rejected": -396.9294128417969,
      "loss": 0.656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5610523223876953,
      "rewards/margins": 0.6098371148109436,
      "rewards/rejected": -2.170889377593994,
      "step": 11051
    },
    {
      "epoch": 1.44622556636951,
      "grad_norm": 9.58073616027832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08942477405071259,
      "logits/rejected": -0.1515728235244751,
      "logps/chosen": -564.0736694335938,
      "logps/rejected": -518.88671875,
      "loss": 0.4828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.951171875,
      "rewards/margins": 1.4004504680633545,
      "rewards/rejected": -3.3516225814819336,
      "step": 11052
    },
    {
      "epoch": 1.4463564243068618,
      "grad_norm": 5.443498134613037,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2876472771167755,
      "logits/rejected": -0.584612250328064,
      "logps/chosen": -534.41943359375,
      "logps/rejected": -595.9791259765625,
      "loss": 0.2509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0804107189178467,
      "rewards/margins": 1.6288408041000366,
      "rewards/rejected": -2.7092514038085938,
      "step": 11053
    },
    {
      "epoch": 1.4464872822442136,
      "grad_norm": 5.526882171630859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30089783668518066,
      "logits/rejected": -0.44369742274284363,
      "logps/chosen": -459.6128234863281,
      "logps/rejected": -350.8533935546875,
      "loss": 0.1607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8808355331420898,
      "rewards/margins": 2.7241055965423584,
      "rewards/rejected": -3.604940891265869,
      "step": 11054
    },
    {
      "epoch": 1.4466181401815654,
      "grad_norm": 7.174219608306885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26729005575180054,
      "logits/rejected": -0.27672845125198364,
      "logps/chosen": -363.8121337890625,
      "logps/rejected": -482.936279296875,
      "loss": 0.3682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6969022750854492,
      "rewards/margins": 1.9961979389190674,
      "rewards/rejected": -3.6931002140045166,
      "step": 11055
    },
    {
      "epoch": 1.4467489981189172,
      "grad_norm": 5.731112957000732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2092357575893402,
      "logits/rejected": -0.454476535320282,
      "logps/chosen": -443.7066650390625,
      "logps/rejected": -364.89776611328125,
      "loss": 0.2941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1505838632583618,
      "rewards/margins": 2.303549289703369,
      "rewards/rejected": -3.4541330337524414,
      "step": 11056
    },
    {
      "epoch": 1.446879856056269,
      "grad_norm": 11.446518898010254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4550126791000366,
      "logits/rejected": -0.593410074710846,
      "logps/chosen": -741.7430419921875,
      "logps/rejected": -636.0556640625,
      "loss": 0.501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.373262405395508,
      "rewards/margins": 0.9689159393310547,
      "rewards/rejected": -3.3421783447265625,
      "step": 11057
    },
    {
      "epoch": 1.4470107139936208,
      "grad_norm": 8.15206241607666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10088260471820831,
      "logits/rejected": -0.2620134651660919,
      "logps/chosen": -382.5790100097656,
      "logps/rejected": -515.6890258789062,
      "loss": 0.3233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9119850397109985,
      "rewards/margins": 2.8999738693237305,
      "rewards/rejected": -3.8119590282440186,
      "step": 11058
    },
    {
      "epoch": 1.4471415719309724,
      "grad_norm": 7.432657241821289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17575392127037048,
      "logits/rejected": -0.08201850950717926,
      "logps/chosen": -478.0637512207031,
      "logps/rejected": -584.4198608398438,
      "loss": 0.3226,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9906765222549438,
      "rewards/margins": 1.4629592895507812,
      "rewards/rejected": -2.4536359310150146,
      "step": 11059
    },
    {
      "epoch": 1.4472724298683242,
      "grad_norm": 5.310056209564209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3795551359653473,
      "logits/rejected": -0.49631059169769287,
      "logps/chosen": -594.2726440429688,
      "logps/rejected": -595.2056274414062,
      "loss": 0.1983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4940570592880249,
      "rewards/margins": 2.376081705093384,
      "rewards/rejected": -2.870138645172119,
      "step": 11060
    },
    {
      "epoch": 1.447403287805676,
      "grad_norm": 11.08001708984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42812925577163696,
      "logits/rejected": -0.3475304841995239,
      "logps/chosen": -416.8360595703125,
      "logps/rejected": -515.8541259765625,
      "loss": 0.648,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2545623779296875,
      "rewards/margins": 0.944239616394043,
      "rewards/rejected": -2.1988019943237305,
      "step": 11061
    },
    {
      "epoch": 1.4475341457430277,
      "grad_norm": 17.208316802978516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07588072121143341,
      "logits/rejected": 0.00310579314827919,
      "logps/chosen": -456.783447265625,
      "logps/rejected": -411.5638427734375,
      "loss": 1.0328,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.422879934310913,
      "rewards/margins": 0.1186322271823883,
      "rewards/rejected": -2.5415122509002686,
      "step": 11062
    },
    {
      "epoch": 1.4476650036803795,
      "grad_norm": 12.217732429504395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1890813112258911,
      "logits/rejected": -0.16233384609222412,
      "logps/chosen": -523.2085571289062,
      "logps/rejected": -631.7373657226562,
      "loss": 0.819,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.4606070518493652,
      "rewards/margins": 1.2173737287521362,
      "rewards/rejected": -3.677980899810791,
      "step": 11063
    },
    {
      "epoch": 1.4477958616177313,
      "grad_norm": 8.76435375213623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11231085658073425,
      "logits/rejected": -0.07398246228694916,
      "logps/chosen": -397.3044128417969,
      "logps/rejected": -405.9109191894531,
      "loss": 0.4191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9206547737121582,
      "rewards/margins": 1.39723801612854,
      "rewards/rejected": -3.317892551422119,
      "step": 11064
    },
    {
      "epoch": 1.447926719555083,
      "grad_norm": 12.571281433105469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22934943437576294,
      "logits/rejected": -0.2632676362991333,
      "logps/chosen": -437.393310546875,
      "logps/rejected": -562.8023071289062,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.8244118690490723,
      "rewards/margins": 0.43773674964904785,
      "rewards/rejected": -3.26214861869812,
      "step": 11065
    },
    {
      "epoch": 1.4480575774924347,
      "grad_norm": 12.576240539550781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11862783133983612,
      "logits/rejected": -0.008210301399230957,
      "logps/chosen": -447.73095703125,
      "logps/rejected": -478.6148986816406,
      "loss": 0.8375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3754961490631104,
      "rewards/margins": 0.40823063254356384,
      "rewards/rejected": -1.783726692199707,
      "step": 11066
    },
    {
      "epoch": 1.4481884354297865,
      "grad_norm": 6.04387903213501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3315114378929138,
      "logits/rejected": -0.6355062127113342,
      "logps/chosen": -362.1700134277344,
      "logps/rejected": -420.74102783203125,
      "loss": 0.2199,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.872077465057373,
      "rewards/margins": 2.3015265464782715,
      "rewards/rejected": -4.1736040115356445,
      "step": 11067
    },
    {
      "epoch": 1.4483192933671383,
      "grad_norm": 11.222256660461426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30220380425453186,
      "logits/rejected": -0.3456665575504303,
      "logps/chosen": -486.43408203125,
      "logps/rejected": -441.0977783203125,
      "loss": 0.5108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.927669882774353,
      "rewards/margins": 0.983655571937561,
      "rewards/rejected": -2.911325454711914,
      "step": 11068
    },
    {
      "epoch": 1.44845015130449,
      "grad_norm": 12.422367095947266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22965776920318604,
      "logits/rejected": -0.16003172099590302,
      "logps/chosen": -396.06048583984375,
      "logps/rejected": -330.0538330078125,
      "loss": 0.859,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.402587652206421,
      "rewards/margins": 0.08890324831008911,
      "rewards/rejected": -1.4914909601211548,
      "step": 11069
    },
    {
      "epoch": 1.4485810092418419,
      "grad_norm": 7.553102016448975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13678613305091858,
      "logits/rejected": -0.16500331461429596,
      "logps/chosen": -624.4209594726562,
      "logps/rejected": -381.1365661621094,
      "loss": 0.4496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3755695819854736,
      "rewards/margins": 1.300145149230957,
      "rewards/rejected": -2.6757149696350098,
      "step": 11070
    },
    {
      "epoch": 1.4487118671791936,
      "grad_norm": 9.118678092956543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043779417872428894,
      "logits/rejected": -0.1498974859714508,
      "logps/chosen": -340.43133544921875,
      "logps/rejected": -405.350830078125,
      "loss": 0.3652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9769790172576904,
      "rewards/margins": 1.6858198642730713,
      "rewards/rejected": -3.6627988815307617,
      "step": 11071
    },
    {
      "epoch": 1.4488427251165454,
      "grad_norm": 14.37403392791748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034153275191783905,
      "logits/rejected": -0.07883795350790024,
      "logps/chosen": -655.1815795898438,
      "logps/rejected": -651.3917236328125,
      "loss": 0.7011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.4943909645080566,
      "rewards/margins": 0.5713924169540405,
      "rewards/rejected": -3.0657835006713867,
      "step": 11072
    },
    {
      "epoch": 1.4489735830538972,
      "grad_norm": 11.007461547851562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25268837809562683,
      "logits/rejected": -0.10262897610664368,
      "logps/chosen": -605.0720825195312,
      "logps/rejected": -607.740478515625,
      "loss": 0.544,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1724066734313965,
      "rewards/margins": 1.9415016174316406,
      "rewards/rejected": -3.113908052444458,
      "step": 11073
    },
    {
      "epoch": 1.4491044409912488,
      "grad_norm": 7.054904937744141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35259905457496643,
      "logits/rejected": -0.4049427807331085,
      "logps/chosen": -481.4593200683594,
      "logps/rejected": -528.4745483398438,
      "loss": 0.312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8137176632881165,
      "rewards/margins": 1.8282504081726074,
      "rewards/rejected": -2.641968011856079,
      "step": 11074
    },
    {
      "epoch": 1.4492352989286006,
      "grad_norm": 14.885456085205078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.305782675743103,
      "logits/rejected": -0.15487855672836304,
      "logps/chosen": -447.52490234375,
      "logps/rejected": -407.8002624511719,
      "loss": 0.8453,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.6079533100128174,
      "rewards/margins": 0.09942375123500824,
      "rewards/rejected": -2.7073769569396973,
      "step": 11075
    },
    {
      "epoch": 1.4493661568659524,
      "grad_norm": 6.683374404907227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3704153299331665,
      "logits/rejected": -0.3955440819263458,
      "logps/chosen": -543.2880859375,
      "logps/rejected": -507.169189453125,
      "loss": 0.3181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7915637493133545,
      "rewards/margins": 1.8830689191818237,
      "rewards/rejected": -3.6746325492858887,
      "step": 11076
    },
    {
      "epoch": 1.4494970148033042,
      "grad_norm": 5.982889652252197,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22279933094978333,
      "logits/rejected": 0.029838573187589645,
      "logps/chosen": -457.4719543457031,
      "logps/rejected": -467.9651794433594,
      "loss": 0.2183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.828757643699646,
      "rewards/margins": 2.3691844940185547,
      "rewards/rejected": -3.1979422569274902,
      "step": 11077
    },
    {
      "epoch": 1.449627872740656,
      "grad_norm": 4.421989917755127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5089091658592224,
      "logits/rejected": -0.3929172158241272,
      "logps/chosen": -589.6311645507812,
      "logps/rejected": -579.71875,
      "loss": 0.1521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4136780500411987,
      "rewards/margins": 2.335526466369629,
      "rewards/rejected": -3.749204397201538,
      "step": 11078
    },
    {
      "epoch": 1.4497587306780078,
      "grad_norm": 7.519597053527832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3526831865310669,
      "logits/rejected": -0.366411030292511,
      "logps/chosen": -421.2686462402344,
      "logps/rejected": -390.9045715332031,
      "loss": 0.4,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.895017385482788,
      "rewards/margins": 1.2475669384002686,
      "rewards/rejected": -3.1425840854644775,
      "step": 11079
    },
    {
      "epoch": 1.4498895886153593,
      "grad_norm": 6.347825050354004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13894431293010712,
      "logits/rejected": -0.14712025225162506,
      "logps/chosen": -596.3421630859375,
      "logps/rejected": -625.4971923828125,
      "loss": 0.2863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40264320373535156,
      "rewards/margins": 2.536043643951416,
      "rewards/rejected": -2.9386868476867676,
      "step": 11080
    },
    {
      "epoch": 1.4500204465527111,
      "grad_norm": 9.659333229064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07459378987550735,
      "logits/rejected": -0.12995311617851257,
      "logps/chosen": -522.86767578125,
      "logps/rejected": -569.4724731445312,
      "loss": 0.3436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8440326452255249,
      "rewards/margins": 1.8040356636047363,
      "rewards/rejected": -2.6480681896209717,
      "step": 11081
    },
    {
      "epoch": 1.450151304490063,
      "grad_norm": 7.422844409942627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16415955126285553,
      "logits/rejected": -0.2489507496356964,
      "logps/chosen": -396.5589599609375,
      "logps/rejected": -411.2283020019531,
      "loss": 0.3658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0056926012039185,
      "rewards/margins": 1.3312492370605469,
      "rewards/rejected": -2.336941957473755,
      "step": 11082
    },
    {
      "epoch": 1.4502821624274147,
      "grad_norm": 7.533945083618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05618656426668167,
      "logits/rejected": -0.04739788547158241,
      "logps/chosen": -513.3818969726562,
      "logps/rejected": -393.7093811035156,
      "loss": 0.4345,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0121971368789673,
      "rewards/margins": 1.4749817848205566,
      "rewards/rejected": -2.4871790409088135,
      "step": 11083
    },
    {
      "epoch": 1.4504130203647665,
      "grad_norm": 6.1339545249938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1892743706703186,
      "logits/rejected": -0.33115053176879883,
      "logps/chosen": -464.2290954589844,
      "logps/rejected": -511.21087646484375,
      "loss": 0.2645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7999998331069946,
      "rewards/margins": 1.4704614877700806,
      "rewards/rejected": -3.270461320877075,
      "step": 11084
    },
    {
      "epoch": 1.4505438783021183,
      "grad_norm": 10.204422950744629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09677422046661377,
      "logits/rejected": -0.03687869757413864,
      "logps/chosen": -445.55548095703125,
      "logps/rejected": -466.0877685546875,
      "loss": 0.6243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.828810214996338,
      "rewards/margins": 1.0140022039413452,
      "rewards/rejected": -2.8428125381469727,
      "step": 11085
    },
    {
      "epoch": 1.45067473623947,
      "grad_norm": 9.041841506958008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3395242393016815,
      "logits/rejected": -0.21603386104106903,
      "logps/chosen": -723.3058471679688,
      "logps/rejected": -571.0791015625,
      "loss": 0.3117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9177625775337219,
      "rewards/margins": 2.0858471393585205,
      "rewards/rejected": -3.0036096572875977,
      "step": 11086
    },
    {
      "epoch": 1.4508055941768219,
      "grad_norm": 7.811751842498779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08620390295982361,
      "logits/rejected": -0.025761546567082405,
      "logps/chosen": -373.0937805175781,
      "logps/rejected": -317.06787109375,
      "loss": 0.6704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1383450031280518,
      "rewards/margins": 0.5128308534622192,
      "rewards/rejected": -1.651175618171692,
      "step": 11087
    },
    {
      "epoch": 1.4509364521141737,
      "grad_norm": 5.4155707359313965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02828584983944893,
      "logits/rejected": -0.033619605004787445,
      "logps/chosen": -488.8931884765625,
      "logps/rejected": -473.50592041015625,
      "loss": 0.217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.323665976524353,
      "rewards/margins": 1.9172102212905884,
      "rewards/rejected": -3.2408764362335205,
      "step": 11088
    },
    {
      "epoch": 1.4510673100515254,
      "grad_norm": 8.880388259887695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0890318751335144,
      "logits/rejected": -0.10884131491184235,
      "logps/chosen": -688.5941162109375,
      "logps/rejected": -570.82568359375,
      "loss": 0.4632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9289190769195557,
      "rewards/margins": 2.001431703567505,
      "rewards/rejected": -2.9303507804870605,
      "step": 11089
    },
    {
      "epoch": 1.451198167988877,
      "grad_norm": 7.347347259521484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3088206946849823,
      "logits/rejected": -0.4940498471260071,
      "logps/chosen": -410.511474609375,
      "logps/rejected": -538.2752685546875,
      "loss": 0.5031,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7030755281448364,
      "rewards/margins": 1.199781060218811,
      "rewards/rejected": -2.9028563499450684,
      "step": 11090
    },
    {
      "epoch": 1.4513290259262288,
      "grad_norm": 7.027970790863037,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19041669368743896,
      "logits/rejected": -0.3753848969936371,
      "logps/chosen": -387.929931640625,
      "logps/rejected": -421.0154724121094,
      "loss": 0.5062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1079009771347046,
      "rewards/margins": 1.9290828704833984,
      "rewards/rejected": -3.0369839668273926,
      "step": 11091
    },
    {
      "epoch": 1.4514598838635806,
      "grad_norm": 10.621939659118652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2481972873210907,
      "logits/rejected": -0.21733859181404114,
      "logps/chosen": -640.5950927734375,
      "logps/rejected": -715.3734130859375,
      "loss": 0.6944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6822483539581299,
      "rewards/margins": 0.5030864477157593,
      "rewards/rejected": -2.1853346824645996,
      "step": 11092
    },
    {
      "epoch": 1.4515907418009324,
      "grad_norm": 7.441710472106934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37637022137641907,
      "logits/rejected": -0.23678475618362427,
      "logps/chosen": -526.2841186523438,
      "logps/rejected": -560.4674072265625,
      "loss": 0.2906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3917702436447144,
      "rewards/margins": 2.384465456008911,
      "rewards/rejected": -3.776235580444336,
      "step": 11093
    },
    {
      "epoch": 1.4517215997382842,
      "grad_norm": 6.6454243659973145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1043466106057167,
      "logits/rejected": -0.17904901504516602,
      "logps/chosen": -390.95892333984375,
      "logps/rejected": -307.6664123535156,
      "loss": 0.3446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5847744941711426,
      "rewards/margins": 1.5410581827163696,
      "rewards/rejected": -3.1258327960968018,
      "step": 11094
    },
    {
      "epoch": 1.4518524576756358,
      "grad_norm": 6.05724573135376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025977060198783875,
      "logits/rejected": -0.040113963186740875,
      "logps/chosen": -606.2116088867188,
      "logps/rejected": -423.7951354980469,
      "loss": 0.3215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49101412296295166,
      "rewards/margins": 2.0049283504486084,
      "rewards/rejected": -2.4959423542022705,
      "step": 11095
    },
    {
      "epoch": 1.4519833156129875,
      "grad_norm": 11.429998397827148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35086771845817566,
      "logits/rejected": -0.44202256202697754,
      "logps/chosen": -527.0643920898438,
      "logps/rejected": -450.2651062011719,
      "loss": 0.5821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7224254608154297,
      "rewards/margins": 1.4039490222930908,
      "rewards/rejected": -3.1263747215270996,
      "step": 11096
    },
    {
      "epoch": 1.4521141735503393,
      "grad_norm": 13.144449234008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23561008274555206,
      "logits/rejected": -0.3684558868408203,
      "logps/chosen": -487.06683349609375,
      "logps/rejected": -462.9633483886719,
      "loss": 0.6454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4118098020553589,
      "rewards/margins": 1.242266058921814,
      "rewards/rejected": -2.654075860977173,
      "step": 11097
    },
    {
      "epoch": 1.4522450314876911,
      "grad_norm": 8.121711730957031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0768839418888092,
      "logits/rejected": -0.02698032185435295,
      "logps/chosen": -477.6348571777344,
      "logps/rejected": -494.31451416015625,
      "loss": 0.3346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.052830696105957,
      "rewards/margins": 1.8117717504501343,
      "rewards/rejected": -2.864602565765381,
      "step": 11098
    },
    {
      "epoch": 1.452375889425043,
      "grad_norm": 10.850700378417969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27529746294021606,
      "logits/rejected": -0.2623058557510376,
      "logps/chosen": -434.1614990234375,
      "logps/rejected": -580.2120361328125,
      "loss": 0.5673,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.15455961227417,
      "rewards/margins": 0.8426883220672607,
      "rewards/rejected": -2.9972479343414307,
      "step": 11099
    },
    {
      "epoch": 1.4525067473623947,
      "grad_norm": 6.185617446899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1671743094921112,
      "logits/rejected": -0.21018750965595245,
      "logps/chosen": -398.2696838378906,
      "logps/rejected": -357.3407287597656,
      "loss": 0.3542,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8240034580230713,
      "rewards/margins": 2.355771064758301,
      "rewards/rejected": -3.179774522781372,
      "step": 11100
    },
    {
      "epoch": 1.4526376052997465,
      "grad_norm": 9.049751281738281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05169929936528206,
      "logits/rejected": -0.09330721944570541,
      "logps/chosen": -489.90875244140625,
      "logps/rejected": -606.6431274414062,
      "loss": 0.5131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9110145568847656,
      "rewards/margins": 1.142863392829895,
      "rewards/rejected": -2.05387806892395,
      "step": 11101
    },
    {
      "epoch": 1.4527684632370983,
      "grad_norm": 4.069657802581787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28973692655563354,
      "logits/rejected": -0.31387078762054443,
      "logps/chosen": -465.62225341796875,
      "logps/rejected": -353.61767578125,
      "loss": 0.1954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2893061637878418,
      "rewards/margins": 2.101655960083008,
      "rewards/rejected": -3.3909621238708496,
      "step": 11102
    },
    {
      "epoch": 1.45289932117445,
      "grad_norm": 7.029448509216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14967286586761475,
      "logits/rejected": -0.32466042041778564,
      "logps/chosen": -422.810546875,
      "logps/rejected": -490.2392883300781,
      "loss": 0.2969,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2657548189163208,
      "rewards/margins": 2.2142298221588135,
      "rewards/rejected": -3.479984760284424,
      "step": 11103
    },
    {
      "epoch": 1.4530301791118019,
      "grad_norm": 12.796676635742188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2556649148464203,
      "logits/rejected": -0.20115116238594055,
      "logps/chosen": -737.97802734375,
      "logps/rejected": -401.591552734375,
      "loss": 0.7389,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.425548553466797,
      "rewards/margins": 0.35261270403862,
      "rewards/rejected": -2.7781612873077393,
      "step": 11104
    },
    {
      "epoch": 1.4531610370491534,
      "grad_norm": 8.71013355255127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20787592232227325,
      "logits/rejected": -0.1391257643699646,
      "logps/chosen": -374.31488037109375,
      "logps/rejected": -667.9810180664062,
      "loss": 0.4026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8678429126739502,
      "rewards/margins": 1.802724838256836,
      "rewards/rejected": -3.670567750930786,
      "step": 11105
    },
    {
      "epoch": 1.4532918949865052,
      "grad_norm": 6.222113132476807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3437919616699219,
      "logits/rejected": -0.29184380173683167,
      "logps/chosen": -245.6142578125,
      "logps/rejected": -340.84796142578125,
      "loss": 0.3846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6990920305252075,
      "rewards/margins": 1.5697563886642456,
      "rewards/rejected": -3.268848419189453,
      "step": 11106
    },
    {
      "epoch": 1.453422752923857,
      "grad_norm": 6.212559223175049,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33105897903442383,
      "logits/rejected": -0.36126047372817993,
      "logps/chosen": -512.751708984375,
      "logps/rejected": -427.43597412109375,
      "loss": 0.4474,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2863022089004517,
      "rewards/margins": 1.4565422534942627,
      "rewards/rejected": -2.742844343185425,
      "step": 11107
    },
    {
      "epoch": 1.4535536108612088,
      "grad_norm": 6.308643817901611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2874615490436554,
      "logits/rejected": -0.25912293791770935,
      "logps/chosen": -462.9676208496094,
      "logps/rejected": -410.0843505859375,
      "loss": 0.3749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4976692199707031,
      "rewards/margins": 1.493924856185913,
      "rewards/rejected": -2.991594076156616,
      "step": 11108
    },
    {
      "epoch": 1.4536844687985606,
      "grad_norm": 7.94374418258667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5207747220993042,
      "logits/rejected": -0.4452816843986511,
      "logps/chosen": -545.2793579101562,
      "logps/rejected": -528.4385986328125,
      "loss": 0.4428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9060559272766113,
      "rewards/margins": 1.3266003131866455,
      "rewards/rejected": -3.2326557636260986,
      "step": 11109
    },
    {
      "epoch": 1.4538153267359124,
      "grad_norm": 4.969460964202881,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0958518385887146,
      "logits/rejected": -0.12239464372396469,
      "logps/chosen": -275.9725036621094,
      "logps/rejected": -335.3379821777344,
      "loss": 0.3601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7814671993255615,
      "rewards/margins": 1.5661503076553345,
      "rewards/rejected": -3.3476173877716064,
      "step": 11110
    },
    {
      "epoch": 1.453946184673264,
      "grad_norm": 7.8119354248046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2920433282852173,
      "logits/rejected": -0.27056050300598145,
      "logps/chosen": -554.33056640625,
      "logps/rejected": -478.9676208496094,
      "loss": 0.2652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.570723533630371,
      "rewards/margins": 2.0118777751922607,
      "rewards/rejected": -3.582601547241211,
      "step": 11111
    },
    {
      "epoch": 1.4540770426106158,
      "grad_norm": 7.550558567047119,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06254908442497253,
      "logits/rejected": 0.05015752464532852,
      "logps/chosen": -413.2053527832031,
      "logps/rejected": -423.6324462890625,
      "loss": 0.5033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8813127875328064,
      "rewards/margins": 1.2613425254821777,
      "rewards/rejected": -2.14265513420105,
      "step": 11112
    },
    {
      "epoch": 1.4542079005479676,
      "grad_norm": 12.200172424316406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052501045167446136,
      "logits/rejected": -0.33168482780456543,
      "logps/chosen": -581.1665649414062,
      "logps/rejected": -376.43829345703125,
      "loss": 0.4727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4788122177124023,
      "rewards/margins": 1.6356995105743408,
      "rewards/rejected": -4.114511966705322,
      "step": 11113
    },
    {
      "epoch": 1.4543387584853193,
      "grad_norm": 4.648159503936768,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43993693590164185,
      "logits/rejected": -0.3682555556297302,
      "logps/chosen": -624.62060546875,
      "logps/rejected": -603.6764526367188,
      "loss": 0.1982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8955608010292053,
      "rewards/margins": 2.2348926067352295,
      "rewards/rejected": -3.130453586578369,
      "step": 11114
    },
    {
      "epoch": 1.4544696164226711,
      "grad_norm": 5.703998565673828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23564667999744415,
      "logits/rejected": 0.0328180231153965,
      "logps/chosen": -379.7976379394531,
      "logps/rejected": -653.3461303710938,
      "loss": 0.2237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9904451370239258,
      "rewards/margins": 2.2808594703674316,
      "rewards/rejected": -4.271304607391357,
      "step": 11115
    },
    {
      "epoch": 1.454600474360023,
      "grad_norm": 4.9046831130981445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35422176122665405,
      "logits/rejected": -0.2163427472114563,
      "logps/chosen": -379.27850341796875,
      "logps/rejected": -409.03277587890625,
      "loss": 0.2201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4963877201080322,
      "rewards/margins": 1.7446112632751465,
      "rewards/rejected": -3.240999221801758,
      "step": 11116
    },
    {
      "epoch": 1.4547313322973747,
      "grad_norm": 8.908684730529785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36015772819519043,
      "logits/rejected": -0.0992204025387764,
      "logps/chosen": -376.29669189453125,
      "logps/rejected": -361.3887023925781,
      "loss": 0.4464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1192920207977295,
      "rewards/margins": 2.065476655960083,
      "rewards/rejected": -3.1847684383392334,
      "step": 11117
    },
    {
      "epoch": 1.4548621902347265,
      "grad_norm": 10.053426742553711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1562662124633789,
      "logits/rejected": -0.23955094814300537,
      "logps/chosen": -456.2718200683594,
      "logps/rejected": -433.8974609375,
      "loss": 0.5592,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2377067804336548,
      "rewards/margins": 1.0783098936080933,
      "rewards/rejected": -2.316016674041748,
      "step": 11118
    },
    {
      "epoch": 1.4549930481720783,
      "grad_norm": 7.147290229797363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34683409333229065,
      "logits/rejected": -0.4608086347579956,
      "logps/chosen": -330.49884033203125,
      "logps/rejected": -296.4340515136719,
      "loss": 0.5305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4163174629211426,
      "rewards/margins": 0.9950871467590332,
      "rewards/rejected": -3.411404848098755,
      "step": 11119
    },
    {
      "epoch": 1.4551239061094299,
      "grad_norm": 7.955088138580322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004235249012708664,
      "logits/rejected": -0.4349583685398102,
      "logps/chosen": -636.1676025390625,
      "logps/rejected": -532.6737060546875,
      "loss": 0.3443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9118173122406006,
      "rewards/margins": 1.6296616792678833,
      "rewards/rejected": -3.5414786338806152,
      "step": 11120
    },
    {
      "epoch": 1.4552547640467817,
      "grad_norm": 8.78221321105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00913267582654953,
      "logits/rejected": -0.10748635977506638,
      "logps/chosen": -363.337158203125,
      "logps/rejected": -391.17352294921875,
      "loss": 0.4622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.186895728111267,
      "rewards/margins": 1.0187022686004639,
      "rewards/rejected": -2.2055981159210205,
      "step": 11121
    },
    {
      "epoch": 1.4553856219841335,
      "grad_norm": 11.477068901062012,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24732518196105957,
      "logits/rejected": -0.034152738749980927,
      "logps/chosen": -592.7587890625,
      "logps/rejected": -587.07177734375,
      "loss": 0.5043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.044317603111267,
      "rewards/margins": 1.7683486938476562,
      "rewards/rejected": -2.812666177749634,
      "step": 11122
    },
    {
      "epoch": 1.4555164799214853,
      "grad_norm": 10.714188575744629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1641824096441269,
      "logits/rejected": -0.050050217658281326,
      "logps/chosen": -484.7768249511719,
      "logps/rejected": -418.8912353515625,
      "loss": 0.659,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7103066444396973,
      "rewards/margins": 0.2770429849624634,
      "rewards/rejected": -1.9873496294021606,
      "step": 11123
    },
    {
      "epoch": 1.455647337858837,
      "grad_norm": 8.788383483886719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.129506915807724,
      "logits/rejected": -0.2844911515712738,
      "logps/chosen": -503.7547607421875,
      "logps/rejected": -414.1601867675781,
      "loss": 0.3204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45090216398239136,
      "rewards/margins": 1.724116325378418,
      "rewards/rejected": -2.175018787384033,
      "step": 11124
    },
    {
      "epoch": 1.4557781957961888,
      "grad_norm": 10.08033561706543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.338667631149292,
      "logits/rejected": -0.4651240408420563,
      "logps/chosen": -699.4418334960938,
      "logps/rejected": -436.35076904296875,
      "loss": 0.4853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0069738626480103,
      "rewards/margins": 0.7431418895721436,
      "rewards/rejected": -1.7501157522201538,
      "step": 11125
    },
    {
      "epoch": 1.4559090537335404,
      "grad_norm": 10.872591972351074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23882804811000824,
      "logits/rejected": -0.4058694541454315,
      "logps/chosen": -432.7563781738281,
      "logps/rejected": -447.9906311035156,
      "loss": 0.4156,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8589798212051392,
      "rewards/margins": 1.3215683698654175,
      "rewards/rejected": -3.1805479526519775,
      "step": 11126
    },
    {
      "epoch": 1.4560399116708922,
      "grad_norm": 10.206263542175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3473016619682312,
      "logits/rejected": -0.44896429777145386,
      "logps/chosen": -419.67779541015625,
      "logps/rejected": -349.59033203125,
      "loss": 0.6134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2809971570968628,
      "rewards/margins": 0.5241663455963135,
      "rewards/rejected": -1.8051636219024658,
      "step": 11127
    },
    {
      "epoch": 1.456170769608244,
      "grad_norm": 10.261151313781738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11652281135320663,
      "logits/rejected": -0.1521519273519516,
      "logps/chosen": -377.65802001953125,
      "logps/rejected": -384.8818664550781,
      "loss": 0.4694,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7120037078857422,
      "rewards/margins": 1.400189757347107,
      "rewards/rejected": -3.1121933460235596,
      "step": 11128
    },
    {
      "epoch": 1.4563016275455958,
      "grad_norm": 7.771570682525635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27827784419059753,
      "logits/rejected": -0.28272271156311035,
      "logps/chosen": -359.84613037109375,
      "logps/rejected": -268.6372375488281,
      "loss": 0.6708,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2853024005889893,
      "rewards/margins": 0.543667197227478,
      "rewards/rejected": -1.8289695978164673,
      "step": 11129
    },
    {
      "epoch": 1.4564324854829476,
      "grad_norm": 5.698550701141357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33919471502304077,
      "logits/rejected": -0.5353145003318787,
      "logps/chosen": -500.3633117675781,
      "logps/rejected": -507.0039367675781,
      "loss": 0.1842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.3341426849365234,
      "rewards/margins": 2.3505666255950928,
      "rewards/rejected": -4.684709548950195,
      "step": 11130
    },
    {
      "epoch": 1.4565633434202994,
      "grad_norm": 5.800743579864502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.291155070066452,
      "logits/rejected": -0.27616196870803833,
      "logps/chosen": -389.32537841796875,
      "logps/rejected": -344.6116638183594,
      "loss": 0.3152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5875205993652344,
      "rewards/margins": 1.643470287322998,
      "rewards/rejected": -3.2309908866882324,
      "step": 11131
    },
    {
      "epoch": 1.4566942013576512,
      "grad_norm": 5.910917282104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17307710647583008,
      "logits/rejected": -0.10269718617200851,
      "logps/chosen": -717.136962890625,
      "logps/rejected": -598.848388671875,
      "loss": 0.2238,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8353350162506104,
      "rewards/margins": 2.348515033721924,
      "rewards/rejected": -3.183850049972534,
      "step": 11132
    },
    {
      "epoch": 1.456825059295003,
      "grad_norm": 3.826801061630249,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1832144856452942,
      "logits/rejected": -0.29600581526756287,
      "logps/chosen": -320.4296875,
      "logps/rejected": -411.12738037109375,
      "loss": 0.2118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0007030963897705,
      "rewards/margins": 2.192091941833496,
      "rewards/rejected": -3.1927947998046875,
      "step": 11133
    },
    {
      "epoch": 1.4569559172323547,
      "grad_norm": 10.272628784179688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1042204350233078,
      "logits/rejected": -0.1844177544116974,
      "logps/chosen": -669.3832397460938,
      "logps/rejected": -497.8974609375,
      "loss": 0.5571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5991724133491516,
      "rewards/margins": 0.8267277479171753,
      "rewards/rejected": -1.4258999824523926,
      "step": 11134
    },
    {
      "epoch": 1.4570867751697063,
      "grad_norm": 9.163973808288574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1751750260591507,
      "logits/rejected": -0.045750997960567474,
      "logps/chosen": -445.09283447265625,
      "logps/rejected": -440.3890380859375,
      "loss": 0.5612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.448472499847412,
      "rewards/margins": 0.4413973391056061,
      "rewards/rejected": -1.8898698091506958,
      "step": 11135
    },
    {
      "epoch": 1.457217633107058,
      "grad_norm": 9.337495803833008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1122860461473465,
      "logits/rejected": -0.17260879278182983,
      "logps/chosen": -391.7860107421875,
      "logps/rejected": -360.9803161621094,
      "loss": 0.6561,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.16988205909729,
      "rewards/margins": 1.2711695432662964,
      "rewards/rejected": -2.441051483154297,
      "step": 11136
    },
    {
      "epoch": 1.4573484910444099,
      "grad_norm": 9.28403377532959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2546592354774475,
      "logits/rejected": -0.3007703423500061,
      "logps/chosen": -381.8235168457031,
      "logps/rejected": -406.8330993652344,
      "loss": 0.492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6449897289276123,
      "rewards/margins": 1.1874803304672241,
      "rewards/rejected": -2.832470178604126,
      "step": 11137
    },
    {
      "epoch": 1.4574793489817617,
      "grad_norm": 6.329486846923828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05613851174712181,
      "logits/rejected": -0.05349288135766983,
      "logps/chosen": -278.773681640625,
      "logps/rejected": -332.3457336425781,
      "loss": 0.2801,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7850140929222107,
      "rewards/margins": 1.937375783920288,
      "rewards/rejected": -2.7223899364471436,
      "step": 11138
    },
    {
      "epoch": 1.4576102069191135,
      "grad_norm": 13.627309799194336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.285474568605423,
      "logits/rejected": -0.09970691800117493,
      "logps/chosen": -489.5066833496094,
      "logps/rejected": -514.21484375,
      "loss": 0.7959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.353643774986267,
      "rewards/margins": 1.5882017612457275,
      "rewards/rejected": -2.941845417022705,
      "step": 11139
    },
    {
      "epoch": 1.4577410648564653,
      "grad_norm": 6.964910507202148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08874498307704926,
      "logits/rejected": -0.34426257014274597,
      "logps/chosen": -454.2369079589844,
      "logps/rejected": -360.50335693359375,
      "loss": 0.3349,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5222307443618774,
      "rewards/margins": 1.260080099105835,
      "rewards/rejected": -2.782310962677002,
      "step": 11140
    },
    {
      "epoch": 1.4578719227938168,
      "grad_norm": 5.934709548950195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01763182133436203,
      "logits/rejected": -0.30136874318122864,
      "logps/chosen": -434.7649230957031,
      "logps/rejected": -411.6507568359375,
      "loss": 0.3323,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7674541473388672,
      "rewards/margins": 1.314089298248291,
      "rewards/rejected": -2.081543445587158,
      "step": 11141
    },
    {
      "epoch": 1.4580027807311686,
      "grad_norm": 14.286545753479004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28725120425224304,
      "logits/rejected": -0.3911186456680298,
      "logps/chosen": -767.58740234375,
      "logps/rejected": -441.43609619140625,
      "loss": 0.843,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.2715718746185303,
      "rewards/margins": 0.149806410074234,
      "rewards/rejected": -2.4213781356811523,
      "step": 11142
    },
    {
      "epoch": 1.4581336386685204,
      "grad_norm": 9.616604804992676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0018874704837799072,
      "logits/rejected": -0.2557314932346344,
      "logps/chosen": -534.3853149414062,
      "logps/rejected": -443.818603515625,
      "loss": 0.4913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9845501184463501,
      "rewards/margins": 1.2254241704940796,
      "rewards/rejected": -2.2099740505218506,
      "step": 11143
    },
    {
      "epoch": 1.4582644966058722,
      "grad_norm": 9.655926704406738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13310551643371582,
      "logits/rejected": 0.03098546527326107,
      "logps/chosen": -317.00634765625,
      "logps/rejected": -393.5130920410156,
      "loss": 0.4333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6800352334976196,
      "rewards/margins": 0.8572223782539368,
      "rewards/rejected": -2.537257671356201,
      "step": 11144
    },
    {
      "epoch": 1.458395354543224,
      "grad_norm": 5.349985599517822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15529723465442657,
      "logits/rejected": -0.04668702185153961,
      "logps/chosen": -364.4271240234375,
      "logps/rejected": -346.3229064941406,
      "loss": 0.2699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8671255111694336,
      "rewards/margins": 1.6953133344650269,
      "rewards/rejected": -2.562438726425171,
      "step": 11145
    },
    {
      "epoch": 1.4585262124805758,
      "grad_norm": 7.16911506652832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2503267824649811,
      "logits/rejected": -0.16302677989006042,
      "logps/chosen": -380.4666748046875,
      "logps/rejected": -346.4020690917969,
      "loss": 0.4121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8603696823120117,
      "rewards/margins": 1.1149568557739258,
      "rewards/rejected": -1.975326657295227,
      "step": 11146
    },
    {
      "epoch": 1.4586570704179276,
      "grad_norm": 10.335641860961914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17248517274856567,
      "logits/rejected": -0.010818654671311378,
      "logps/chosen": -552.4172973632812,
      "logps/rejected": -526.9194946289062,
      "loss": 0.5473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6764506101608276,
      "rewards/margins": 0.8742509484291077,
      "rewards/rejected": -1.5507014989852905,
      "step": 11147
    },
    {
      "epoch": 1.4587879283552794,
      "grad_norm": 9.797703742980957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25665080547332764,
      "logits/rejected": -0.1474718153476715,
      "logps/chosen": -325.52557373046875,
      "logps/rejected": -414.6158447265625,
      "loss": 0.5641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.468763530254364,
      "rewards/margins": 0.9273509979248047,
      "rewards/rejected": -1.396114468574524,
      "step": 11148
    },
    {
      "epoch": 1.4589187862926312,
      "grad_norm": 9.368257522583008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11875006556510925,
      "logits/rejected": -0.14080995321273804,
      "logps/chosen": -688.8397827148438,
      "logps/rejected": -635.58251953125,
      "loss": 0.4963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7807072401046753,
      "rewards/margins": 1.0065269470214844,
      "rewards/rejected": -1.7872340679168701,
      "step": 11149
    },
    {
      "epoch": 1.459049644229983,
      "grad_norm": 7.825669288635254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.4204060435295105,
      "logits/rejected": 0.4035075902938843,
      "logps/chosen": -673.228515625,
      "logps/rejected": -664.5714111328125,
      "loss": 0.3526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4823176860809326,
      "rewards/margins": 1.488980770111084,
      "rewards/rejected": -2.9712984561920166,
      "step": 11150
    },
    {
      "epoch": 1.4591805021673345,
      "grad_norm": 5.551901817321777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0898667648434639,
      "logits/rejected": 0.008587440475821495,
      "logps/chosen": -522.6106567382812,
      "logps/rejected": -523.6160888671875,
      "loss": 0.2521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1817950010299683,
      "rewards/margins": 2.193833827972412,
      "rewards/rejected": -3.3756284713745117,
      "step": 11151
    },
    {
      "epoch": 1.4593113601046863,
      "grad_norm": 8.294718742370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013420140370726585,
      "logits/rejected": 0.08896571397781372,
      "logps/chosen": -511.8155212402344,
      "logps/rejected": -530.4126586914062,
      "loss": 0.739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2877386808395386,
      "rewards/margins": 1.2282986640930176,
      "rewards/rejected": -2.516037702560425,
      "step": 11152
    },
    {
      "epoch": 1.459442218042038,
      "grad_norm": 3.569186210632324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18180646002292633,
      "logits/rejected": -0.41511595249176025,
      "logps/chosen": -575.1109008789062,
      "logps/rejected": -368.68780517578125,
      "loss": 0.1999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45874521136283875,
      "rewards/margins": 2.1852376461029053,
      "rewards/rejected": -2.6439828872680664,
      "step": 11153
    },
    {
      "epoch": 1.45957307597939,
      "grad_norm": 10.24224853515625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.061734817922115326,
      "logits/rejected": -0.2879900634288788,
      "logps/chosen": -502.96905517578125,
      "logps/rejected": -421.005615234375,
      "loss": 0.5003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1743478775024414,
      "rewards/margins": 1.9841506481170654,
      "rewards/rejected": -3.158498764038086,
      "step": 11154
    },
    {
      "epoch": 1.4597039339167417,
      "grad_norm": 9.155525207519531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10353703796863556,
      "logits/rejected": 0.09581981599330902,
      "logps/chosen": -363.8329772949219,
      "logps/rejected": -480.91558837890625,
      "loss": 0.5476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0722010135650635,
      "rewards/margins": 1.272982120513916,
      "rewards/rejected": -2.3451831340789795,
      "step": 11155
    },
    {
      "epoch": 1.4598347918540935,
      "grad_norm": 3.5445780754089355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19805870950222015,
      "logits/rejected": 0.04982074350118637,
      "logps/chosen": -520.433837890625,
      "logps/rejected": -400.513916015625,
      "loss": 0.2048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19818134605884552,
      "rewards/margins": 2.532623529434204,
      "rewards/rejected": -2.730804681777954,
      "step": 11156
    },
    {
      "epoch": 1.459965649791445,
      "grad_norm": 6.923519611358643,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20196956396102905,
      "logits/rejected": -0.2022051066160202,
      "logps/chosen": -625.4566650390625,
      "logps/rejected": -520.9815673828125,
      "loss": 0.3116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2971492409706116,
      "rewards/margins": 1.5742827653884888,
      "rewards/rejected": -1.8714319467544556,
      "step": 11157
    },
    {
      "epoch": 1.4600965077287968,
      "grad_norm": 6.326293468475342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07698966562747955,
      "logits/rejected": -0.10149968415498734,
      "logps/chosen": -489.34722900390625,
      "logps/rejected": -418.3812255859375,
      "loss": 0.2342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7133377194404602,
      "rewards/margins": 3.200930595397949,
      "rewards/rejected": -3.9142684936523438,
      "step": 11158
    },
    {
      "epoch": 1.4602273656661486,
      "grad_norm": 8.017742156982422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11608120054006577,
      "logits/rejected": -0.027752289548516273,
      "logps/chosen": -356.2096862792969,
      "logps/rejected": -376.04595947265625,
      "loss": 0.3196,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3285396695137024,
      "rewards/margins": 1.7270774841308594,
      "rewards/rejected": -2.055617094039917,
      "step": 11159
    },
    {
      "epoch": 1.4603582236035004,
      "grad_norm": 7.1370930671691895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07020889222621918,
      "logits/rejected": -0.10912831127643585,
      "logps/chosen": -605.3786010742188,
      "logps/rejected": -581.2109375,
      "loss": 0.5347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2001444101333618,
      "rewards/margins": 0.4509143531322479,
      "rewards/rejected": -1.6510587930679321,
      "step": 11160
    },
    {
      "epoch": 1.4604890815408522,
      "grad_norm": 8.735858917236328,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04210001975297928,
      "logits/rejected": -0.42604169249534607,
      "logps/chosen": -559.0224609375,
      "logps/rejected": -421.0030517578125,
      "loss": 0.5324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1135029792785645,
      "rewards/margins": 1.2157560586929321,
      "rewards/rejected": -2.329258918762207,
      "step": 11161
    },
    {
      "epoch": 1.460619939478204,
      "grad_norm": 4.964788913726807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14196260273456573,
      "logits/rejected": -0.2854331135749817,
      "logps/chosen": -369.41668701171875,
      "logps/rejected": -326.5623779296875,
      "loss": 0.2329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7787197828292847,
      "rewards/margins": 1.8175750970840454,
      "rewards/rejected": -2.596294641494751,
      "step": 11162
    },
    {
      "epoch": 1.4607507974155558,
      "grad_norm": 10.218741416931152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33028191328048706,
      "logits/rejected": -0.253307044506073,
      "logps/chosen": -426.89111328125,
      "logps/rejected": -364.444580078125,
      "loss": 0.4877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3778480291366577,
      "rewards/margins": 1.4403736591339111,
      "rewards/rejected": -2.8182215690612793,
      "step": 11163
    },
    {
      "epoch": 1.4608816553529076,
      "grad_norm": 6.2572855949401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2340884655714035,
      "logits/rejected": -0.023840198293328285,
      "logps/chosen": -384.7982482910156,
      "logps/rejected": -417.68292236328125,
      "loss": 0.2894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4976403713226318,
      "rewards/margins": 1.7271077632904053,
      "rewards/rejected": -3.2247486114501953,
      "step": 11164
    },
    {
      "epoch": 1.4610125132902594,
      "grad_norm": 10.280863761901855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2449111044406891,
      "logits/rejected": -0.1869039535522461,
      "logps/chosen": -305.89678955078125,
      "logps/rejected": -485.3934631347656,
      "loss": 0.3617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4338878393173218,
      "rewards/margins": 2.4593238830566406,
      "rewards/rejected": -3.893211603164673,
      "step": 11165
    },
    {
      "epoch": 1.461143371227611,
      "grad_norm": 7.277440071105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4607231020927429,
      "logits/rejected": -0.2640950381755829,
      "logps/chosen": -598.1741943359375,
      "logps/rejected": -526.2763671875,
      "loss": 0.3869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9683939814567566,
      "rewards/margins": 1.4553793668746948,
      "rewards/rejected": -2.4237735271453857,
      "step": 11166
    },
    {
      "epoch": 1.4612742291649627,
      "grad_norm": 12.926850318908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24465572834014893,
      "logits/rejected": -0.27585461735725403,
      "logps/chosen": -641.3903198242188,
      "logps/rejected": -662.3811645507812,
      "loss": 0.7918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9812850952148438,
      "rewards/margins": 0.6202574968338013,
      "rewards/rejected": -2.6015427112579346,
      "step": 11167
    },
    {
      "epoch": 1.4614050871023145,
      "grad_norm": 12.036510467529297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2752518951892853,
      "logits/rejected": -0.19895654916763306,
      "logps/chosen": -490.4417724609375,
      "logps/rejected": -455.836181640625,
      "loss": 0.6482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.302401065826416,
      "rewards/margins": 0.8675451278686523,
      "rewards/rejected": -2.1699461936950684,
      "step": 11168
    },
    {
      "epoch": 1.4615359450396663,
      "grad_norm": 9.396727561950684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.264968603849411,
      "logits/rejected": -0.15247470140457153,
      "logps/chosen": -379.331298828125,
      "logps/rejected": -336.8581848144531,
      "loss": 0.579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.819392442703247,
      "rewards/margins": 0.5990238189697266,
      "rewards/rejected": -2.4184162616729736,
      "step": 11169
    },
    {
      "epoch": 1.4616668029770181,
      "grad_norm": 11.205344200134277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06599132716655731,
      "logits/rejected": -0.08395107090473175,
      "logps/chosen": -290.11431884765625,
      "logps/rejected": -337.8384704589844,
      "loss": 0.6539,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4628679752349854,
      "rewards/margins": 0.6224429607391357,
      "rewards/rejected": -2.085310935974121,
      "step": 11170
    },
    {
      "epoch": 1.46179766091437,
      "grad_norm": 11.900205612182617,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06266892701387405,
      "logits/rejected": -0.04589081183075905,
      "logps/chosen": -525.4205932617188,
      "logps/rejected": -610.9502563476562,
      "loss": 0.684,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4509503841400146,
      "rewards/margins": 0.34315913915634155,
      "rewards/rejected": -1.7941093444824219,
      "step": 11171
    },
    {
      "epoch": 1.4619285188517215,
      "grad_norm": 11.611953735351562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6292531490325928,
      "logits/rejected": -0.6634842157363892,
      "logps/chosen": -478.3271484375,
      "logps/rejected": -548.06298828125,
      "loss": 0.5671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.123852014541626,
      "rewards/margins": 1.4384466409683228,
      "rewards/rejected": -2.562298536300659,
      "step": 11172
    },
    {
      "epoch": 1.4620593767890733,
      "grad_norm": 9.735124588012695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19241401553153992,
      "logits/rejected": 0.09641323238611221,
      "logps/chosen": -514.7769775390625,
      "logps/rejected": -339.86224365234375,
      "loss": 0.46,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0476768016815186,
      "rewards/margins": 1.0333940982818604,
      "rewards/rejected": -2.081070899963379,
      "step": 11173
    },
    {
      "epoch": 1.462190234726425,
      "grad_norm": 10.500650405883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24831822514533997,
      "logits/rejected": -0.12416213005781174,
      "logps/chosen": -517.599609375,
      "logps/rejected": -687.816650390625,
      "loss": 0.676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.081329345703125,
      "rewards/margins": 1.6690518856048584,
      "rewards/rejected": -2.7503814697265625,
      "step": 11174
    },
    {
      "epoch": 1.4623210926637769,
      "grad_norm": 8.665510177612305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06693582236766815,
      "logits/rejected": -0.06691750884056091,
      "logps/chosen": -311.2362365722656,
      "logps/rejected": -321.4377746582031,
      "loss": 0.6481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.08242928981781,
      "rewards/margins": 1.006181001663208,
      "rewards/rejected": -2.0886104106903076,
      "step": 11175
    },
    {
      "epoch": 1.4624519506011286,
      "grad_norm": 8.590511322021484,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22060002386569977,
      "logits/rejected": 0.1150885745882988,
      "logps/chosen": -355.8840637207031,
      "logps/rejected": -385.6632995605469,
      "loss": 0.6092,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5093646049499512,
      "rewards/margins": 0.9075348973274231,
      "rewards/rejected": -1.4168994426727295,
      "step": 11176
    },
    {
      "epoch": 1.4625828085384804,
      "grad_norm": 8.24809455871582,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19451278448104858,
      "logits/rejected": 0.06135178357362747,
      "logps/chosen": -748.1358642578125,
      "logps/rejected": -555.0253295898438,
      "loss": 0.3474,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0605766773223877,
      "rewards/margins": 1.5633434057235718,
      "rewards/rejected": -2.623920202255249,
      "step": 11177
    },
    {
      "epoch": 1.4627136664758322,
      "grad_norm": 7.944300651550293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01731860637664795,
      "logits/rejected": -0.022465776652097702,
      "logps/chosen": -276.6654968261719,
      "logps/rejected": -291.02008056640625,
      "loss": 0.4676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9754331707954407,
      "rewards/margins": 1.258182168006897,
      "rewards/rejected": -2.2336153984069824,
      "step": 11178
    },
    {
      "epoch": 1.462844524413184,
      "grad_norm": 9.498746871948242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11865445971488953,
      "logits/rejected": -0.2493835687637329,
      "logps/chosen": -582.9177856445312,
      "logps/rejected": -535.1614379882812,
      "loss": 0.3952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4174655675888062,
      "rewards/margins": 1.8965473175048828,
      "rewards/rejected": -3.3140130043029785,
      "step": 11179
    },
    {
      "epoch": 1.4629753823505358,
      "grad_norm": 12.201559066772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33344918489456177,
      "logits/rejected": -0.33039310574531555,
      "logps/chosen": -264.98138427734375,
      "logps/rejected": -312.7474365234375,
      "loss": 0.4531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.19527268409729,
      "rewards/margins": 0.9119515419006348,
      "rewards/rejected": -2.107224225997925,
      "step": 11180
    },
    {
      "epoch": 1.4631062402878874,
      "grad_norm": 8.468974113464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2258470356464386,
      "logits/rejected": 0.03826881945133209,
      "logps/chosen": -410.2314453125,
      "logps/rejected": -434.193603515625,
      "loss": 0.415,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.080950379371643,
      "rewards/margins": 1.643765926361084,
      "rewards/rejected": -2.7247164249420166,
      "step": 11181
    },
    {
      "epoch": 1.4632370982252392,
      "grad_norm": 9.95498275756836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11237048357725143,
      "logits/rejected": -0.2566235065460205,
      "logps/chosen": -517.6756591796875,
      "logps/rejected": -484.82958984375,
      "loss": 0.578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6861572265625,
      "rewards/margins": 1.3303476572036743,
      "rewards/rejected": -2.0165047645568848,
      "step": 11182
    },
    {
      "epoch": 1.463367956162591,
      "grad_norm": 3.515808582305908,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09276214241981506,
      "logits/rejected": -0.04883731156587601,
      "logps/chosen": -480.895751953125,
      "logps/rejected": -398.84136962890625,
      "loss": 0.144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8652647137641907,
      "rewards/margins": 2.864471435546875,
      "rewards/rejected": -1.999206781387329,
      "step": 11183
    },
    {
      "epoch": 1.4634988140999428,
      "grad_norm": 11.452065467834473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08021195977926254,
      "logits/rejected": 0.10974109172821045,
      "logps/chosen": -537.417236328125,
      "logps/rejected": -446.3401184082031,
      "loss": 0.6949,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2537155151367188,
      "rewards/margins": 0.8694552183151245,
      "rewards/rejected": -2.123170852661133,
      "step": 11184
    },
    {
      "epoch": 1.4636296720372945,
      "grad_norm": 6.458566665649414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6435781121253967,
      "logits/rejected": -0.5927485227584839,
      "logps/chosen": -598.2681274414062,
      "logps/rejected": -319.9657897949219,
      "loss": 0.2549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7969042062759399,
      "rewards/margins": 2.080056667327881,
      "rewards/rejected": -2.8769607543945312,
      "step": 11185
    },
    {
      "epoch": 1.4637605299746463,
      "grad_norm": 3.6114392280578613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10572895407676697,
      "logits/rejected": -0.22468969225883484,
      "logps/chosen": -333.57421875,
      "logps/rejected": -361.6911926269531,
      "loss": 0.2847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8167266845703125,
      "rewards/margins": 1.6535217761993408,
      "rewards/rejected": -2.4702486991882324,
      "step": 11186
    },
    {
      "epoch": 1.463891387911998,
      "grad_norm": 14.865045547485352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03356153145432472,
      "logits/rejected": -0.12930449843406677,
      "logps/chosen": -475.38751220703125,
      "logps/rejected": -419.6614990234375,
      "loss": 1.1674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.436417818069458,
      "rewards/margins": 0.689932107925415,
      "rewards/rejected": -2.126349925994873,
      "step": 11187
    },
    {
      "epoch": 1.4640222458493497,
      "grad_norm": 10.61774730682373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05008997023105621,
      "logits/rejected": 0.06422296166419983,
      "logps/chosen": -370.23150634765625,
      "logps/rejected": -419.3115539550781,
      "loss": 0.379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6730010509490967,
      "rewards/margins": 1.931388020515442,
      "rewards/rejected": -3.60438871383667,
      "step": 11188
    },
    {
      "epoch": 1.4641531037867015,
      "grad_norm": 5.871667861938477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.017850182950496674,
      "logits/rejected": -0.05998622626066208,
      "logps/chosen": -625.37744140625,
      "logps/rejected": -544.6646118164062,
      "loss": 0.4283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03129236400127411,
      "rewards/margins": 1.1063392162322998,
      "rewards/rejected": -1.1376316547393799,
      "step": 11189
    },
    {
      "epoch": 1.4642839617240533,
      "grad_norm": 14.178323745727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03209942579269409,
      "logits/rejected": 0.09312322735786438,
      "logps/chosen": -395.073486328125,
      "logps/rejected": -488.8529968261719,
      "loss": 1.178,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6962658166885376,
      "rewards/margins": -0.5285158157348633,
      "rewards/rejected": -0.16774997115135193,
      "step": 11190
    },
    {
      "epoch": 1.464414819661405,
      "grad_norm": 4.694228649139404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6299136877059937,
      "logits/rejected": -0.8905944228172302,
      "logps/chosen": -508.245361328125,
      "logps/rejected": -347.4084777832031,
      "loss": 0.3225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.20525324344635,
      "rewards/margins": 1.4798730611801147,
      "rewards/rejected": -2.685126543045044,
      "step": 11191
    },
    {
      "epoch": 1.4645456775987569,
      "grad_norm": 5.208218097686768,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09578956663608551,
      "logits/rejected": -0.16019617021083832,
      "logps/chosen": -578.5972900390625,
      "logps/rejected": -487.51605224609375,
      "loss": 0.2305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03132856637239456,
      "rewards/margins": 3.3126416206359863,
      "rewards/rejected": -3.281312942504883,
      "step": 11192
    },
    {
      "epoch": 1.4646765355361087,
      "grad_norm": 5.800443172454834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5222046375274658,
      "logits/rejected": -0.44819268584251404,
      "logps/chosen": -303.58642578125,
      "logps/rejected": -414.02862548828125,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1415104866027832,
      "rewards/margins": 2.321164131164551,
      "rewards/rejected": -3.462675094604492,
      "step": 11193
    },
    {
      "epoch": 1.4648073934734605,
      "grad_norm": 10.367833137512207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0859755352139473,
      "logits/rejected": -0.39689067006111145,
      "logps/chosen": -448.94561767578125,
      "logps/rejected": -448.6316833496094,
      "loss": 0.4251,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8135957717895508,
      "rewards/margins": 1.4328393936157227,
      "rewards/rejected": -2.2464351654052734,
      "step": 11194
    },
    {
      "epoch": 1.4649382514108122,
      "grad_norm": 6.20897912979126,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07227125763893127,
      "logits/rejected": 0.0025878623127937317,
      "logps/chosen": -507.904052734375,
      "logps/rejected": -502.54193115234375,
      "loss": 0.4423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03330536559224129,
      "rewards/margins": 0.712600588798523,
      "rewards/rejected": -0.745905876159668,
      "step": 11195
    },
    {
      "epoch": 1.465069109348164,
      "grad_norm": 9.104991912841797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15699994564056396,
      "logits/rejected": -0.22002704441547394,
      "logps/chosen": -348.376953125,
      "logps/rejected": -449.8555603027344,
      "loss": 0.5697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7990667819976807,
      "rewards/margins": 0.9615316390991211,
      "rewards/rejected": -1.7605984210968018,
      "step": 11196
    },
    {
      "epoch": 1.4651999672855156,
      "grad_norm": 16.873676300048828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29952260851860046,
      "logits/rejected": -0.11496013402938843,
      "logps/chosen": -640.7103881835938,
      "logps/rejected": -486.21112060546875,
      "loss": 1.2667,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.2668215036392212,
      "rewards/margins": -0.7312215566635132,
      "rewards/rejected": -0.535599946975708,
      "step": 11197
    },
    {
      "epoch": 1.4653308252228674,
      "grad_norm": 6.03058385848999,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046204499900341034,
      "logits/rejected": -0.23492272198200226,
      "logps/chosen": -575.2913818359375,
      "logps/rejected": -657.429931640625,
      "loss": 0.2324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24873925745487213,
      "rewards/margins": 2.3759586811065674,
      "rewards/rejected": -2.1272196769714355,
      "step": 11198
    },
    {
      "epoch": 1.4654616831602192,
      "grad_norm": 9.535428047180176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05637243390083313,
      "logits/rejected": -0.07096205651760101,
      "logps/chosen": -487.2680969238281,
      "logps/rejected": -462.8243408203125,
      "loss": 0.4311,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4512407183647156,
      "rewards/margins": 1.4807155132293701,
      "rewards/rejected": -1.9319562911987305,
      "step": 11199
    },
    {
      "epoch": 1.465592541097571,
      "grad_norm": 6.602407455444336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2207411527633667,
      "logits/rejected": -0.3806004822254181,
      "logps/chosen": -558.4266357421875,
      "logps/rejected": -465.3284606933594,
      "loss": 0.3213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46035274863243103,
      "rewards/margins": 1.5479695796966553,
      "rewards/rejected": -2.008322238922119,
      "step": 11200
    },
    {
      "epoch": 1.4657233990349228,
      "grad_norm": 5.998157978057861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12528447806835175,
      "logits/rejected": -0.3829633295536041,
      "logps/chosen": -411.4720153808594,
      "logps/rejected": -345.3009338378906,
      "loss": 0.3448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5152205228805542,
      "rewards/margins": 1.4487701654434204,
      "rewards/rejected": -1.9639906883239746,
      "step": 11201
    },
    {
      "epoch": 1.4658542569722743,
      "grad_norm": 10.091780662536621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05912851169705391,
      "logits/rejected": -0.14703112840652466,
      "logps/chosen": -431.0472106933594,
      "logps/rejected": -548.4071044921875,
      "loss": 0.595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9154361486434937,
      "rewards/margins": 1.8576664924621582,
      "rewards/rejected": -2.7731027603149414,
      "step": 11202
    },
    {
      "epoch": 1.4659851149096261,
      "grad_norm": 4.802576065063477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.144928976893425,
      "logits/rejected": -0.14957429468631744,
      "logps/chosen": -579.107666015625,
      "logps/rejected": -376.973876953125,
      "loss": 0.2514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8607480525970459,
      "rewards/margins": 1.546539306640625,
      "rewards/rejected": -2.407287359237671,
      "step": 11203
    },
    {
      "epoch": 1.466115972846978,
      "grad_norm": 7.43477725982666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4019160270690918,
      "logits/rejected": -0.20739558339118958,
      "logps/chosen": -451.27508544921875,
      "logps/rejected": -487.5826416015625,
      "loss": 0.335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11635438352823257,
      "rewards/margins": 1.887963891029358,
      "rewards/rejected": -1.7716095447540283,
      "step": 11204
    },
    {
      "epoch": 1.4662468307843297,
      "grad_norm": 5.7382025718688965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.160823792219162,
      "logits/rejected": -0.31368499994277954,
      "logps/chosen": -388.330078125,
      "logps/rejected": -381.7829284667969,
      "loss": 0.3499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6276252269744873,
      "rewards/margins": 1.6053600311279297,
      "rewards/rejected": -2.232985258102417,
      "step": 11205
    },
    {
      "epoch": 1.4663776887216815,
      "grad_norm": 10.083992004394531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22061172127723694,
      "logits/rejected": -0.19656625390052795,
      "logps/chosen": -459.945556640625,
      "logps/rejected": -506.93408203125,
      "loss": 0.6551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7847381830215454,
      "rewards/margins": 0.42659080028533936,
      "rewards/rejected": -2.2113287448883057,
      "step": 11206
    },
    {
      "epoch": 1.4665085466590333,
      "grad_norm": 9.293902397155762,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13996559381484985,
      "logits/rejected": 0.37111085653305054,
      "logps/chosen": -409.6507568359375,
      "logps/rejected": -502.9126281738281,
      "loss": 0.7548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.125040665268898,
      "rewards/margins": 1.3315578699111938,
      "rewards/rejected": -1.206517219543457,
      "step": 11207
    },
    {
      "epoch": 1.466639404596385,
      "grad_norm": 14.891067504882812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19789168238639832,
      "logits/rejected": 0.035015128552913666,
      "logps/chosen": -383.297607421875,
      "logps/rejected": -338.80450439453125,
      "loss": 0.7934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7740187048912048,
      "rewards/margins": 0.42855989933013916,
      "rewards/rejected": -1.2025785446166992,
      "step": 11208
    },
    {
      "epoch": 1.4667702625337369,
      "grad_norm": 8.519261360168457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08351342380046844,
      "logits/rejected": -0.0834418535232544,
      "logps/chosen": -722.3990478515625,
      "logps/rejected": -690.10791015625,
      "loss": 0.4914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6107671856880188,
      "rewards/margins": 0.8332455158233643,
      "rewards/rejected": -1.4440126419067383,
      "step": 11209
    },
    {
      "epoch": 1.4669011204710887,
      "grad_norm": 4.656375885009766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.404135525226593,
      "logits/rejected": -0.26270976662635803,
      "logps/chosen": -429.025634765625,
      "logps/rejected": -360.23614501953125,
      "loss": 0.2257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.245081901550293,
      "rewards/margins": 1.9844088554382324,
      "rewards/rejected": -3.2294907569885254,
      "step": 11210
    },
    {
      "epoch": 1.4670319784084405,
      "grad_norm": 7.403291702270508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053165361285209656,
      "logits/rejected": -0.23016774654388428,
      "logps/chosen": -552.9361572265625,
      "logps/rejected": -536.8292236328125,
      "loss": 0.2707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33742526173591614,
      "rewards/margins": 1.772931694984436,
      "rewards/rejected": -2.1103568077087402,
      "step": 11211
    },
    {
      "epoch": 1.467162836345792,
      "grad_norm": 6.499310493469238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23942212760448456,
      "logits/rejected": -0.46915483474731445,
      "logps/chosen": -575.6326904296875,
      "logps/rejected": -600.8067626953125,
      "loss": 0.3304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1378790140151978,
      "rewards/margins": 1.3297662734985352,
      "rewards/rejected": -2.4676451683044434,
      "step": 11212
    },
    {
      "epoch": 1.4672936942831438,
      "grad_norm": 6.275931358337402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1985863447189331,
      "logits/rejected": 0.32044631242752075,
      "logps/chosen": -580.79345703125,
      "logps/rejected": -610.8192138671875,
      "loss": 0.2774,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14852045476436615,
      "rewards/margins": 1.7297182083129883,
      "rewards/rejected": -1.581197738647461,
      "step": 11213
    },
    {
      "epoch": 1.4674245522204956,
      "grad_norm": 9.872468948364258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25172439217567444,
      "logits/rejected": -0.33324605226516724,
      "logps/chosen": -462.22845458984375,
      "logps/rejected": -381.8778991699219,
      "loss": 0.4636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7864494919776917,
      "rewards/margins": 1.3015780448913574,
      "rewards/rejected": -2.088027238845825,
      "step": 11214
    },
    {
      "epoch": 1.4675554101578474,
      "grad_norm": 13.702776908874512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.254313588142395,
      "logits/rejected": -0.35529929399490356,
      "logps/chosen": -317.5290222167969,
      "logps/rejected": -260.8739013671875,
      "loss": 0.9831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4247807264328003,
      "rewards/margins": 0.2991017699241638,
      "rewards/rejected": -1.7238826751708984,
      "step": 11215
    },
    {
      "epoch": 1.4676862680951992,
      "grad_norm": 5.3525614738464355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012844040989875793,
      "logits/rejected": 0.11889246106147766,
      "logps/chosen": -456.28314208984375,
      "logps/rejected": -424.09686279296875,
      "loss": 0.244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5198789238929749,
      "rewards/margins": 1.7592493295669556,
      "rewards/rejected": -2.279128074645996,
      "step": 11216
    },
    {
      "epoch": 1.467817126032551,
      "grad_norm": 7.1486334800720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39941373467445374,
      "logits/rejected": -0.3066772520542145,
      "logps/chosen": -336.29681396484375,
      "logps/rejected": -459.0611572265625,
      "loss": 0.4149,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.230400562286377,
      "rewards/margins": 1.363120198249817,
      "rewards/rejected": -2.5935208797454834,
      "step": 11217
    },
    {
      "epoch": 1.4679479839699026,
      "grad_norm": 9.091292381286621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3994653820991516,
      "logits/rejected": -0.1786227524280548,
      "logps/chosen": -623.493408203125,
      "logps/rejected": -683.8516845703125,
      "loss": 0.5188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011142522096633911,
      "rewards/margins": 1.000261664390564,
      "rewards/rejected": -1.011404037475586,
      "step": 11218
    },
    {
      "epoch": 1.4680788419072544,
      "grad_norm": 8.511249542236328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40253904461860657,
      "logits/rejected": -0.5416831970214844,
      "logps/chosen": -342.91015625,
      "logps/rejected": -299.5672607421875,
      "loss": 0.5224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3014717102050781,
      "rewards/margins": 0.8827717304229736,
      "rewards/rejected": -2.1842434406280518,
      "step": 11219
    },
    {
      "epoch": 1.4682096998446061,
      "grad_norm": 7.397639274597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005923910066485405,
      "logits/rejected": -0.10588784515857697,
      "logps/chosen": -471.5645446777344,
      "logps/rejected": -437.8370056152344,
      "loss": 0.4441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13392312824726105,
      "rewards/margins": 2.1823272705078125,
      "rewards/rejected": -2.0484039783477783,
      "step": 11220
    },
    {
      "epoch": 1.468340557781958,
      "grad_norm": 7.530636787414551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12904034554958344,
      "logits/rejected": -0.3223317563533783,
      "logps/chosen": -453.44024658203125,
      "logps/rejected": -427.2831726074219,
      "loss": 0.3424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5433115363121033,
      "rewards/margins": 1.699871301651001,
      "rewards/rejected": -2.243182897567749,
      "step": 11221
    },
    {
      "epoch": 1.4684714157193097,
      "grad_norm": 6.7660603523254395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19233523309230804,
      "logits/rejected": -0.11898911744356155,
      "logps/chosen": -350.0912170410156,
      "logps/rejected": -348.9750061035156,
      "loss": 0.3571,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9434192776679993,
      "rewards/margins": 1.1301952600479126,
      "rewards/rejected": -2.0736145973205566,
      "step": 11222
    },
    {
      "epoch": 1.4686022736566615,
      "grad_norm": 14.836645126342773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16891519725322723,
      "logits/rejected": -0.23160354793071747,
      "logps/chosen": -626.9526977539062,
      "logps/rejected": -547.765380859375,
      "loss": 0.955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9615570306777954,
      "rewards/margins": 0.34313684701919556,
      "rewards/rejected": -2.3046939373016357,
      "step": 11223
    },
    {
      "epoch": 1.4687331315940133,
      "grad_norm": 10.092964172363281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12067914009094238,
      "logits/rejected": -0.4366298019886017,
      "logps/chosen": -571.2918090820312,
      "logps/rejected": -507.965087890625,
      "loss": 0.3478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1168060302734375,
      "rewards/margins": 1.2832896709442139,
      "rewards/rejected": -2.4000957012176514,
      "step": 11224
    },
    {
      "epoch": 1.468863989531365,
      "grad_norm": 6.6863813400268555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03708972781896591,
      "logits/rejected": -0.07863542437553406,
      "logps/chosen": -396.4169006347656,
      "logps/rejected": -384.5532531738281,
      "loss": 0.4371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8483136892318726,
      "rewards/margins": 1.093552827835083,
      "rewards/rejected": -1.941866397857666,
      "step": 11225
    },
    {
      "epoch": 1.468994847468717,
      "grad_norm": 5.518030643463135,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06030552089214325,
      "logits/rejected": 0.05730913579463959,
      "logps/chosen": -485.96038818359375,
      "logps/rejected": -394.1640625,
      "loss": 0.2904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13372956216335297,
      "rewards/margins": 1.569530725479126,
      "rewards/rejected": -1.7032603025436401,
      "step": 11226
    },
    {
      "epoch": 1.4691257054060685,
      "grad_norm": 3.7336254119873047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03405536711215973,
      "logits/rejected": 0.05736874043941498,
      "logps/chosen": -484.43963623046875,
      "logps/rejected": -454.8819580078125,
      "loss": 0.1219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5639339089393616,
      "rewards/margins": 2.32902193069458,
      "rewards/rejected": -2.892956018447876,
      "step": 11227
    },
    {
      "epoch": 1.4692565633434203,
      "grad_norm": 6.384886741638184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013074949383735657,
      "logits/rejected": -0.18685567378997803,
      "logps/chosen": -529.338623046875,
      "logps/rejected": -510.795654296875,
      "loss": 0.4195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5914365649223328,
      "rewards/margins": 1.1408450603485107,
      "rewards/rejected": -1.7322816848754883,
      "step": 11228
    },
    {
      "epoch": 1.469387421280772,
      "grad_norm": 11.177929878234863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11723476648330688,
      "logits/rejected": -0.1831817626953125,
      "logps/chosen": -501.66204833984375,
      "logps/rejected": -435.2944030761719,
      "loss": 0.4695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27513161301612854,
      "rewards/margins": 1.3068939447402954,
      "rewards/rejected": -1.5820255279541016,
      "step": 11229
    },
    {
      "epoch": 1.4695182792181238,
      "grad_norm": 9.452167510986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15077456831932068,
      "logits/rejected": -0.20419499278068542,
      "logps/chosen": -302.6603088378906,
      "logps/rejected": -340.275390625,
      "loss": 0.5993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9837102293968201,
      "rewards/margins": 0.5597836375236511,
      "rewards/rejected": -1.5434938669204712,
      "step": 11230
    },
    {
      "epoch": 1.4696491371554756,
      "grad_norm": 8.263873100280762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2103639841079712,
      "logits/rejected": -0.0790988951921463,
      "logps/chosen": -348.1144104003906,
      "logps/rejected": -448.5118713378906,
      "loss": 0.4153,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5136539936065674,
      "rewards/margins": 0.872654914855957,
      "rewards/rejected": -2.3863089084625244,
      "step": 11231
    },
    {
      "epoch": 1.4697799950928274,
      "grad_norm": 7.942692279815674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19861020147800446,
      "logits/rejected": -0.1805712729692459,
      "logps/chosen": -425.5830078125,
      "logps/rejected": -463.291259765625,
      "loss": 0.4638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2019510269165039,
      "rewards/margins": 0.8523074984550476,
      "rewards/rejected": -0.6503564715385437,
      "step": 11232
    },
    {
      "epoch": 1.469910853030179,
      "grad_norm": 7.418036460876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09487111866474152,
      "logits/rejected": -0.09103111922740936,
      "logps/chosen": -377.51611328125,
      "logps/rejected": -569.1405639648438,
      "loss": 0.3141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3001747131347656,
      "rewards/margins": 2.575713872909546,
      "rewards/rejected": -2.8758888244628906,
      "step": 11233
    },
    {
      "epoch": 1.4700417109675308,
      "grad_norm": 8.081511497497559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11076559871435165,
      "logits/rejected": -0.3963392376899719,
      "logps/chosen": -508.4385681152344,
      "logps/rejected": -421.1575622558594,
      "loss": 0.3977,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5899776220321655,
      "rewards/margins": 1.5866552591323853,
      "rewards/rejected": -3.176632881164551,
      "step": 11234
    },
    {
      "epoch": 1.4701725689048826,
      "grad_norm": 8.573075294494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1454046368598938,
      "logits/rejected": -0.008664246648550034,
      "logps/chosen": -509.10498046875,
      "logps/rejected": -531.7971801757812,
      "loss": 0.5017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5939054489135742,
      "rewards/margins": 1.113878607749939,
      "rewards/rejected": -1.7077841758728027,
      "step": 11235
    },
    {
      "epoch": 1.4703034268422344,
      "grad_norm": 7.410531520843506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0995945855975151,
      "logits/rejected": 0.14322058856487274,
      "logps/chosen": -508.79791259765625,
      "logps/rejected": -492.3960266113281,
      "loss": 0.2966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.563862681388855,
      "rewards/margins": 1.4291486740112305,
      "rewards/rejected": -1.9930113554000854,
      "step": 11236
    },
    {
      "epoch": 1.4704342847795862,
      "grad_norm": 6.928552150726318,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06961230933666229,
      "logits/rejected": -0.2427484095096588,
      "logps/chosen": -464.0119934082031,
      "logps/rejected": -490.8641357421875,
      "loss": 0.2525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2824643850326538,
      "rewards/margins": 1.7547681331634521,
      "rewards/rejected": -2.0372326374053955,
      "step": 11237
    },
    {
      "epoch": 1.470565142716938,
      "grad_norm": 8.490272521972656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1900968849658966,
      "logits/rejected": -0.15274573862552643,
      "logps/chosen": -649.1812744140625,
      "logps/rejected": -602.493408203125,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0745952129364014,
      "rewards/margins": 1.499380111694336,
      "rewards/rejected": -2.5739753246307373,
      "step": 11238
    },
    {
      "epoch": 1.4706960006542897,
      "grad_norm": 8.542923927307129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12411046773195267,
      "logits/rejected": -0.1278560757637024,
      "logps/chosen": -594.2740478515625,
      "logps/rejected": -452.3784484863281,
      "loss": 0.5386,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5675467252731323,
      "rewards/margins": 0.831359326839447,
      "rewards/rejected": -1.398905873298645,
      "step": 11239
    },
    {
      "epoch": 1.4708268585916415,
      "grad_norm": 7.183689117431641,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1448281705379486,
      "logits/rejected": 0.08099406957626343,
      "logps/chosen": -373.1439514160156,
      "logps/rejected": -465.8767395019531,
      "loss": 0.4068,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26804330945014954,
      "rewards/margins": 1.4475682973861694,
      "rewards/rejected": -1.7156116962432861,
      "step": 11240
    },
    {
      "epoch": 1.4709577165289933,
      "grad_norm": 8.789575576782227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029151752591133118,
      "logits/rejected": -0.060455694794654846,
      "logps/chosen": -430.45819091796875,
      "logps/rejected": -384.0406188964844,
      "loss": 0.4189,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0870162695646286,
      "rewards/margins": 1.1228833198547363,
      "rewards/rejected": -1.2098995447158813,
      "step": 11241
    },
    {
      "epoch": 1.4710885744663451,
      "grad_norm": 9.844891548156738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14247608184814453,
      "logits/rejected": -0.07598508149385452,
      "logps/chosen": -406.25433349609375,
      "logps/rejected": -424.0174560546875,
      "loss": 0.6374,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8272785544395447,
      "rewards/margins": 0.4789867103099823,
      "rewards/rejected": -1.3062653541564941,
      "step": 11242
    },
    {
      "epoch": 1.4712194324036967,
      "grad_norm": 6.014716148376465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12763871252536774,
      "logits/rejected": 0.12828969955444336,
      "logps/chosen": -460.2497253417969,
      "logps/rejected": -538.7513427734375,
      "loss": 0.254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8429884314537048,
      "rewards/margins": 1.996933937072754,
      "rewards/rejected": -2.8399224281311035,
      "step": 11243
    },
    {
      "epoch": 1.4713502903410485,
      "grad_norm": 14.181864738464355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2529236078262329,
      "logits/rejected": -0.43997418880462646,
      "logps/chosen": -364.1275329589844,
      "logps/rejected": -490.1390380859375,
      "loss": 0.9727,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8338861465454102,
      "rewards/margins": 0.7593387365341187,
      "rewards/rejected": -2.5932250022888184,
      "step": 11244
    },
    {
      "epoch": 1.4714811482784003,
      "grad_norm": 8.441427230834961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04367412254214287,
      "logits/rejected": 0.10045477747917175,
      "logps/chosen": -425.88482666015625,
      "logps/rejected": -433.286865234375,
      "loss": 0.3421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5779995918273926,
      "rewards/margins": 1.4014883041381836,
      "rewards/rejected": -2.9794881343841553,
      "step": 11245
    },
    {
      "epoch": 1.471612006215752,
      "grad_norm": 4.193783760070801,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1417469084262848,
      "logits/rejected": 0.06445040553808212,
      "logps/chosen": -568.0199584960938,
      "logps/rejected": -463.40570068359375,
      "loss": 0.1973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8861591219902039,
      "rewards/margins": 1.6537063121795654,
      "rewards/rejected": -2.539865255355835,
      "step": 11246
    },
    {
      "epoch": 1.4717428641531038,
      "grad_norm": 10.694314956665039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15990032255649567,
      "logits/rejected": -0.005201232619583607,
      "logps/chosen": -399.84075927734375,
      "logps/rejected": -325.85516357421875,
      "loss": 0.9968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1807175874710083,
      "rewards/margins": -0.1504286527633667,
      "rewards/rejected": -1.030288815498352,
      "step": 11247
    },
    {
      "epoch": 1.4718737220904554,
      "grad_norm": 6.996222496032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36309272050857544,
      "logits/rejected": -0.33073922991752625,
      "logps/chosen": -225.0379638671875,
      "logps/rejected": -310.8566589355469,
      "loss": 0.5307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6198102235794067,
      "rewards/margins": 0.9969661831855774,
      "rewards/rejected": -2.61677622795105,
      "step": 11248
    },
    {
      "epoch": 1.4720045800278072,
      "grad_norm": 3.732633352279663,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3989083170890808,
      "logits/rejected": -0.5419746041297913,
      "logps/chosen": -345.3824462890625,
      "logps/rejected": -391.6716003417969,
      "loss": 0.2218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14085084199905396,
      "rewards/margins": 2.880991220474243,
      "rewards/rejected": -3.0218420028686523,
      "step": 11249
    },
    {
      "epoch": 1.472135437965159,
      "grad_norm": 5.384639739990234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09265947341918945,
      "logits/rejected": -0.09912149608135223,
      "logps/chosen": -463.7193603515625,
      "logps/rejected": -359.9034729003906,
      "loss": 0.307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11063764989376068,
      "rewards/margins": 1.9521422386169434,
      "rewards/rejected": -1.8415048122406006,
      "step": 11250
    },
    {
      "epoch": 1.4722662959025108,
      "grad_norm": 10.762377738952637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30023959279060364,
      "logits/rejected": -0.348354697227478,
      "logps/chosen": -479.7245178222656,
      "logps/rejected": -561.5595092773438,
      "loss": 0.5975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9330750107765198,
      "rewards/margins": 0.5857207775115967,
      "rewards/rejected": -1.5187957286834717,
      "step": 11251
    },
    {
      "epoch": 1.4723971538398626,
      "grad_norm": 10.484651565551758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0887470617890358,
      "logits/rejected": -0.00611269474029541,
      "logps/chosen": -527.402587890625,
      "logps/rejected": -462.65948486328125,
      "loss": 0.5296,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6822448968887329,
      "rewards/margins": 1.5503945350646973,
      "rewards/rejected": -2.2326395511627197,
      "step": 11252
    },
    {
      "epoch": 1.4725280117772144,
      "grad_norm": 3.759495258331299,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0734044536948204,
      "logits/rejected": -0.06071699783205986,
      "logps/chosen": -652.3731079101562,
      "logps/rejected": -569.1090087890625,
      "loss": 0.2178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.028822138905525208,
      "rewards/margins": 1.8146132230758667,
      "rewards/rejected": -1.8434354066848755,
      "step": 11253
    },
    {
      "epoch": 1.4726588697145662,
      "grad_norm": 6.944675922393799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07561419904232025,
      "logits/rejected": -0.28913378715515137,
      "logps/chosen": -491.89959716796875,
      "logps/rejected": -452.388671875,
      "loss": 0.3988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3644542694091797,
      "rewards/margins": 1.164616346359253,
      "rewards/rejected": -2.5290708541870117,
      "step": 11254
    },
    {
      "epoch": 1.472789727651918,
      "grad_norm": 7.510439395904541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015408686362206936,
      "logits/rejected": -0.1358855664730072,
      "logps/chosen": -796.35009765625,
      "logps/rejected": -472.81231689453125,
      "loss": 0.4366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07292327284812927,
      "rewards/margins": 1.756392240524292,
      "rewards/rejected": -1.6834688186645508,
      "step": 11255
    },
    {
      "epoch": 1.4729205855892697,
      "grad_norm": 4.540046215057373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3094344139099121,
      "logits/rejected": -0.4088166654109955,
      "logps/chosen": -433.3079528808594,
      "logps/rejected": -365.2406311035156,
      "loss": 0.322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47890132665634155,
      "rewards/margins": 1.387174129486084,
      "rewards/rejected": -1.8660752773284912,
      "step": 11256
    },
    {
      "epoch": 1.4730514435266215,
      "grad_norm": 6.369920253753662,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025686338543891907,
      "logits/rejected": -0.2443511188030243,
      "logps/chosen": -504.350341796875,
      "logps/rejected": -445.2558288574219,
      "loss": 0.2966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1892135739326477,
      "rewards/margins": 1.894078254699707,
      "rewards/rejected": -2.08329176902771,
      "step": 11257
    },
    {
      "epoch": 1.473182301463973,
      "grad_norm": 9.93843936920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009561747312545776,
      "logits/rejected": -0.043328262865543365,
      "logps/chosen": -497.7212829589844,
      "logps/rejected": -510.1171875,
      "loss": 0.6846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8089010119438171,
      "rewards/margins": 0.637211263179779,
      "rewards/rejected": -1.4461121559143066,
      "step": 11258
    },
    {
      "epoch": 1.473313159401325,
      "grad_norm": 5.690695762634277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08558740466833115,
      "logits/rejected": -0.0584731251001358,
      "logps/chosen": -514.5797729492188,
      "logps/rejected": -398.31842041015625,
      "loss": 0.242,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6431623697280884,
      "rewards/margins": 1.9799984693527222,
      "rewards/rejected": -2.6231606006622314,
      "step": 11259
    },
    {
      "epoch": 1.4734440173386767,
      "grad_norm": 12.619837760925293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11125976592302322,
      "logits/rejected": -0.2301870733499527,
      "logps/chosen": -496.8759765625,
      "logps/rejected": -356.54730224609375,
      "loss": 0.6614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0650582313537598,
      "rewards/margins": 0.43489229679107666,
      "rewards/rejected": -1.499950647354126,
      "step": 11260
    },
    {
      "epoch": 1.4735748752760285,
      "grad_norm": 9.709291458129883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3225003182888031,
      "logits/rejected": -0.3777018189430237,
      "logps/chosen": -363.7225646972656,
      "logps/rejected": -364.1650390625,
      "loss": 0.5684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4675862789154053,
      "rewards/margins": 0.7155580520629883,
      "rewards/rejected": -2.1831443309783936,
      "step": 11261
    },
    {
      "epoch": 1.4737057332133803,
      "grad_norm": 9.260310173034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005575999617576599,
      "logits/rejected": 0.15382592380046844,
      "logps/chosen": -459.2398681640625,
      "logps/rejected": -622.4793701171875,
      "loss": 0.4589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8021517395973206,
      "rewards/margins": 1.6509263515472412,
      "rewards/rejected": -2.453078031539917,
      "step": 11262
    },
    {
      "epoch": 1.473836591150732,
      "grad_norm": 8.15235424041748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05333635210990906,
      "logits/rejected": -0.037081606686115265,
      "logps/chosen": -630.3076171875,
      "logps/rejected": -438.2875671386719,
      "loss": 0.368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11664678156375885,
      "rewards/margins": 1.3265670537948608,
      "rewards/rejected": -1.443213939666748,
      "step": 11263
    },
    {
      "epoch": 1.4739674490880836,
      "grad_norm": 10.40144157409668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10435124486684799,
      "logits/rejected": -0.012045242823660374,
      "logps/chosen": -405.46429443359375,
      "logps/rejected": -615.3223266601562,
      "loss": 0.3726,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5012302398681641,
      "rewards/margins": 2.143240451812744,
      "rewards/rejected": -2.644470691680908,
      "step": 11264
    },
    {
      "epoch": 1.4740983070254354,
      "grad_norm": 4.1632466316223145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0317525751888752,
      "logits/rejected": -0.016579899936914444,
      "logps/chosen": -469.6647644042969,
      "logps/rejected": -477.604248046875,
      "loss": 0.2016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07718982547521591,
      "rewards/margins": 2.1513426303863525,
      "rewards/rejected": -2.074152946472168,
      "step": 11265
    },
    {
      "epoch": 1.4742291649627872,
      "grad_norm": 6.7734503746032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07066565752029419,
      "logits/rejected": -0.1948520541191101,
      "logps/chosen": -391.8045349121094,
      "logps/rejected": -427.49505615234375,
      "loss": 0.1926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3988906741142273,
      "rewards/margins": 3.0216472148895264,
      "rewards/rejected": -3.4205379486083984,
      "step": 11266
    },
    {
      "epoch": 1.474360022900139,
      "grad_norm": 7.099654197692871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36857494711875916,
      "logits/rejected": -0.4017821252346039,
      "logps/chosen": -486.2267150878906,
      "logps/rejected": -375.4276123046875,
      "loss": 0.4229,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2717221975326538,
      "rewards/margins": 1.2322360277175903,
      "rewards/rejected": -2.503958225250244,
      "step": 11267
    },
    {
      "epoch": 1.4744908808374908,
      "grad_norm": 8.168389320373535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21702158451080322,
      "logits/rejected": -0.029848821461200714,
      "logps/chosen": -829.6531372070312,
      "logps/rejected": -468.7605285644531,
      "loss": 0.4018,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4604389667510986,
      "rewards/margins": 1.3939627408981323,
      "rewards/rejected": -2.8544018268585205,
      "step": 11268
    },
    {
      "epoch": 1.4746217387748426,
      "grad_norm": 4.694648265838623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08333633095026016,
      "logits/rejected": -0.004353534430265427,
      "logps/chosen": -575.9229736328125,
      "logps/rejected": -422.0837707519531,
      "loss": 0.2605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6605842709541321,
      "rewards/margins": 1.9956097602844238,
      "rewards/rejected": -1.335025429725647,
      "step": 11269
    },
    {
      "epoch": 1.4747525967121944,
      "grad_norm": 7.789569854736328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19844526052474976,
      "logits/rejected": -0.24347221851348877,
      "logps/chosen": -359.9023132324219,
      "logps/rejected": -315.7486877441406,
      "loss": 0.5976,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5534816980361938,
      "rewards/margins": 0.3945200741291046,
      "rewards/rejected": -0.9480018019676208,
      "step": 11270
    },
    {
      "epoch": 1.4748834546495462,
      "grad_norm": 11.117783546447754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09238926321268082,
      "logits/rejected": 0.10179737210273743,
      "logps/chosen": -480.5675964355469,
      "logps/rejected": -497.77838134765625,
      "loss": 0.8729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7602005004882812,
      "rewards/margins": 0.7907137870788574,
      "rewards/rejected": -1.5509144067764282,
      "step": 11271
    },
    {
      "epoch": 1.475014312586898,
      "grad_norm": 4.94354248046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5713677406311035,
      "logits/rejected": -0.5623608231544495,
      "logps/chosen": -336.5673828125,
      "logps/rejected": -280.30828857421875,
      "loss": 0.4219,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9155844449996948,
      "rewards/margins": 1.9967067241668701,
      "rewards/rejected": -2.9122912883758545,
      "step": 11272
    },
    {
      "epoch": 1.4751451705242495,
      "grad_norm": 11.76901626586914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09571734815835953,
      "logits/rejected": 0.08021845668554306,
      "logps/chosen": -553.39794921875,
      "logps/rejected": -486.3345947265625,
      "loss": 0.8131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5823923349380493,
      "rewards/margins": 0.15174618363380432,
      "rewards/rejected": -0.734138548374176,
      "step": 11273
    },
    {
      "epoch": 1.4752760284616013,
      "grad_norm": 9.671734809875488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02388177439570427,
      "logits/rejected": -0.11361957341432571,
      "logps/chosen": -418.9103698730469,
      "logps/rejected": -555.3228759765625,
      "loss": 0.4529,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4998399019241333,
      "rewards/margins": 0.9730901122093201,
      "rewards/rejected": -1.4729299545288086,
      "step": 11274
    },
    {
      "epoch": 1.4754068863989531,
      "grad_norm": 9.832291603088379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2663763165473938,
      "logits/rejected": -0.40465742349624634,
      "logps/chosen": -418.805908203125,
      "logps/rejected": -576.1296997070312,
      "loss": 0.5432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.558121681213379,
      "rewards/margins": 0.6230881214141846,
      "rewards/rejected": -2.1812095642089844,
      "step": 11275
    },
    {
      "epoch": 1.475537744336305,
      "grad_norm": 7.883757591247559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0783596783876419,
      "logits/rejected": -0.28349053859710693,
      "logps/chosen": -510.8238525390625,
      "logps/rejected": -450.2701110839844,
      "loss": 0.2971,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4896703362464905,
      "rewards/margins": 2.0706329345703125,
      "rewards/rejected": -2.560303211212158,
      "step": 11276
    },
    {
      "epoch": 1.4756686022736567,
      "grad_norm": 9.24445629119873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2135334014892578,
      "logits/rejected": 0.10607907176017761,
      "logps/chosen": -295.3289794921875,
      "logps/rejected": -325.2709655761719,
      "loss": 0.3685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7398144006729126,
      "rewards/margins": 1.8178178071975708,
      "rewards/rejected": -2.5576324462890625,
      "step": 11277
    },
    {
      "epoch": 1.4757994602110085,
      "grad_norm": 4.473765850067139,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14973555505275726,
      "logits/rejected": -0.1733379364013672,
      "logps/chosen": -463.07342529296875,
      "logps/rejected": -420.99725341796875,
      "loss": 0.2333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4077434539794922,
      "rewards/margins": 1.9849907159805298,
      "rewards/rejected": -2.3927340507507324,
      "step": 11278
    },
    {
      "epoch": 1.47593031814836,
      "grad_norm": 7.955394268035889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15358778834342957,
      "logits/rejected": -0.12880896031856537,
      "logps/chosen": -326.3980407714844,
      "logps/rejected": -301.4200134277344,
      "loss": 0.422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7448362112045288,
      "rewards/margins": 0.9926682114601135,
      "rewards/rejected": -1.737504482269287,
      "step": 11279
    },
    {
      "epoch": 1.4760611760857119,
      "grad_norm": 9.059561729431152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18662698566913605,
      "logits/rejected": -0.36044275760650635,
      "logps/chosen": -377.1443786621094,
      "logps/rejected": -371.73077392578125,
      "loss": 0.3317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6663063168525696,
      "rewards/margins": 2.076385021209717,
      "rewards/rejected": -2.7426915168762207,
      "step": 11280
    },
    {
      "epoch": 1.4761920340230636,
      "grad_norm": 14.917805671691895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21015214920043945,
      "logits/rejected": -0.16806364059448242,
      "logps/chosen": -416.944580078125,
      "logps/rejected": -434.0960998535156,
      "loss": 0.668,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0726948976516724,
      "rewards/margins": 0.5070464611053467,
      "rewards/rejected": -1.5797412395477295,
      "step": 11281
    },
    {
      "epoch": 1.4763228919604154,
      "grad_norm": 9.959308624267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6551434993743896,
      "logits/rejected": -0.39281103014945984,
      "logps/chosen": -439.1363830566406,
      "logps/rejected": -316.9456787109375,
      "loss": 0.6406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8923513889312744,
      "rewards/margins": 0.5385265350341797,
      "rewards/rejected": -2.430877685546875,
      "step": 11282
    },
    {
      "epoch": 1.4764537498977672,
      "grad_norm": 9.107656478881836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013105301186442375,
      "logits/rejected": -0.21170859038829803,
      "logps/chosen": -485.27655029296875,
      "logps/rejected": -324.52178955078125,
      "loss": 0.4529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5043358206748962,
      "rewards/margins": 1.2356457710266113,
      "rewards/rejected": -1.7399816513061523,
      "step": 11283
    },
    {
      "epoch": 1.476584607835119,
      "grad_norm": 5.519648551940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04738558456301689,
      "logits/rejected": -0.18077629804611206,
      "logps/chosen": -439.04583740234375,
      "logps/rejected": -455.48828125,
      "loss": 0.2515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9604290127754211,
      "rewards/margins": 1.6879757642745972,
      "rewards/rejected": -2.648404836654663,
      "step": 11284
    },
    {
      "epoch": 1.4767154657724708,
      "grad_norm": 7.36331844329834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2289629727602005,
      "logits/rejected": -0.13227587938308716,
      "logps/chosen": -429.0185546875,
      "logps/rejected": -447.95025634765625,
      "loss": 0.401,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4693063795566559,
      "rewards/margins": 0.9856885671615601,
      "rewards/rejected": -1.454995036125183,
      "step": 11285
    },
    {
      "epoch": 1.4768463237098226,
      "grad_norm": 5.418732643127441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08334276080131531,
      "logits/rejected": -0.19289085268974304,
      "logps/chosen": -557.9052124023438,
      "logps/rejected": -716.6343994140625,
      "loss": 0.241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7058746814727783,
      "rewards/margins": 2.1222286224365234,
      "rewards/rejected": -2.8281030654907227,
      "step": 11286
    },
    {
      "epoch": 1.4769771816471744,
      "grad_norm": 4.7652201652526855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2126573622226715,
      "logits/rejected": -0.19828666746616364,
      "logps/chosen": -498.55316162109375,
      "logps/rejected": -480.6786804199219,
      "loss": 0.1936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8928499221801758,
      "rewards/margins": 1.9335641860961914,
      "rewards/rejected": -2.826414108276367,
      "step": 11287
    },
    {
      "epoch": 1.477108039584526,
      "grad_norm": 8.636472702026367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17051467299461365,
      "logits/rejected": -0.09631527960300446,
      "logps/chosen": -549.5647583007812,
      "logps/rejected": -692.9971313476562,
      "loss": 0.4014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.259091854095459,
      "rewards/margins": 1.4323077201843262,
      "rewards/rejected": -3.6913998126983643,
      "step": 11288
    },
    {
      "epoch": 1.4772388975218778,
      "grad_norm": 10.1144437789917,
      "learning_rate": 5e-06,
      "logits/chosen": -0.467677503824234,
      "logits/rejected": -0.17563435435295105,
      "logps/chosen": -501.42449951171875,
      "logps/rejected": -495.8400573730469,
      "loss": 0.6423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3493050336837769,
      "rewards/margins": 0.502236008644104,
      "rewards/rejected": -1.8515411615371704,
      "step": 11289
    },
    {
      "epoch": 1.4773697554592295,
      "grad_norm": 4.25802755355835,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3627404570579529,
      "logits/rejected": -0.36671119928359985,
      "logps/chosen": -575.07470703125,
      "logps/rejected": -582.1748657226562,
      "loss": 0.1944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49594366550445557,
      "rewards/margins": 2.4316465854644775,
      "rewards/rejected": -2.9275901317596436,
      "step": 11290
    },
    {
      "epoch": 1.4775006133965813,
      "grad_norm": 7.261876106262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18221327662467957,
      "logits/rejected": -0.14978548884391785,
      "logps/chosen": -532.2193603515625,
      "logps/rejected": -425.3630065917969,
      "loss": 0.46,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8008002042770386,
      "rewards/margins": 1.1370726823806763,
      "rewards/rejected": -1.9378728866577148,
      "step": 11291
    },
    {
      "epoch": 1.4776314713339331,
      "grad_norm": 6.5620598793029785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17498059570789337,
      "logits/rejected": -0.1264095902442932,
      "logps/chosen": -530.3108520507812,
      "logps/rejected": -559.14453125,
      "loss": 0.4395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.701471745967865,
      "rewards/margins": 0.9852268695831299,
      "rewards/rejected": -1.68669855594635,
      "step": 11292
    },
    {
      "epoch": 1.477762329271285,
      "grad_norm": 10.023158073425293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0315559059381485,
      "logits/rejected": 0.011362217366695404,
      "logps/chosen": -558.0196533203125,
      "logps/rejected": -523.5532836914062,
      "loss": 0.5346,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3296254873275757,
      "rewards/margins": 1.8471940755844116,
      "rewards/rejected": -2.176819324493408,
      "step": 11293
    },
    {
      "epoch": 1.4778931872086365,
      "grad_norm": 8.79681396484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0009071081876754761,
      "logits/rejected": 0.0340941920876503,
      "logps/chosen": -511.46331787109375,
      "logps/rejected": -596.6565551757812,
      "loss": 0.5576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1167690753936768,
      "rewards/margins": 0.7121925354003906,
      "rewards/rejected": -1.8289616107940674,
      "step": 11294
    },
    {
      "epoch": 1.4780240451459883,
      "grad_norm": 12.865876197814941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.042800694704055786,
      "logits/rejected": -0.08547227084636688,
      "logps/chosen": -739.5064697265625,
      "logps/rejected": -572.38330078125,
      "loss": 0.4392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.821532666683197,
      "rewards/margins": 1.6890292167663574,
      "rewards/rejected": -2.510561943054199,
      "step": 11295
    },
    {
      "epoch": 1.47815490308334,
      "grad_norm": 15.22387409210205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3296349346637726,
      "logits/rejected": -0.31113821268081665,
      "logps/chosen": -741.2742919921875,
      "logps/rejected": -504.1654968261719,
      "loss": 1.054,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8429847955703735,
      "rewards/margins": 0.5639733076095581,
      "rewards/rejected": -2.4069578647613525,
      "step": 11296
    },
    {
      "epoch": 1.4782857610206919,
      "grad_norm": 9.002848625183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1796351969242096,
      "logits/rejected": -0.10686711966991425,
      "logps/chosen": -549.1953735351562,
      "logps/rejected": -590.68896484375,
      "loss": 0.5888,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0738333463668823,
      "rewards/margins": 0.8798189163208008,
      "rewards/rejected": -1.953652262687683,
      "step": 11297
    },
    {
      "epoch": 1.4784166189580437,
      "grad_norm": 15.9857816696167,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2794061303138733,
      "logits/rejected": -0.19786149263381958,
      "logps/chosen": -531.7662963867188,
      "logps/rejected": -593.6649169921875,
      "loss": 0.7953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3071867227554321,
      "rewards/margins": 0.8192567229270935,
      "rewards/rejected": -2.12644362449646,
      "step": 11298
    },
    {
      "epoch": 1.4785474768953955,
      "grad_norm": 13.186720848083496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1305345743894577,
      "logits/rejected": 0.03587011247873306,
      "logps/chosen": -488.10614013671875,
      "logps/rejected": -351.5738220214844,
      "loss": 0.7732,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8978191614151,
      "rewards/margins": 1.6395330429077148,
      "rewards/rejected": -3.5373520851135254,
      "step": 11299
    },
    {
      "epoch": 1.4786783348327472,
      "grad_norm": 14.637202262878418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05070666968822479,
      "logits/rejected": -0.03813375532627106,
      "logps/chosen": -426.9698791503906,
      "logps/rejected": -523.9755249023438,
      "loss": 1.0606,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6914081573486328,
      "rewards/margins": 0.01615181565284729,
      "rewards/rejected": -1.7075599431991577,
      "step": 11300
    },
    {
      "epoch": 1.478809192770099,
      "grad_norm": 7.326458930969238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006877139210700989,
      "logits/rejected": 0.0927969366312027,
      "logps/chosen": -514.7080688476562,
      "logps/rejected": -579.1403198242188,
      "loss": 0.3338,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2704486846923828,
      "rewards/margins": 1.9494075775146484,
      "rewards/rejected": -2.2198562622070312,
      "step": 11301
    },
    {
      "epoch": 1.4789400507074508,
      "grad_norm": 4.540536880493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12238664925098419,
      "logits/rejected": -0.40880531072616577,
      "logps/chosen": -429.114990234375,
      "logps/rejected": -398.1144714355469,
      "loss": 0.2587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7241744995117188,
      "rewards/margins": 1.378359317779541,
      "rewards/rejected": -2.102534055709839,
      "step": 11302
    },
    {
      "epoch": 1.4790709086448026,
      "grad_norm": 7.082618236541748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2193669229745865,
      "logits/rejected": -0.06774073839187622,
      "logps/chosen": -591.6021728515625,
      "logps/rejected": -638.3341064453125,
      "loss": 0.4276,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5844004154205322,
      "rewards/margins": 1.2696222066879272,
      "rewards/rejected": -1.85402250289917,
      "step": 11303
    },
    {
      "epoch": 1.4792017665821542,
      "grad_norm": 11.500643730163574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1814022958278656,
      "logits/rejected": -0.40157198905944824,
      "logps/chosen": -516.2112426757812,
      "logps/rejected": -376.19793701171875,
      "loss": 0.725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6416101455688477,
      "rewards/margins": 0.20816195011138916,
      "rewards/rejected": -1.8497720956802368,
      "step": 11304
    },
    {
      "epoch": 1.479332624519506,
      "grad_norm": 9.81446647644043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1668393909931183,
      "logits/rejected": -0.15916919708251953,
      "logps/chosen": -406.66009521484375,
      "logps/rejected": -452.77496337890625,
      "loss": 0.4462,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7563737630844116,
      "rewards/margins": 1.5005223751068115,
      "rewards/rejected": -3.2568962574005127,
      "step": 11305
    },
    {
      "epoch": 1.4794634824568578,
      "grad_norm": 10.572479248046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2906305491924286,
      "logits/rejected": -0.3784096837043762,
      "logps/chosen": -422.3487548828125,
      "logps/rejected": -325.5279846191406,
      "loss": 0.8529,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6902095079421997,
      "rewards/margins": -0.1367601454257965,
      "rewards/rejected": -1.553449273109436,
      "step": 11306
    },
    {
      "epoch": 1.4795943403942096,
      "grad_norm": 3.2262961864471436,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15684382617473602,
      "logits/rejected": -0.062491632997989655,
      "logps/chosen": -344.67620849609375,
      "logps/rejected": -424.5369567871094,
      "loss": 0.1298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4460269808769226,
      "rewards/margins": 2.18414568901062,
      "rewards/rejected": -2.6301727294921875,
      "step": 11307
    },
    {
      "epoch": 1.4797251983315614,
      "grad_norm": 9.483942985534668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06178617849946022,
      "logits/rejected": -0.19843196868896484,
      "logps/chosen": -685.4423828125,
      "logps/rejected": -529.154052734375,
      "loss": 0.4206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6659923791885376,
      "rewards/margins": 1.148916482925415,
      "rewards/rejected": -2.814908981323242,
      "step": 11308
    },
    {
      "epoch": 1.479856056268913,
      "grad_norm": 4.781831741333008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021751798689365387,
      "logits/rejected": -0.23131144046783447,
      "logps/chosen": -390.52685546875,
      "logps/rejected": -317.6395263671875,
      "loss": 0.2116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2983537912368774,
      "rewards/margins": 2.0292060375213623,
      "rewards/rejected": -3.3275599479675293,
      "step": 11309
    },
    {
      "epoch": 1.4799869142062647,
      "grad_norm": 6.493124485015869,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23226439952850342,
      "logits/rejected": -0.1227063238620758,
      "logps/chosen": -513.7246704101562,
      "logps/rejected": -426.2122497558594,
      "loss": 0.2935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6615292429924011,
      "rewards/margins": 1.6012024879455566,
      "rewards/rejected": -2.2627315521240234,
      "step": 11310
    },
    {
      "epoch": 1.4801177721436165,
      "grad_norm": 13.403177261352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11541742086410522,
      "logits/rejected": -0.24095356464385986,
      "logps/chosen": -462.6282958984375,
      "logps/rejected": -650.8350830078125,
      "loss": 0.7552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3551642894744873,
      "rewards/margins": 0.9250430464744568,
      "rewards/rejected": -2.2802071571350098,
      "step": 11311
    },
    {
      "epoch": 1.4802486300809683,
      "grad_norm": 10.54006290435791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11858788877725601,
      "logits/rejected": 0.13059109449386597,
      "logps/chosen": -427.79779052734375,
      "logps/rejected": -450.9259033203125,
      "loss": 0.5118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9492350220680237,
      "rewards/margins": 0.9087707996368408,
      "rewards/rejected": -1.8580058813095093,
      "step": 11312
    },
    {
      "epoch": 1.48037948801832,
      "grad_norm": 9.186698913574219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13339869678020477,
      "logits/rejected": -0.20469364523887634,
      "logps/chosen": -577.7799072265625,
      "logps/rejected": -380.1996765136719,
      "loss": 0.4866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1626667976379395,
      "rewards/margins": 1.2554324865341187,
      "rewards/rejected": -2.4180994033813477,
      "step": 11313
    },
    {
      "epoch": 1.4805103459556719,
      "grad_norm": 6.802104949951172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3850618004798889,
      "logits/rejected": -0.4932607114315033,
      "logps/chosen": -596.77685546875,
      "logps/rejected": -688.2504272460938,
      "loss": 0.3065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4156237840652466,
      "rewards/margins": 1.8251371383666992,
      "rewards/rejected": -3.2407610416412354,
      "step": 11314
    },
    {
      "epoch": 1.4806412038930237,
      "grad_norm": 4.417330741882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46535947918891907,
      "logits/rejected": -0.5602147579193115,
      "logps/chosen": -322.9875183105469,
      "logps/rejected": -386.7294616699219,
      "loss": 0.1753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43906861543655396,
      "rewards/margins": 2.65483021736145,
      "rewards/rejected": -3.0938987731933594,
      "step": 11315
    },
    {
      "epoch": 1.4807720618303755,
      "grad_norm": 11.853236198425293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20007309317588806,
      "logits/rejected": -0.08325593918561935,
      "logps/chosen": -453.92901611328125,
      "logps/rejected": -470.3476867675781,
      "loss": 0.7344,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5602738857269287,
      "rewards/margins": 0.42461296916007996,
      "rewards/rejected": -1.9848870038986206,
      "step": 11316
    },
    {
      "epoch": 1.4809029197677273,
      "grad_norm": 15.101704597473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25392767786979675,
      "logits/rejected": -0.06614239513874054,
      "logps/chosen": -555.041748046875,
      "logps/rejected": -354.4899597167969,
      "loss": 0.7639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4278631210327148,
      "rewards/margins": 0.8437832593917847,
      "rewards/rejected": -2.27164626121521,
      "step": 11317
    },
    {
      "epoch": 1.481033777705079,
      "grad_norm": 9.554311752319336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24081917107105255,
      "logits/rejected": -0.39630311727523804,
      "logps/chosen": -373.7586364746094,
      "logps/rejected": -369.8935546875,
      "loss": 0.5292,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7521132230758667,
      "rewards/margins": 1.1151080131530762,
      "rewards/rejected": -1.8672213554382324,
      "step": 11318
    },
    {
      "epoch": 1.4811646356424306,
      "grad_norm": 11.00452995300293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06568550318479538,
      "logits/rejected": 0.0596197172999382,
      "logps/chosen": -576.218505859375,
      "logps/rejected": -467.7564392089844,
      "loss": 0.5135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9311255216598511,
      "rewards/margins": 1.3734874725341797,
      "rewards/rejected": -2.3046131134033203,
      "step": 11319
    },
    {
      "epoch": 1.4812954935797824,
      "grad_norm": 4.982591152191162,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16210882365703583,
      "logits/rejected": -0.05131467059254646,
      "logps/chosen": -662.3550415039062,
      "logps/rejected": -473.2550964355469,
      "loss": 0.2039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17788353562355042,
      "rewards/margins": 2.4733381271362305,
      "rewards/rejected": -2.651221752166748,
      "step": 11320
    },
    {
      "epoch": 1.4814263515171342,
      "grad_norm": 6.235281944274902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33855539560317993,
      "logits/rejected": -0.163835346698761,
      "logps/chosen": -531.1638793945312,
      "logps/rejected": -430.6432800292969,
      "loss": 0.3562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.264582872390747,
      "rewards/margins": 1.2905223369598389,
      "rewards/rejected": -2.555105209350586,
      "step": 11321
    },
    {
      "epoch": 1.481557209454486,
      "grad_norm": 10.167792320251465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21286308765411377,
      "logits/rejected": -0.49778592586517334,
      "logps/chosen": -402.1986389160156,
      "logps/rejected": -350.8241271972656,
      "loss": 0.4195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4921610355377197,
      "rewards/margins": 1.6142864227294922,
      "rewards/rejected": -3.106447458267212,
      "step": 11322
    },
    {
      "epoch": 1.4816880673918378,
      "grad_norm": 11.379530906677246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13375067710876465,
      "logits/rejected": -0.019945167005062103,
      "logps/chosen": -423.06561279296875,
      "logps/rejected": -584.9227294921875,
      "loss": 0.602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6538650989532471,
      "rewards/margins": 0.5900229811668396,
      "rewards/rejected": -1.2438879013061523,
      "step": 11323
    },
    {
      "epoch": 1.4818189253291896,
      "grad_norm": 8.402438163757324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15155328810214996,
      "logits/rejected": -0.24056117236614227,
      "logps/chosen": -427.80218505859375,
      "logps/rejected": -461.4429016113281,
      "loss": 0.466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30002519488334656,
      "rewards/margins": 0.7716526389122009,
      "rewards/rejected": -1.071677803993225,
      "step": 11324
    },
    {
      "epoch": 1.4819497832665411,
      "grad_norm": 10.038398742675781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0836368054151535,
      "logits/rejected": -0.21153269708156586,
      "logps/chosen": -352.40771484375,
      "logps/rejected": -375.327392578125,
      "loss": 0.4969,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5509159564971924,
      "rewards/margins": 0.8336499929428101,
      "rewards/rejected": -2.384566068649292,
      "step": 11325
    },
    {
      "epoch": 1.482080641203893,
      "grad_norm": 8.739384651184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2284557819366455,
      "logits/rejected": -0.20102164149284363,
      "logps/chosen": -463.9561767578125,
      "logps/rejected": -552.6685791015625,
      "loss": 0.5376,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.00286066532135,
      "rewards/margins": 0.8680835962295532,
      "rewards/rejected": -1.8709442615509033,
      "step": 11326
    },
    {
      "epoch": 1.4822114991412447,
      "grad_norm": 10.127634048461914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14347216486930847,
      "logits/rejected": -0.0426279753446579,
      "logps/chosen": -257.2412109375,
      "logps/rejected": -400.7526550292969,
      "loss": 0.6202,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5403056144714355,
      "rewards/margins": 0.9604859352111816,
      "rewards/rejected": -2.500791549682617,
      "step": 11327
    },
    {
      "epoch": 1.4823423570785965,
      "grad_norm": 8.723962783813477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24021974205970764,
      "logits/rejected": -0.14662857353687286,
      "logps/chosen": -351.5481872558594,
      "logps/rejected": -294.13250732421875,
      "loss": 0.5505,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6518603563308716,
      "rewards/margins": 0.7097543478012085,
      "rewards/rejected": -2.36161470413208,
      "step": 11328
    },
    {
      "epoch": 1.4824732150159483,
      "grad_norm": 7.484821796417236,
      "learning_rate": 5e-06,
      "logits/chosen": 0.036081649363040924,
      "logits/rejected": 0.09380374103784561,
      "logps/chosen": -538.01904296875,
      "logps/rejected": -448.59088134765625,
      "loss": 0.6123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9709354639053345,
      "rewards/margins": 0.955034613609314,
      "rewards/rejected": -1.9259700775146484,
      "step": 11329
    },
    {
      "epoch": 1.4826040729533,
      "grad_norm": 9.520146369934082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13300441205501556,
      "logits/rejected": -0.47066155076026917,
      "logps/chosen": -478.99749755859375,
      "logps/rejected": -518.2470703125,
      "loss": 0.6071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3881447315216064,
      "rewards/margins": 1.4976983070373535,
      "rewards/rejected": -2.885843276977539,
      "step": 11330
    },
    {
      "epoch": 1.482734930890652,
      "grad_norm": 4.690060615539551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.059606555849313736,
      "logits/rejected": -0.07285191118717194,
      "logps/chosen": -450.6445007324219,
      "logps/rejected": -453.6991882324219,
      "loss": 0.2087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1374503374099731,
      "rewards/margins": 1.6674209833145142,
      "rewards/rejected": -2.8048715591430664,
      "step": 11331
    },
    {
      "epoch": 1.4828657888280037,
      "grad_norm": 7.013062477111816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22165566682815552,
      "logits/rejected": -0.36529409885406494,
      "logps/chosen": -567.4678344726562,
      "logps/rejected": -526.716796875,
      "loss": 0.2693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08000260591506958,
      "rewards/margins": 2.944147825241089,
      "rewards/rejected": -2.864145040512085,
      "step": 11332
    },
    {
      "epoch": 1.4829966467653555,
      "grad_norm": 9.171761512756348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1263180822134018,
      "logits/rejected": -0.17752376198768616,
      "logps/chosen": -580.5510864257812,
      "logps/rejected": -544.3646240234375,
      "loss": 0.5967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3896573781967163,
      "rewards/margins": 0.479053258895874,
      "rewards/rejected": -1.8687107563018799,
      "step": 11333
    },
    {
      "epoch": 1.483127504702707,
      "grad_norm": 10.131265640258789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04976535215973854,
      "logits/rejected": -0.05026941001415253,
      "logps/chosen": -707.8372802734375,
      "logps/rejected": -517.7117309570312,
      "loss": 0.3494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1467843055725098,
      "rewards/margins": 1.6033531427383423,
      "rewards/rejected": -2.7501373291015625,
      "step": 11334
    },
    {
      "epoch": 1.4832583626400588,
      "grad_norm": 14.310608863830566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2978017330169678,
      "logits/rejected": -0.11121998727321625,
      "logps/chosen": -540.7985229492188,
      "logps/rejected": -548.9937744140625,
      "loss": 0.5494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7243061661720276,
      "rewards/margins": 1.3113375902175903,
      "rewards/rejected": -2.0356435775756836,
      "step": 11335
    },
    {
      "epoch": 1.4833892205774106,
      "grad_norm": 9.150936126708984,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16523662209510803,
      "logits/rejected": -0.04631461203098297,
      "logps/chosen": -487.20172119140625,
      "logps/rejected": -460.4853210449219,
      "loss": 0.5565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7853636145591736,
      "rewards/margins": 1.1179429292678833,
      "rewards/rejected": -1.903306484222412,
      "step": 11336
    },
    {
      "epoch": 1.4835200785147624,
      "grad_norm": 8.224939346313477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1739071011543274,
      "logits/rejected": -0.27270379662513733,
      "logps/chosen": -499.4322509765625,
      "logps/rejected": -479.9895935058594,
      "loss": 0.3555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0337530374526978,
      "rewards/margins": 1.8665207624435425,
      "rewards/rejected": -2.9002740383148193,
      "step": 11337
    },
    {
      "epoch": 1.4836509364521142,
      "grad_norm": 5.518965721130371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21118316054344177,
      "logits/rejected": -0.3236309885978699,
      "logps/chosen": -601.1094970703125,
      "logps/rejected": -521.9515991210938,
      "loss": 0.2018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.99245685338974,
      "rewards/margins": 2.084301471710205,
      "rewards/rejected": -3.0767581462860107,
      "step": 11338
    },
    {
      "epoch": 1.483781794389466,
      "grad_norm": 6.705977916717529,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2947136163711548,
      "logits/rejected": 0.03153320401906967,
      "logps/chosen": -593.6611328125,
      "logps/rejected": -396.15155029296875,
      "loss": 0.3672,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2305275201797485,
      "rewards/margins": 1.1751325130462646,
      "rewards/rejected": -2.4056599140167236,
      "step": 11339
    },
    {
      "epoch": 1.4839126523268176,
      "grad_norm": 14.048054695129395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07599393278360367,
      "logits/rejected": -0.09151093661785126,
      "logps/chosen": -550.8666381835938,
      "logps/rejected": -352.82427978515625,
      "loss": 0.6792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.321443796157837,
      "rewards/margins": 0.269948810338974,
      "rewards/rejected": -1.5913925170898438,
      "step": 11340
    },
    {
      "epoch": 1.4840435102641694,
      "grad_norm": 10.362466812133789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1610678732395172,
      "logits/rejected": -0.3373120129108429,
      "logps/chosen": -436.70916748046875,
      "logps/rejected": -407.4044189453125,
      "loss": 0.6696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.510440468788147,
      "rewards/margins": 0.2002672255039215,
      "rewards/rejected": -1.710707664489746,
      "step": 11341
    },
    {
      "epoch": 1.4841743682015212,
      "grad_norm": 6.384459972381592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1627751886844635,
      "logits/rejected": -0.07517610490322113,
      "logps/chosen": -527.5552368164062,
      "logps/rejected": -390.66827392578125,
      "loss": 0.2599,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.190739393234253,
      "rewards/margins": 1.7615975141525269,
      "rewards/rejected": -2.9523367881774902,
      "step": 11342
    },
    {
      "epoch": 1.484305226138873,
      "grad_norm": 10.028144836425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10424577444791794,
      "logits/rejected": -0.16169501841068268,
      "logps/chosen": -383.4100341796875,
      "logps/rejected": -469.27581787109375,
      "loss": 0.4711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2450774908065796,
      "rewards/margins": 1.81645667552948,
      "rewards/rejected": -2.0615341663360596,
      "step": 11343
    },
    {
      "epoch": 1.4844360840762247,
      "grad_norm": 7.812681674957275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12418094277381897,
      "logits/rejected": -0.20659637451171875,
      "logps/chosen": -402.3243408203125,
      "logps/rejected": -449.56317138671875,
      "loss": 0.4568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.501983642578125,
      "rewards/margins": 0.970959484577179,
      "rewards/rejected": -2.472943067550659,
      "step": 11344
    },
    {
      "epoch": 1.4845669420135765,
      "grad_norm": 5.8375244140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0014663320034742355,
      "logits/rejected": -0.12264499813318253,
      "logps/chosen": -591.9901123046875,
      "logps/rejected": -709.8901977539062,
      "loss": 0.3045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19034212827682495,
      "rewards/margins": 1.9664230346679688,
      "rewards/rejected": -2.1567652225494385,
      "step": 11345
    },
    {
      "epoch": 1.4846977999509283,
      "grad_norm": 7.610057830810547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009459920227527618,
      "logits/rejected": 0.03126171976327896,
      "logps/chosen": -352.3836975097656,
      "logps/rejected": -472.45452880859375,
      "loss": 0.5664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44279712438583374,
      "rewards/margins": 1.3510966300964355,
      "rewards/rejected": -1.793893814086914,
      "step": 11346
    },
    {
      "epoch": 1.4848286578882801,
      "grad_norm": 6.162649154663086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0045900484547019005,
      "logits/rejected": 0.07634676247835159,
      "logps/chosen": -326.4761047363281,
      "logps/rejected": -432.67913818359375,
      "loss": 0.3958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8297481536865234,
      "rewards/margins": 1.7287039756774902,
      "rewards/rejected": -2.5584518909454346,
      "step": 11347
    },
    {
      "epoch": 1.484959515825632,
      "grad_norm": 8.755656242370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18905198574066162,
      "logits/rejected": -0.06928939372301102,
      "logps/chosen": -371.0521240234375,
      "logps/rejected": -306.0224304199219,
      "loss": 0.532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.170816421508789,
      "rewards/margins": 0.7062159180641174,
      "rewards/rejected": -1.8770322799682617,
      "step": 11348
    },
    {
      "epoch": 1.4850903737629837,
      "grad_norm": 8.033287048339844,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10245325416326523,
      "logits/rejected": 0.0018929727375507355,
      "logps/chosen": -406.90576171875,
      "logps/rejected": -482.7659606933594,
      "loss": 0.4249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11021594703197479,
      "rewards/margins": 1.2585209608078003,
      "rewards/rejected": -1.3687368631362915,
      "step": 11349
    },
    {
      "epoch": 1.4852212317003353,
      "grad_norm": 8.649456024169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20431207120418549,
      "logits/rejected": -0.5311144590377808,
      "logps/chosen": -563.652099609375,
      "logps/rejected": -499.5312805175781,
      "loss": 0.4312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1901425123214722,
      "rewards/margins": 1.3121745586395264,
      "rewards/rejected": -2.502316951751709,
      "step": 11350
    },
    {
      "epoch": 1.485352089637687,
      "grad_norm": 7.43481969833374,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04624206945300102,
      "logits/rejected": -0.07385218888521194,
      "logps/chosen": -632.499267578125,
      "logps/rejected": -587.379150390625,
      "loss": 0.305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6695511341094971,
      "rewards/margins": 1.6567904949188232,
      "rewards/rejected": -2.3263416290283203,
      "step": 11351
    },
    {
      "epoch": 1.4854829475750388,
      "grad_norm": 13.128562927246094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02308512106537819,
      "logits/rejected": -0.25184690952301025,
      "logps/chosen": -557.4652099609375,
      "logps/rejected": -466.1769714355469,
      "loss": 0.7507,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.844168484210968,
      "rewards/margins": 0.4164617359638214,
      "rewards/rejected": -1.2606302499771118,
      "step": 11352
    },
    {
      "epoch": 1.4856138055123906,
      "grad_norm": 8.367886543273926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16076120734214783,
      "logits/rejected": 0.33414334058761597,
      "logps/chosen": -443.93487548828125,
      "logps/rejected": -540.44384765625,
      "loss": 0.3678,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23064041137695312,
      "rewards/margins": 1.4074721336364746,
      "rewards/rejected": -1.6381126642227173,
      "step": 11353
    },
    {
      "epoch": 1.4857446634497424,
      "grad_norm": 6.151772499084473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1490587443113327,
      "logits/rejected": -0.18765857815742493,
      "logps/chosen": -357.47271728515625,
      "logps/rejected": -421.2393798828125,
      "loss": 0.4009,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9398044347763062,
      "rewards/margins": 1.3444945812225342,
      "rewards/rejected": -2.284298896789551,
      "step": 11354
    },
    {
      "epoch": 1.485875521387094,
      "grad_norm": 5.755120754241943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20360404253005981,
      "logits/rejected": -0.31727200746536255,
      "logps/chosen": -519.150146484375,
      "logps/rejected": -456.1977233886719,
      "loss": 0.2002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6793725490570068,
      "rewards/margins": 2.347687244415283,
      "rewards/rejected": -3.027059555053711,
      "step": 11355
    },
    {
      "epoch": 1.4860063793244458,
      "grad_norm": 12.67752742767334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23469792306423187,
      "logits/rejected": -0.10412853956222534,
      "logps/chosen": -542.1915283203125,
      "logps/rejected": -547.3386840820312,
      "loss": 0.65,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4880324602127075,
      "rewards/margins": 0.3673701286315918,
      "rewards/rejected": -1.8554025888442993,
      "step": 11356
    },
    {
      "epoch": 1.4861372372617976,
      "grad_norm": 3.4917545318603516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2614370882511139,
      "logits/rejected": -0.24938468635082245,
      "logps/chosen": -539.4790649414062,
      "logps/rejected": -377.3214111328125,
      "loss": 0.1628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3160478472709656,
      "rewards/margins": 2.308907985687256,
      "rewards/rejected": -2.624955654144287,
      "step": 11357
    },
    {
      "epoch": 1.4862680951991494,
      "grad_norm": 10.757895469665527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40825560688972473,
      "logits/rejected": -0.5237535238265991,
      "logps/chosen": -415.26446533203125,
      "logps/rejected": -387.3457946777344,
      "loss": 0.7181,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0483295917510986,
      "rewards/margins": 0.347136914730072,
      "rewards/rejected": -1.3954664468765259,
      "step": 11358
    },
    {
      "epoch": 1.4863989531365012,
      "grad_norm": 8.352662086486816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18920697271823883,
      "logits/rejected": -0.22977352142333984,
      "logps/chosen": -530.8489990234375,
      "logps/rejected": -512.9181518554688,
      "loss": 0.4916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3241746425628662,
      "rewards/margins": 0.6632711291313171,
      "rewards/rejected": -1.9874458312988281,
      "step": 11359
    },
    {
      "epoch": 1.486529811073853,
      "grad_norm": 3.3349618911743164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1857602745294571,
      "logits/rejected": -0.16353252530097961,
      "logps/chosen": -446.4189453125,
      "logps/rejected": -393.9399719238281,
      "loss": 0.1534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15262222290039062,
      "rewards/margins": 2.271214008331299,
      "rewards/rejected": -2.4238362312316895,
      "step": 11360
    },
    {
      "epoch": 1.4866606690112047,
      "grad_norm": 5.513166904449463,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09563055634498596,
      "logits/rejected": -0.118388332426548,
      "logps/chosen": -379.9815673828125,
      "logps/rejected": -418.2047119140625,
      "loss": 0.2696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2536317110061646,
      "rewards/margins": 1.4259867668151855,
      "rewards/rejected": -2.6796185970306396,
      "step": 11361
    },
    {
      "epoch": 1.4867915269485565,
      "grad_norm": 12.192062377929688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.035551249980926514,
      "logits/rejected": -0.06486578285694122,
      "logps/chosen": -360.2629089355469,
      "logps/rejected": -594.6014404296875,
      "loss": 0.5759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.862977147102356,
      "rewards/margins": 0.7416176795959473,
      "rewards/rejected": -1.6045948266983032,
      "step": 11362
    },
    {
      "epoch": 1.4869223848859083,
      "grad_norm": 6.815382957458496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03407365456223488,
      "logits/rejected": -0.16559836268424988,
      "logps/chosen": -572.2352905273438,
      "logps/rejected": -651.5565185546875,
      "loss": 0.3604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5092966556549072,
      "rewards/margins": 1.1855366230010986,
      "rewards/rejected": -1.6948332786560059,
      "step": 11363
    },
    {
      "epoch": 1.4870532428232601,
      "grad_norm": 10.32583999633789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14956629276275635,
      "logits/rejected": -0.18978671729564667,
      "logps/chosen": -354.9031066894531,
      "logps/rejected": -450.1814270019531,
      "loss": 0.489,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0832483768463135,
      "rewards/margins": 0.7611806988716125,
      "rewards/rejected": -1.8444291353225708,
      "step": 11364
    },
    {
      "epoch": 1.4871841007606117,
      "grad_norm": 8.975579261779785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014486445114016533,
      "logits/rejected": 0.011529967188835144,
      "logps/chosen": -544.0963134765625,
      "logps/rejected": -890.6527709960938,
      "loss": 0.3612,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6840158700942993,
      "rewards/margins": 1.5498483180999756,
      "rewards/rejected": -2.2338643074035645,
      "step": 11365
    },
    {
      "epoch": 1.4873149586979635,
      "grad_norm": 8.60851001739502,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14329782128334045,
      "logits/rejected": 0.07415817677974701,
      "logps/chosen": -496.96990966796875,
      "logps/rejected": -393.5038146972656,
      "loss": 0.4241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5467901825904846,
      "rewards/margins": 1.0585728883743286,
      "rewards/rejected": -1.605363130569458,
      "step": 11366
    },
    {
      "epoch": 1.4874458166353153,
      "grad_norm": 12.834863662719727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.25614306330680847,
      "logits/rejected": 0.18398520350456238,
      "logps/chosen": -412.1113586425781,
      "logps/rejected": -407.5633850097656,
      "loss": 0.9204,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5868505239486694,
      "rewards/margins": -0.08171018958091736,
      "rewards/rejected": -1.5051401853561401,
      "step": 11367
    },
    {
      "epoch": 1.487576674572667,
      "grad_norm": 8.149449348449707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022507719695568085,
      "logits/rejected": 0.1099453940987587,
      "logps/chosen": -330.7113342285156,
      "logps/rejected": -499.0837707519531,
      "loss": 0.3689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6115468740463257,
      "rewards/margins": 1.4779307842254639,
      "rewards/rejected": -3.0894775390625,
      "step": 11368
    },
    {
      "epoch": 1.4877075325100189,
      "grad_norm": 5.360073089599609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1811748445034027,
      "logits/rejected": -0.0941963642835617,
      "logps/chosen": -507.08526611328125,
      "logps/rejected": -437.9920654296875,
      "loss": 0.2569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6128261089324951,
      "rewards/margins": 1.8468821048736572,
      "rewards/rejected": -3.4597084522247314,
      "step": 11369
    },
    {
      "epoch": 1.4878383904473707,
      "grad_norm": 6.698061943054199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17464545369148254,
      "logits/rejected": -0.204385444521904,
      "logps/chosen": -425.5494079589844,
      "logps/rejected": -443.2415771484375,
      "loss": 0.2841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0875730514526367,
      "rewards/margins": 1.7109582424163818,
      "rewards/rejected": -2.7985312938690186,
      "step": 11370
    },
    {
      "epoch": 1.4879692483847222,
      "grad_norm": 9.637411117553711,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05809953063726425,
      "logits/rejected": -0.13992033898830414,
      "logps/chosen": -564.0233764648438,
      "logps/rejected": -555.6736450195312,
      "loss": 0.4312,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2441774606704712,
      "rewards/margins": 1.5549911260604858,
      "rewards/rejected": -2.799168586730957,
      "step": 11371
    },
    {
      "epoch": 1.488100106322074,
      "grad_norm": 12.956899642944336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03225093334913254,
      "logits/rejected": -0.1780788004398346,
      "logps/chosen": -681.5658569335938,
      "logps/rejected": -508.4757385253906,
      "loss": 0.7417,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.323638677597046,
      "rewards/margins": 0.9209650158882141,
      "rewards/rejected": -2.2446036338806152,
      "step": 11372
    },
    {
      "epoch": 1.4882309642594258,
      "grad_norm": 5.50309944152832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08669336885213852,
      "logits/rejected": -0.15591007471084595,
      "logps/chosen": -358.2628173828125,
      "logps/rejected": -380.3284912109375,
      "loss": 0.3,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6647884845733643,
      "rewards/margins": 1.4659082889556885,
      "rewards/rejected": -2.1306965351104736,
      "step": 11373
    },
    {
      "epoch": 1.4883618221967776,
      "grad_norm": 10.104109764099121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38208553194999695,
      "logits/rejected": -0.10295701026916504,
      "logps/chosen": -361.67657470703125,
      "logps/rejected": -409.72906494140625,
      "loss": 0.4222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.160253882408142,
      "rewards/margins": 1.7743465900421143,
      "rewards/rejected": -2.934600353240967,
      "step": 11374
    },
    {
      "epoch": 1.4884926801341294,
      "grad_norm": 10.12913990020752,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12356499582529068,
      "logits/rejected": -0.031914833933115005,
      "logps/chosen": -527.4993286132812,
      "logps/rejected": -539.2366943359375,
      "loss": 0.5199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.559466004371643,
      "rewards/margins": 1.518433928489685,
      "rewards/rejected": -3.077899932861328,
      "step": 11375
    },
    {
      "epoch": 1.4886235380714812,
      "grad_norm": 5.227829933166504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11564290523529053,
      "logits/rejected": -0.01573259010910988,
      "logps/chosen": -496.6578063964844,
      "logps/rejected": -523.5823974609375,
      "loss": 0.2869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0883797407150269,
      "rewards/margins": 1.4446556568145752,
      "rewards/rejected": -2.5330355167388916,
      "step": 11376
    },
    {
      "epoch": 1.488754396008833,
      "grad_norm": 8.94675350189209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3798859119415283,
      "logits/rejected": -0.1627328097820282,
      "logps/chosen": -364.6310729980469,
      "logps/rejected": -382.3218078613281,
      "loss": 0.6254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7281216382980347,
      "rewards/margins": 0.5763275623321533,
      "rewards/rejected": -2.3044493198394775,
      "step": 11377
    },
    {
      "epoch": 1.4888852539461848,
      "grad_norm": 3.832285165786743,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06257472932338715,
      "logits/rejected": 0.02871061861515045,
      "logps/chosen": -382.73651123046875,
      "logps/rejected": -390.2826843261719,
      "loss": 0.1036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6463381052017212,
      "rewards/margins": 2.9926345348358154,
      "rewards/rejected": -3.638972282409668,
      "step": 11378
    },
    {
      "epoch": 1.4890161118835366,
      "grad_norm": 3.805821657180786,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4140334725379944,
      "logits/rejected": -0.673554003238678,
      "logps/chosen": -240.76751708984375,
      "logps/rejected": -262.28009033203125,
      "loss": 0.207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8140567541122437,
      "rewards/margins": 2.0016896724700928,
      "rewards/rejected": -2.8157460689544678,
      "step": 11379
    },
    {
      "epoch": 1.4891469698208881,
      "grad_norm": 7.390861511230469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2584984600543976,
      "logits/rejected": -0.36959782242774963,
      "logps/chosen": -408.474853515625,
      "logps/rejected": -373.43609619140625,
      "loss": 0.4104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1956639289855957,
      "rewards/margins": 1.297169804573059,
      "rewards/rejected": -2.4928338527679443,
      "step": 11380
    },
    {
      "epoch": 1.48927782775824,
      "grad_norm": 8.473299980163574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2902829945087433,
      "logits/rejected": -0.47398772835731506,
      "logps/chosen": -393.7207946777344,
      "logps/rejected": -368.5758056640625,
      "loss": 0.4842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.455054759979248,
      "rewards/margins": 1.0980337858200073,
      "rewards/rejected": -3.553088426589966,
      "step": 11381
    },
    {
      "epoch": 1.4894086856955917,
      "grad_norm": 10.508378982543945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30828726291656494,
      "logits/rejected": -0.2661840319633484,
      "logps/chosen": -645.31884765625,
      "logps/rejected": -457.06927490234375,
      "loss": 0.5107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.444275975227356,
      "rewards/margins": 1.5824192762374878,
      "rewards/rejected": -3.0266952514648438,
      "step": 11382
    },
    {
      "epoch": 1.4895395436329435,
      "grad_norm": 8.827201843261719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21937721967697144,
      "logits/rejected": -0.1789921224117279,
      "logps/chosen": -393.36456298828125,
      "logps/rejected": -335.99322509765625,
      "loss": 0.6108,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2483532428741455,
      "rewards/margins": 0.6723647713661194,
      "rewards/rejected": -1.9207180738449097,
      "step": 11383
    },
    {
      "epoch": 1.4896704015702953,
      "grad_norm": 12.805853843688965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28263914585113525,
      "logits/rejected": -0.3071478307247162,
      "logps/chosen": -488.15997314453125,
      "logps/rejected": -367.42132568359375,
      "loss": 0.7911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4723573923110962,
      "rewards/margins": 1.3480042219161987,
      "rewards/rejected": -2.820361614227295,
      "step": 11384
    },
    {
      "epoch": 1.489801259507647,
      "grad_norm": 10.607537269592285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4378241300582886,
      "logits/rejected": -0.33469358086586,
      "logps/chosen": -567.2169189453125,
      "logps/rejected": -492.9149169921875,
      "loss": 0.536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9623880386352539,
      "rewards/margins": 1.0849111080169678,
      "rewards/rejected": -2.0472991466522217,
      "step": 11385
    },
    {
      "epoch": 1.4899321174449986,
      "grad_norm": 57.64712905883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3895101249217987,
      "logits/rejected": -0.28033509850502014,
      "logps/chosen": -833.62548828125,
      "logps/rejected": -872.427734375,
      "loss": 0.9691,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.1928695440292358,
      "rewards/margins": -0.09592305123806,
      "rewards/rejected": -1.0969464778900146,
      "step": 11386
    },
    {
      "epoch": 1.4900629753823504,
      "grad_norm": 10.142215728759766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.039744146168231964,
      "logits/rejected": -0.26649463176727295,
      "logps/chosen": -377.83203125,
      "logps/rejected": -442.47857666015625,
      "loss": 0.5304,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8643890619277954,
      "rewards/margins": 0.7311010956764221,
      "rewards/rejected": -2.5954902172088623,
      "step": 11387
    },
    {
      "epoch": 1.4901938333197022,
      "grad_norm": 7.569471836090088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1760222464799881,
      "logits/rejected": -0.3506760895252228,
      "logps/chosen": -432.78515625,
      "logps/rejected": -457.0629577636719,
      "loss": 0.4933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8793851137161255,
      "rewards/margins": 0.9168539643287659,
      "rewards/rejected": -2.796239137649536,
      "step": 11388
    },
    {
      "epoch": 1.490324691257054,
      "grad_norm": 15.337552070617676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13807928562164307,
      "logits/rejected": -0.1074749231338501,
      "logps/chosen": -671.0404052734375,
      "logps/rejected": -541.5460815429688,
      "loss": 0.8002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5978312492370605,
      "rewards/margins": 0.5554866790771484,
      "rewards/rejected": -3.153317928314209,
      "step": 11389
    },
    {
      "epoch": 1.4904555491944058,
      "grad_norm": 17.728939056396484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34819403290748596,
      "logits/rejected": -0.19838160276412964,
      "logps/chosen": -487.34375,
      "logps/rejected": -469.91485595703125,
      "loss": 0.9346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.829882264137268,
      "rewards/margins": 0.5011206865310669,
      "rewards/rejected": -2.331002950668335,
      "step": 11390
    },
    {
      "epoch": 1.4905864071317576,
      "grad_norm": 7.484279632568359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29467064142227173,
      "logits/rejected": -0.28371718525886536,
      "logps/chosen": -303.83740234375,
      "logps/rejected": -386.320068359375,
      "loss": 0.3439,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.199462890625,
      "rewards/margins": 1.4026635885238647,
      "rewards/rejected": -2.6021265983581543,
      "step": 11391
    },
    {
      "epoch": 1.4907172650691094,
      "grad_norm": 11.408227920532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4539085924625397,
      "logits/rejected": -0.36075061559677124,
      "logps/chosen": -602.04541015625,
      "logps/rejected": -584.9744873046875,
      "loss": 0.5178,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3818747997283936,
      "rewards/margins": 1.5087944269180298,
      "rewards/rejected": -2.890669345855713,
      "step": 11392
    },
    {
      "epoch": 1.4908481230064612,
      "grad_norm": 6.576616287231445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15793240070343018,
      "logits/rejected": -0.13437405228614807,
      "logps/chosen": -406.2174377441406,
      "logps/rejected": -398.30126953125,
      "loss": 0.2608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3654189109802246,
      "rewards/margins": 1.56975519657135,
      "rewards/rejected": -2.935173988342285,
      "step": 11393
    },
    {
      "epoch": 1.490978980943813,
      "grad_norm": 12.835309028625488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29960325360298157,
      "logits/rejected": -0.29488590359687805,
      "logps/chosen": -332.4791259765625,
      "logps/rejected": -331.0166931152344,
      "loss": 0.4354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2562752962112427,
      "rewards/margins": 1.0722103118896484,
      "rewards/rejected": -2.3284857273101807,
      "step": 11394
    },
    {
      "epoch": 1.4911098388811648,
      "grad_norm": 3.2821192741394043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2643607556819916,
      "logits/rejected": -0.33653947710990906,
      "logps/chosen": -550.3052368164062,
      "logps/rejected": -441.4950256347656,
      "loss": 0.0994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4001392722129822,
      "rewards/margins": 2.5290918350219727,
      "rewards/rejected": -2.9292311668395996,
      "step": 11395
    },
    {
      "epoch": 1.4912406968185163,
      "grad_norm": 6.270965576171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11532949656248093,
      "logits/rejected": -0.07927638292312622,
      "logps/chosen": -344.1430969238281,
      "logps/rejected": -486.82550048828125,
      "loss": 0.2901,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1506617069244385,
      "rewards/margins": 1.702728271484375,
      "rewards/rejected": -2.8533899784088135,
      "step": 11396
    },
    {
      "epoch": 1.4913715547558681,
      "grad_norm": 6.9745588302612305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020115084946155548,
      "logits/rejected": -0.13289102911949158,
      "logps/chosen": -352.8970947265625,
      "logps/rejected": -509.7057189941406,
      "loss": 0.4263,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7854623794555664,
      "rewards/margins": 1.2732858657836914,
      "rewards/rejected": -3.058748245239258,
      "step": 11397
    },
    {
      "epoch": 1.49150241269322,
      "grad_norm": 9.667283058166504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021366380155086517,
      "logits/rejected": -0.03590072691440582,
      "logps/chosen": -501.9681091308594,
      "logps/rejected": -769.6058349609375,
      "loss": 0.3802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0040149688720703,
      "rewards/margins": 2.2068474292755127,
      "rewards/rejected": -3.210862398147583,
      "step": 11398
    },
    {
      "epoch": 1.4916332706305717,
      "grad_norm": 10.005319595336914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6200201511383057,
      "logits/rejected": -0.3266790211200714,
      "logps/chosen": -353.6015930175781,
      "logps/rejected": -503.73260498046875,
      "loss": 0.5622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.249826431274414,
      "rewards/margins": 0.6335654258728027,
      "rewards/rejected": -1.8833917379379272,
      "step": 11399
    },
    {
      "epoch": 1.4917641285679235,
      "grad_norm": 7.711019515991211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2589374780654907,
      "logits/rejected": -0.19334322214126587,
      "logps/chosen": -414.996337890625,
      "logps/rejected": -409.7958679199219,
      "loss": 0.3841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8881962299346924,
      "rewards/margins": 1.6339013576507568,
      "rewards/rejected": -3.522097587585449,
      "step": 11400
    },
    {
      "epoch": 1.491894986505275,
      "grad_norm": 8.3786039352417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29720133543014526,
      "logits/rejected": -0.2604868710041046,
      "logps/chosen": -403.36407470703125,
      "logps/rejected": -424.93212890625,
      "loss": 0.4352,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.258449912071228,
      "rewards/margins": 0.7744022607803345,
      "rewards/rejected": -2.0328519344329834,
      "step": 11401
    },
    {
      "epoch": 1.4920258444426269,
      "grad_norm": 8.981746673583984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42711368203163147,
      "logits/rejected": -0.6343366503715515,
      "logps/chosen": -474.6927795410156,
      "logps/rejected": -418.1343994140625,
      "loss": 0.3878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.525299310684204,
      "rewards/margins": 1.1408237218856812,
      "rewards/rejected": -2.666123151779175,
      "step": 11402
    },
    {
      "epoch": 1.4921567023799787,
      "grad_norm": 10.263318061828613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17088018357753754,
      "logits/rejected": -0.26438477635383606,
      "logps/chosen": -648.5325927734375,
      "logps/rejected": -539.278076171875,
      "loss": 0.6508,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5041290521621704,
      "rewards/margins": 0.8540495038032532,
      "rewards/rejected": -2.3581786155700684,
      "step": 11403
    },
    {
      "epoch": 1.4922875603173305,
      "grad_norm": 6.557051658630371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2559986114501953,
      "logits/rejected": -0.3274441063404083,
      "logps/chosen": -495.2364501953125,
      "logps/rejected": -433.9794006347656,
      "loss": 0.3446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0109286308288574,
      "rewards/margins": 1.127816081047058,
      "rewards/rejected": -3.138744592666626,
      "step": 11404
    },
    {
      "epoch": 1.4924184182546822,
      "grad_norm": 11.663384437561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026571335271000862,
      "logits/rejected": -0.2895864248275757,
      "logps/chosen": -322.7728576660156,
      "logps/rejected": -509.89324951171875,
      "loss": 0.7349,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.523651123046875,
      "rewards/margins": 0.42403051257133484,
      "rewards/rejected": -2.947681427001953,
      "step": 11405
    },
    {
      "epoch": 1.492549276192034,
      "grad_norm": 7.912986755371094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34793350100517273,
      "logits/rejected": -0.3660029172897339,
      "logps/chosen": -455.3041687011719,
      "logps/rejected": -378.8782043457031,
      "loss": 0.462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2521746158599854,
      "rewards/margins": 0.8782673478126526,
      "rewards/rejected": -2.1304421424865723,
      "step": 11406
    },
    {
      "epoch": 1.4926801341293858,
      "grad_norm": 8.81174087524414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2794290781021118,
      "logits/rejected": -0.3431907594203949,
      "logps/chosen": -445.1424560546875,
      "logps/rejected": -608.9846801757812,
      "loss": 0.447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0446410179138184,
      "rewards/margins": 1.0336745977401733,
      "rewards/rejected": -3.078315496444702,
      "step": 11407
    },
    {
      "epoch": 1.4928109920667376,
      "grad_norm": 5.613194942474365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21095019578933716,
      "logits/rejected": -0.21965453028678894,
      "logps/chosen": -322.4517822265625,
      "logps/rejected": -295.09161376953125,
      "loss": 0.2899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1922650337219238,
      "rewards/margins": 1.4339580535888672,
      "rewards/rejected": -2.626222848892212,
      "step": 11408
    },
    {
      "epoch": 1.4929418500040894,
      "grad_norm": 8.076499938964844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3674783706665039,
      "logits/rejected": -0.24405069649219513,
      "logps/chosen": -415.93548583984375,
      "logps/rejected": -492.944091796875,
      "loss": 0.4896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0034799575805664,
      "rewards/margins": 1.1147032976150513,
      "rewards/rejected": -3.118183135986328,
      "step": 11409
    },
    {
      "epoch": 1.4930727079414412,
      "grad_norm": 12.61716079711914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0033637359738349915,
      "logits/rejected": 0.0324663482606411,
      "logps/chosen": -594.9279174804688,
      "logps/rejected": -689.613037109375,
      "loss": 0.508,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5461530685424805,
      "rewards/margins": 1.198146104812622,
      "rewards/rejected": -2.7442989349365234,
      "step": 11410
    },
    {
      "epoch": 1.4932035658787928,
      "grad_norm": 9.656436920166016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2191542237997055,
      "logits/rejected": -0.3843417167663574,
      "logps/chosen": -368.18231201171875,
      "logps/rejected": -371.50970458984375,
      "loss": 0.3937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9349644184112549,
      "rewards/margins": 1.5773406028747559,
      "rewards/rejected": -2.5123047828674316,
      "step": 11411
    },
    {
      "epoch": 1.4933344238161446,
      "grad_norm": 11.504782676696777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18559016287326813,
      "logits/rejected": -0.08218607306480408,
      "logps/chosen": -493.9740905761719,
      "logps/rejected": -596.9766235351562,
      "loss": 0.7155,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.201568603515625,
      "rewards/margins": 0.7117048501968384,
      "rewards/rejected": -3.913273811340332,
      "step": 11412
    },
    {
      "epoch": 1.4934652817534964,
      "grad_norm": 6.502353668212891,
      "learning_rate": 5e-06,
      "logits/chosen": -0.403841108083725,
      "logits/rejected": -0.445175439119339,
      "logps/chosen": -371.8553771972656,
      "logps/rejected": -411.6537780761719,
      "loss": 0.2595,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4030179977416992,
      "rewards/margins": 2.0512664318084717,
      "rewards/rejected": -3.454284429550171,
      "step": 11413
    },
    {
      "epoch": 1.4935961396908481,
      "grad_norm": 5.500950813293457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17871582508087158,
      "logits/rejected": -0.12838566303253174,
      "logps/chosen": -630.2469482421875,
      "logps/rejected": -543.9378662109375,
      "loss": 0.2904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1133873462677002,
      "rewards/margins": 2.1205759048461914,
      "rewards/rejected": -3.2339630126953125,
      "step": 11414
    },
    {
      "epoch": 1.4937269976282,
      "grad_norm": 8.56893253326416,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028741173446178436,
      "logits/rejected": -0.24358013272285461,
      "logps/chosen": -376.3283386230469,
      "logps/rejected": -378.2747497558594,
      "loss": 0.4549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7321456670761108,
      "rewards/margins": 1.5482244491577148,
      "rewards/rejected": -3.2803704738616943,
      "step": 11415
    },
    {
      "epoch": 1.4938578555655517,
      "grad_norm": 12.153703689575195,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010106708854436874,
      "logits/rejected": 0.008867897093296051,
      "logps/chosen": -473.7676086425781,
      "logps/rejected": -531.9553833007812,
      "loss": 0.6853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6886537075042725,
      "rewards/margins": 0.9709658026695251,
      "rewards/rejected": -2.6596195697784424,
      "step": 11416
    },
    {
      "epoch": 1.4939887135029033,
      "grad_norm": 4.741099834442139,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11394962668418884,
      "logits/rejected": -0.1798744797706604,
      "logps/chosen": -579.3680419921875,
      "logps/rejected": -459.3039855957031,
      "loss": 0.1548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7463805675506592,
      "rewards/margins": 2.3995981216430664,
      "rewards/rejected": -4.145978927612305,
      "step": 11417
    },
    {
      "epoch": 1.494119571440255,
      "grad_norm": 7.680670261383057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3326529860496521,
      "logits/rejected": -0.381700336933136,
      "logps/chosen": -596.85400390625,
      "logps/rejected": -513.2220458984375,
      "loss": 0.3446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7116087675094604,
      "rewards/margins": 1.5446107387542725,
      "rewards/rejected": -3.2562191486358643,
      "step": 11418
    },
    {
      "epoch": 1.4942504293776069,
      "grad_norm": 13.516422271728516,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08743423223495483,
      "logits/rejected": -0.03531099855899811,
      "logps/chosen": -411.13427734375,
      "logps/rejected": -402.25262451171875,
      "loss": 0.7756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9705170392990112,
      "rewards/margins": 0.5470152497291565,
      "rewards/rejected": -2.5175323486328125,
      "step": 11419
    },
    {
      "epoch": 1.4943812873149587,
      "grad_norm": 12.290239334106445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3743996322154999,
      "logits/rejected": -0.2761402726173401,
      "logps/chosen": -614.3638305664062,
      "logps/rejected": -664.147216796875,
      "loss": 0.5654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7296921014785767,
      "rewards/margins": 1.066714882850647,
      "rewards/rejected": -2.7964069843292236,
      "step": 11420
    },
    {
      "epoch": 1.4945121452523105,
      "grad_norm": 20.210378646850586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41711390018463135,
      "logits/rejected": -0.4990164041519165,
      "logps/chosen": -568.5291137695312,
      "logps/rejected": -555.2876586914062,
      "loss": 0.7663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.0195038318634033,
      "rewards/margins": 0.8873187899589539,
      "rewards/rejected": -3.906822443008423,
      "step": 11421
    },
    {
      "epoch": 1.4946430031896623,
      "grad_norm": 4.267780780792236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16760936379432678,
      "logits/rejected": -0.3663403391838074,
      "logps/chosen": -557.6834106445312,
      "logps/rejected": -527.4108276367188,
      "loss": 0.1882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9805206060409546,
      "rewards/margins": 2.476738452911377,
      "rewards/rejected": -3.457258939743042,
      "step": 11422
    },
    {
      "epoch": 1.494773861127014,
      "grad_norm": 5.973083019256592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42842504382133484,
      "logits/rejected": -0.41967594623565674,
      "logps/chosen": -514.1680908203125,
      "logps/rejected": -378.9599914550781,
      "loss": 0.3576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2681186199188232,
      "rewards/margins": 1.279870867729187,
      "rewards/rejected": -3.5479896068573,
      "step": 11423
    },
    {
      "epoch": 1.4949047190643658,
      "grad_norm": 7.023331165313721,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16739483177661896,
      "logits/rejected": -0.32668277621269226,
      "logps/chosen": -547.5505981445312,
      "logps/rejected": -449.89202880859375,
      "loss": 0.3159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8901491165161133,
      "rewards/margins": 1.7700865268707275,
      "rewards/rejected": -2.660235643386841,
      "step": 11424
    },
    {
      "epoch": 1.4950355770017176,
      "grad_norm": 9.178391456604004,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015669677406549454,
      "logits/rejected": 0.18753983080387115,
      "logps/chosen": -504.5631103515625,
      "logps/rejected": -530.1378173828125,
      "loss": 0.4952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6856292486190796,
      "rewards/margins": 1.7674777507781982,
      "rewards/rejected": -2.4531071186065674,
      "step": 11425
    },
    {
      "epoch": 1.4951664349390692,
      "grad_norm": 15.466231346130371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34579384326934814,
      "logits/rejected": -0.3620976507663727,
      "logps/chosen": -589.3709106445312,
      "logps/rejected": -519.8519287109375,
      "loss": 0.8152,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8087096214294434,
      "rewards/margins": 1.204848051071167,
      "rewards/rejected": -4.013557434082031,
      "step": 11426
    },
    {
      "epoch": 1.495297292876421,
      "grad_norm": 8.79183578491211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21615225076675415,
      "logits/rejected": -0.47086116671562195,
      "logps/chosen": -452.79107666015625,
      "logps/rejected": -468.7611999511719,
      "loss": 0.5408,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.56239652633667,
      "rewards/margins": 1.2357826232910156,
      "rewards/rejected": -2.7981791496276855,
      "step": 11427
    },
    {
      "epoch": 1.4954281508137728,
      "grad_norm": 9.124067306518555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19232042133808136,
      "logits/rejected": -0.08177397400140762,
      "logps/chosen": -410.336669921875,
      "logps/rejected": -525.71337890625,
      "loss": 0.3828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0069085359573364,
      "rewards/margins": 2.2406232357025146,
      "rewards/rejected": -3.2475321292877197,
      "step": 11428
    },
    {
      "epoch": 1.4955590087511246,
      "grad_norm": 8.96733283996582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3971153497695923,
      "logits/rejected": -0.6160216927528381,
      "logps/chosen": -342.5124206542969,
      "logps/rejected": -488.87799072265625,
      "loss": 0.4025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5592763423919678,
      "rewards/margins": 1.74349045753479,
      "rewards/rejected": -3.302766799926758,
      "step": 11429
    },
    {
      "epoch": 1.4956898666884764,
      "grad_norm": 7.859408855438232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18421271443367004,
      "logits/rejected": -0.031663037836551666,
      "logps/chosen": -379.0842590332031,
      "logps/rejected": -352.5022888183594,
      "loss": 0.3411,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4821025133132935,
      "rewards/margins": 1.4042261838912964,
      "rewards/rejected": -2.88632869720459,
      "step": 11430
    },
    {
      "epoch": 1.4958207246258282,
      "grad_norm": 8.81680679321289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38248583674430847,
      "logits/rejected": -0.5005359053611755,
      "logps/chosen": -397.1316223144531,
      "logps/rejected": -388.3145446777344,
      "loss": 0.3968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6219511032104492,
      "rewards/margins": 1.2386791706085205,
      "rewards/rejected": -2.8606302738189697,
      "step": 11431
    },
    {
      "epoch": 1.4959515825631797,
      "grad_norm": 11.38847827911377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12247101217508316,
      "logits/rejected": -0.13052377104759216,
      "logps/chosen": -557.9290771484375,
      "logps/rejected": -549.2867431640625,
      "loss": 0.5619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8746975064277649,
      "rewards/margins": 1.3018742799758911,
      "rewards/rejected": -2.176571846008301,
      "step": 11432
    },
    {
      "epoch": 1.4960824405005315,
      "grad_norm": 8.425314903259277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1088225394487381,
      "logits/rejected": -0.2096511423587799,
      "logps/chosen": -547.8859252929688,
      "logps/rejected": -667.29443359375,
      "loss": 0.3536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.114840507507324,
      "rewards/margins": 1.8799315690994263,
      "rewards/rejected": -3.994771957397461,
      "step": 11433
    },
    {
      "epoch": 1.4962132984378833,
      "grad_norm": 6.510402202606201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25732487440109253,
      "logits/rejected": -0.37576594948768616,
      "logps/chosen": -470.32421875,
      "logps/rejected": -535.6182861328125,
      "loss": 0.2671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.961643934249878,
      "rewards/margins": 1.7471542358398438,
      "rewards/rejected": -3.708798408508301,
      "step": 11434
    },
    {
      "epoch": 1.496344156375235,
      "grad_norm": 5.584936618804932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15002676844596863,
      "logits/rejected": -0.5176147222518921,
      "logps/chosen": -621.0762939453125,
      "logps/rejected": -431.306884765625,
      "loss": 0.3329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0599220991134644,
      "rewards/margins": 1.7907850742340088,
      "rewards/rejected": -2.8507072925567627,
      "step": 11435
    },
    {
      "epoch": 1.496475014312587,
      "grad_norm": 6.256953716278076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13780871033668518,
      "logits/rejected": -0.34147632122039795,
      "logps/chosen": -660.35302734375,
      "logps/rejected": -688.4597778320312,
      "loss": 0.2401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2342087030410767,
      "rewards/margins": 1.8641263246536255,
      "rewards/rejected": -3.098335027694702,
      "step": 11436
    },
    {
      "epoch": 1.4966058722499387,
      "grad_norm": 10.533526420593262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5291510820388794,
      "logits/rejected": -0.2904810309410095,
      "logps/chosen": -389.9578552246094,
      "logps/rejected": -384.2412414550781,
      "loss": 0.5589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1451205015182495,
      "rewards/margins": 0.5566266775131226,
      "rewards/rejected": -1.701747179031372,
      "step": 11437
    },
    {
      "epoch": 1.4967367301872905,
      "grad_norm": 8.600995063781738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5694713592529297,
      "logits/rejected": -0.4799889922142029,
      "logps/chosen": -473.0036926269531,
      "logps/rejected": -347.1898193359375,
      "loss": 0.4024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5528440475463867,
      "rewards/margins": 1.460623860359192,
      "rewards/rejected": -4.013468265533447,
      "step": 11438
    },
    {
      "epoch": 1.4968675881246423,
      "grad_norm": 3.420945644378662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4096246361732483,
      "logits/rejected": -0.3734401762485504,
      "logps/chosen": -271.1612243652344,
      "logps/rejected": -463.7180480957031,
      "loss": 0.1223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7411677837371826,
      "rewards/margins": 2.802238941192627,
      "rewards/rejected": -3.5434064865112305,
      "step": 11439
    },
    {
      "epoch": 1.496998446061994,
      "grad_norm": 7.107016563415527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40804022550582886,
      "logits/rejected": -0.373549222946167,
      "logps/chosen": -379.7718811035156,
      "logps/rejected": -474.47698974609375,
      "loss": 0.2555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1902661323547363,
      "rewards/margins": 1.9488998651504517,
      "rewards/rejected": -3.1391658782958984,
      "step": 11440
    },
    {
      "epoch": 1.4971293039993456,
      "grad_norm": 7.26545524597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4159071743488312,
      "logits/rejected": -0.5084704756736755,
      "logps/chosen": -389.283203125,
      "logps/rejected": -432.225830078125,
      "loss": 0.5232,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8734890222549438,
      "rewards/margins": 0.7572870254516602,
      "rewards/rejected": -2.6307761669158936,
      "step": 11441
    },
    {
      "epoch": 1.4972601619366974,
      "grad_norm": 4.309539318084717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3864229619503021,
      "logits/rejected": -0.23053939640522003,
      "logps/chosen": -447.35150146484375,
      "logps/rejected": -505.42645263671875,
      "loss": 0.1652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1552224159240723,
      "rewards/margins": 2.099736452102661,
      "rewards/rejected": -4.254958629608154,
      "step": 11442
    },
    {
      "epoch": 1.4973910198740492,
      "grad_norm": 13.680840492248535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15058907866477966,
      "logits/rejected": -0.23647227883338928,
      "logps/chosen": -467.50048828125,
      "logps/rejected": -517.9737548828125,
      "loss": 0.7654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5444241762161255,
      "rewards/margins": 0.532738447189331,
      "rewards/rejected": -2.077162504196167,
      "step": 11443
    },
    {
      "epoch": 1.497521877811401,
      "grad_norm": 7.5894646644592285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3455400764942169,
      "logits/rejected": -0.5044906735420227,
      "logps/chosen": -354.97216796875,
      "logps/rejected": -385.3418884277344,
      "loss": 0.4086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.469943642616272,
      "rewards/margins": 0.9607818126678467,
      "rewards/rejected": -2.430725574493408,
      "step": 11444
    },
    {
      "epoch": 1.4976527357487528,
      "grad_norm": 8.065881729125977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21218480169773102,
      "logits/rejected": -0.2488444596529007,
      "logps/chosen": -512.621337890625,
      "logps/rejected": -511.11090087890625,
      "loss": 0.3611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2263271808624268,
      "rewards/margins": 1.6169912815093994,
      "rewards/rejected": -3.843318223953247,
      "step": 11445
    },
    {
      "epoch": 1.4977835936861046,
      "grad_norm": 6.6880059242248535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07017490267753601,
      "logits/rejected": -0.4739406108856201,
      "logps/chosen": -378.9654541015625,
      "logps/rejected": -389.8856506347656,
      "loss": 0.3085,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.941153645515442,
      "rewards/margins": 2.0381364822387695,
      "rewards/rejected": -3.979290246963501,
      "step": 11446
    },
    {
      "epoch": 1.4979144516234562,
      "grad_norm": 8.15898323059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4211326837539673,
      "logits/rejected": -0.53309166431427,
      "logps/chosen": -426.9301452636719,
      "logps/rejected": -444.97198486328125,
      "loss": 0.3663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.570072889328003,
      "rewards/margins": 2.1880264282226562,
      "rewards/rejected": -3.758099317550659,
      "step": 11447
    },
    {
      "epoch": 1.498045309560808,
      "grad_norm": 8.638404846191406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0858895555138588,
      "logits/rejected": -0.21299540996551514,
      "logps/chosen": -477.6564636230469,
      "logps/rejected": -458.1859130859375,
      "loss": 0.495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4134087562561035,
      "rewards/margins": 0.622128963470459,
      "rewards/rejected": -2.0355377197265625,
      "step": 11448
    },
    {
      "epoch": 1.4981761674981597,
      "grad_norm": 9.29655647277832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0508807897567749,
      "logits/rejected": -0.3978414237499237,
      "logps/chosen": -637.8217163085938,
      "logps/rejected": -470.8242492675781,
      "loss": 0.3906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2724369764328003,
      "rewards/margins": 1.4101133346557617,
      "rewards/rejected": -2.6825504302978516,
      "step": 11449
    },
    {
      "epoch": 1.4983070254355115,
      "grad_norm": 7.370894432067871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36133381724357605,
      "logits/rejected": -0.33448684215545654,
      "logps/chosen": -384.863525390625,
      "logps/rejected": -462.85198974609375,
      "loss": 0.4095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6907286643981934,
      "rewards/margins": 0.9891279339790344,
      "rewards/rejected": -2.679856300354004,
      "step": 11450
    },
    {
      "epoch": 1.4984378833728633,
      "grad_norm": 12.661423683166504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5428383350372314,
      "logits/rejected": -0.5124842524528503,
      "logps/chosen": -527.0423583984375,
      "logps/rejected": -638.4228515625,
      "loss": 0.66,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.202206611633301,
      "rewards/margins": 0.9203373193740845,
      "rewards/rejected": -3.122544050216675,
      "step": 11451
    },
    {
      "epoch": 1.4985687413102151,
      "grad_norm": 9.006101608276367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07155841588973999,
      "logits/rejected": -0.24268127977848053,
      "logps/chosen": -642.4481201171875,
      "logps/rejected": -492.77520751953125,
      "loss": 0.4261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.415413498878479,
      "rewards/margins": 1.436884880065918,
      "rewards/rejected": -2.8522982597351074,
      "step": 11452
    },
    {
      "epoch": 1.498699599247567,
      "grad_norm": 9.594466209411621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021753838285803795,
      "logits/rejected": -0.034842438995838165,
      "logps/chosen": -637.189208984375,
      "logps/rejected": -575.7390747070312,
      "loss": 0.6364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7126433849334717,
      "rewards/margins": 1.2087715864181519,
      "rewards/rejected": -3.921414852142334,
      "step": 11453
    },
    {
      "epoch": 1.4988304571849187,
      "grad_norm": 10.264049530029297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09654691815376282,
      "logits/rejected": -0.16204890608787537,
      "logps/chosen": -552.925048828125,
      "logps/rejected": -532.9732055664062,
      "loss": 0.3672,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0619640350341797,
      "rewards/margins": 1.4273877143859863,
      "rewards/rejected": -3.489351511001587,
      "step": 11454
    },
    {
      "epoch": 1.4989613151222705,
      "grad_norm": 8.572005271911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6508054137229919,
      "logits/rejected": -0.4562031626701355,
      "logps/chosen": -898.191162109375,
      "logps/rejected": -523.5918579101562,
      "loss": 0.3872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.3903939723968506,
      "rewards/margins": 1.7242558002471924,
      "rewards/rejected": -4.114649772644043,
      "step": 11455
    },
    {
      "epoch": 1.4990921730596223,
      "grad_norm": 6.137001037597656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22474277019500732,
      "logits/rejected": -0.44818243384361267,
      "logps/chosen": -477.2856750488281,
      "logps/rejected": -450.5358581542969,
      "loss": 0.2778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.223515272140503,
      "rewards/margins": 2.644263744354248,
      "rewards/rejected": -3.867779016494751,
      "step": 11456
    },
    {
      "epoch": 1.4992230309969738,
      "grad_norm": 6.281087398529053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15911337733268738,
      "logits/rejected": -0.18415802717208862,
      "logps/chosen": -633.28173828125,
      "logps/rejected": -535.0261840820312,
      "loss": 0.1886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3980395793914795,
      "rewards/margins": 2.231682300567627,
      "rewards/rejected": -3.6297221183776855,
      "step": 11457
    },
    {
      "epoch": 1.4993538889343256,
      "grad_norm": 13.044903755187988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1539103388786316,
      "logits/rejected": -0.027167554944753647,
      "logps/chosen": -456.77093505859375,
      "logps/rejected": -432.2450866699219,
      "loss": 0.8295,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.1341114044189453,
      "rewards/margins": 0.5270705819129944,
      "rewards/rejected": -2.661181926727295,
      "step": 11458
    },
    {
      "epoch": 1.4994847468716774,
      "grad_norm": 11.618698120117188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030901439487934113,
      "logits/rejected": -0.3566184639930725,
      "logps/chosen": -731.0068359375,
      "logps/rejected": -464.52587890625,
      "loss": 0.5401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.228614330291748,
      "rewards/margins": 1.1286267042160034,
      "rewards/rejected": -2.357240915298462,
      "step": 11459
    },
    {
      "epoch": 1.4996156048090292,
      "grad_norm": 10.55298137664795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4407770037651062,
      "logits/rejected": -0.6845635771751404,
      "logps/chosen": -484.37445068359375,
      "logps/rejected": -532.1492919921875,
      "loss": 0.4884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.59989333152771,
      "rewards/margins": 1.07571542263031,
      "rewards/rejected": -3.6756091117858887,
      "step": 11460
    },
    {
      "epoch": 1.499746462746381,
      "grad_norm": 10.599178314208984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23229525983333588,
      "logits/rejected": -0.3011787533760071,
      "logps/chosen": -455.1977844238281,
      "logps/rejected": -468.62310791015625,
      "loss": 0.6193,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9129691123962402,
      "rewards/margins": 0.7474203705787659,
      "rewards/rejected": -2.6603894233703613,
      "step": 11461
    },
    {
      "epoch": 1.4998773206837326,
      "grad_norm": 12.51822280883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010688118636608124,
      "logits/rejected": -0.1015913337469101,
      "logps/chosen": -539.7134399414062,
      "logps/rejected": -474.82415771484375,
      "loss": 0.5374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5742995738983154,
      "rewards/margins": 1.1821751594543457,
      "rewards/rejected": -2.756474494934082,
      "step": 11462
    },
    {
      "epoch": 1.5000081786210844,
      "grad_norm": 6.328069686889648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12372995167970657,
      "logits/rejected": -0.22812460362911224,
      "logps/chosen": -486.2270812988281,
      "logps/rejected": -459.255126953125,
      "loss": 0.3039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2161823511123657,
      "rewards/margins": 1.8666703701019287,
      "rewards/rejected": -3.082852840423584,
      "step": 11463
    },
    {
      "epoch": 1.5001390365584362,
      "grad_norm": 8.499714851379395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38022562861442566,
      "logits/rejected": -0.3239619731903076,
      "logps/chosen": -469.49786376953125,
      "logps/rejected": -290.0467834472656,
      "loss": 0.4502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4025700092315674,
      "rewards/margins": 0.902522087097168,
      "rewards/rejected": -3.3050920963287354,
      "step": 11464
    },
    {
      "epoch": 1.500269894495788,
      "grad_norm": 6.573788166046143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.163237527012825,
      "logits/rejected": 0.009125448763370514,
      "logps/chosen": -449.57159423828125,
      "logps/rejected": -543.3984375,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9736371040344238,
      "rewards/margins": 1.5168559551239014,
      "rewards/rejected": -3.490493059158325,
      "step": 11465
    },
    {
      "epoch": 1.5004007524331398,
      "grad_norm": 10.457606315612793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25074830651283264,
      "logits/rejected": -0.4085027277469635,
      "logps/chosen": -470.6346740722656,
      "logps/rejected": -472.0989685058594,
      "loss": 0.5493,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4644691944122314,
      "rewards/margins": 0.9430192708969116,
      "rewards/rejected": -3.4074885845184326,
      "step": 11466
    },
    {
      "epoch": 1.5005316103704915,
      "grad_norm": 13.011281967163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4939728081226349,
      "logits/rejected": -0.4846784472465515,
      "logps/chosen": -318.26220703125,
      "logps/rejected": -415.8916931152344,
      "loss": 0.8843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.5178909301757812,
      "rewards/margins": 0.7410256862640381,
      "rewards/rejected": -3.2589168548583984,
      "step": 11467
    },
    {
      "epoch": 1.5006624683078433,
      "grad_norm": 4.579956531524658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33046430349349976,
      "logits/rejected": -0.4064139425754547,
      "logps/chosen": -582.0264282226562,
      "logps/rejected": -663.0638427734375,
      "loss": 0.2163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0806838274002075,
      "rewards/margins": 2.4781363010406494,
      "rewards/rejected": -3.5588204860687256,
      "step": 11468
    },
    {
      "epoch": 1.5007933262451951,
      "grad_norm": 5.567379474639893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19286862015724182,
      "logits/rejected": -0.21137137711048126,
      "logps/chosen": -570.1439819335938,
      "logps/rejected": -654.690673828125,
      "loss": 0.2308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3141741752624512,
      "rewards/margins": 2.18426775932312,
      "rewards/rejected": -3.4984421730041504,
      "step": 11469
    },
    {
      "epoch": 1.500924184182547,
      "grad_norm": 6.1300435066223145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23525458574295044,
      "logits/rejected": -0.2973737120628357,
      "logps/chosen": -507.04638671875,
      "logps/rejected": -424.43011474609375,
      "loss": 0.387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9883873462677002,
      "rewards/margins": 1.4316648244857788,
      "rewards/rejected": -3.4200522899627686,
      "step": 11470
    },
    {
      "epoch": 1.5010550421198987,
      "grad_norm": 11.533028602600098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4981066882610321,
      "logits/rejected": -0.3770359456539154,
      "logps/chosen": -497.2238464355469,
      "logps/rejected": -532.242431640625,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9913816452026367,
      "rewards/margins": 0.3491290807723999,
      "rewards/rejected": -2.340510845184326,
      "step": 11471
    },
    {
      "epoch": 1.5011859000572505,
      "grad_norm": 13.175582885742188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46096497774124146,
      "logits/rejected": -0.1595754623413086,
      "logps/chosen": -443.1971740722656,
      "logps/rejected": -447.9534912109375,
      "loss": 0.6482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9727251529693604,
      "rewards/margins": 1.101988434791565,
      "rewards/rejected": -3.0747134685516357,
      "step": 11472
    },
    {
      "epoch": 1.501316757994602,
      "grad_norm": 3.707313299179077,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6495023369789124,
      "logits/rejected": -0.3734229803085327,
      "logps/chosen": -367.08447265625,
      "logps/rejected": -414.8945617675781,
      "loss": 0.1865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1076202392578125,
      "rewards/margins": 2.158783435821533,
      "rewards/rejected": -3.2664036750793457,
      "step": 11473
    },
    {
      "epoch": 1.5014476159319539,
      "grad_norm": 6.82712984085083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3766341209411621,
      "logits/rejected": -0.40955278277397156,
      "logps/chosen": -316.5438232421875,
      "logps/rejected": -329.7261657714844,
      "loss": 0.4007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2202469110488892,
      "rewards/margins": 1.5544787645339966,
      "rewards/rejected": -2.7747254371643066,
      "step": 11474
    },
    {
      "epoch": 1.5015784738693057,
      "grad_norm": 4.02445650100708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4679308533668518,
      "logits/rejected": -0.44957801699638367,
      "logps/chosen": -419.2878112792969,
      "logps/rejected": -430.66253662109375,
      "loss": 0.1722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.229461669921875,
      "rewards/margins": 2.124593734741211,
      "rewards/rejected": -3.354055643081665,
      "step": 11475
    },
    {
      "epoch": 1.5017093318066574,
      "grad_norm": 5.236974716186523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11451086401939392,
      "logits/rejected": -0.25236377120018005,
      "logps/chosen": -415.1900939941406,
      "logps/rejected": -464.349853515625,
      "loss": 0.2962,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2878421545028687,
      "rewards/margins": 1.606359839439392,
      "rewards/rejected": -2.89420223236084,
      "step": 11476
    },
    {
      "epoch": 1.501840189744009,
      "grad_norm": 4.852171421051025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13719837367534637,
      "logits/rejected": -0.09897966682910919,
      "logps/chosen": -395.5262451171875,
      "logps/rejected": -517.9678955078125,
      "loss": 0.245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8934035897254944,
      "rewards/margins": 1.5704460144042969,
      "rewards/rejected": -2.4638495445251465,
      "step": 11477
    },
    {
      "epoch": 1.5019710476813608,
      "grad_norm": 8.330527305603027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36193516850471497,
      "logits/rejected": -0.40050938725471497,
      "logps/chosen": -420.5788879394531,
      "logps/rejected": -611.1840209960938,
      "loss": 0.4403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2168054580688477,
      "rewards/margins": 1.444035530090332,
      "rewards/rejected": -2.660841226577759,
      "step": 11478
    },
    {
      "epoch": 1.5021019056187126,
      "grad_norm": 11.592784881591797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37876102328300476,
      "logits/rejected": -0.2707628309726715,
      "logps/chosen": -545.3650512695312,
      "logps/rejected": -412.9220275878906,
      "loss": 0.438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5881364345550537,
      "rewards/margins": 0.9037724733352661,
      "rewards/rejected": -3.4919087886810303,
      "step": 11479
    },
    {
      "epoch": 1.5022327635560644,
      "grad_norm": 8.600101470947266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29239127039909363,
      "logits/rejected": -0.17368435859680176,
      "logps/chosen": -556.1485595703125,
      "logps/rejected": -613.243896484375,
      "loss": 0.3876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4693617820739746,
      "rewards/margins": 1.5259736776351929,
      "rewards/rejected": -2.995335578918457,
      "step": 11480
    },
    {
      "epoch": 1.5023636214934162,
      "grad_norm": 7.786635398864746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.359408974647522,
      "logits/rejected": -0.10829170048236847,
      "logps/chosen": -333.65069580078125,
      "logps/rejected": -437.07061767578125,
      "loss": 0.3794,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4018352031707764,
      "rewards/margins": 1.859971523284912,
      "rewards/rejected": -3.2618069648742676,
      "step": 11481
    },
    {
      "epoch": 1.502494479430768,
      "grad_norm": 9.891900062561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5519692301750183,
      "logits/rejected": -0.4720098376274109,
      "logps/chosen": -428.15179443359375,
      "logps/rejected": -518.7354736328125,
      "loss": 0.6477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4785131216049194,
      "rewards/margins": 0.8174289464950562,
      "rewards/rejected": -2.2959423065185547,
      "step": 11482
    },
    {
      "epoch": 1.5026253373681198,
      "grad_norm": 7.991831302642822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3914245069026947,
      "logits/rejected": -0.3588836193084717,
      "logps/chosen": -406.7137145996094,
      "logps/rejected": -387.8897705078125,
      "loss": 0.3995,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4375437498092651,
      "rewards/margins": 1.2175483703613281,
      "rewards/rejected": -2.6550920009613037,
      "step": 11483
    },
    {
      "epoch": 1.5027561953054716,
      "grad_norm": 7.280370235443115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1840200275182724,
      "logits/rejected": 0.11638469994068146,
      "logps/chosen": -543.5728759765625,
      "logps/rejected": -580.1315307617188,
      "loss": 0.4777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2637624740600586,
      "rewards/margins": 0.8188926577568054,
      "rewards/rejected": -2.082655191421509,
      "step": 11484
    },
    {
      "epoch": 1.5028870532428233,
      "grad_norm": 8.215778350830078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3192474842071533,
      "logits/rejected": -0.2878553867340088,
      "logps/chosen": -364.3559265136719,
      "logps/rejected": -529.2279052734375,
      "loss": 0.3476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5001212358474731,
      "rewards/margins": 1.2450227737426758,
      "rewards/rejected": -2.7451438903808594,
      "step": 11485
    },
    {
      "epoch": 1.5030179111801751,
      "grad_norm": 5.011546611785889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2237389087677002,
      "logits/rejected": -0.3570995330810547,
      "logps/chosen": -545.2634887695312,
      "logps/rejected": -390.3185729980469,
      "loss": 0.2654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4367977380752563,
      "rewards/margins": 1.33785879611969,
      "rewards/rejected": -2.7746567726135254,
      "step": 11486
    },
    {
      "epoch": 1.503148769117527,
      "grad_norm": 27.53611183166504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40358540415763855,
      "logits/rejected": -0.41550973057746887,
      "logps/chosen": -553.9951171875,
      "logps/rejected": -463.3761901855469,
      "loss": 0.8685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.102935552597046,
      "rewards/margins": 0.5099972486495972,
      "rewards/rejected": -2.6129326820373535,
      "step": 11487
    },
    {
      "epoch": 1.5032796270548785,
      "grad_norm": 11.454669952392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2699531316757202,
      "logits/rejected": -0.34687483310699463,
      "logps/chosen": -560.1803588867188,
      "logps/rejected": -465.7613525390625,
      "loss": 0.8144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9771232008934021,
      "rewards/margins": 0.99313884973526,
      "rewards/rejected": -1.9702622890472412,
      "step": 11488
    },
    {
      "epoch": 1.5034104849922303,
      "grad_norm": 13.006392478942871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37139177322387695,
      "logits/rejected": -0.5243044495582581,
      "logps/chosen": -485.7864685058594,
      "logps/rejected": -400.933349609375,
      "loss": 0.7559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9304200410842896,
      "rewards/margins": 1.1265610456466675,
      "rewards/rejected": -3.056981325149536,
      "step": 11489
    },
    {
      "epoch": 1.503541342929582,
      "grad_norm": 9.225001335144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29353588819503784,
      "logits/rejected": -0.1220732033252716,
      "logps/chosen": -394.0777282714844,
      "logps/rejected": -406.1754455566406,
      "loss": 0.2951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4504061937332153,
      "rewards/margins": 2.1725807189941406,
      "rewards/rejected": -3.6229865550994873,
      "step": 11490
    },
    {
      "epoch": 1.5036722008669339,
      "grad_norm": 14.728495597839355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16700081527233124,
      "logits/rejected": -0.35889101028442383,
      "logps/chosen": -522.5437622070312,
      "logps/rejected": -479.81549072265625,
      "loss": 0.8232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2047863006591797,
      "rewards/margins": 0.33045002818107605,
      "rewards/rejected": -2.535236358642578,
      "step": 11491
    },
    {
      "epoch": 1.5038030588042854,
      "grad_norm": 10.397266387939453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15570777654647827,
      "logits/rejected": -0.3349633812904358,
      "logps/chosen": -458.52044677734375,
      "logps/rejected": -550.929931640625,
      "loss": 0.4576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.7946255207061768,
      "rewards/margins": 1.017006516456604,
      "rewards/rejected": -3.8116321563720703,
      "step": 11492
    },
    {
      "epoch": 1.5039339167416372,
      "grad_norm": 8.827811241149902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22372031211853027,
      "logits/rejected": -0.446681410074234,
      "logps/chosen": -520.6913452148438,
      "logps/rejected": -455.7497253417969,
      "loss": 0.481,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.568415880203247,
      "rewards/margins": 1.1511465311050415,
      "rewards/rejected": -2.719562530517578,
      "step": 11493
    },
    {
      "epoch": 1.504064774678989,
      "grad_norm": 5.015070915222168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.083860382437706,
      "logits/rejected": -0.19369962811470032,
      "logps/chosen": -430.97216796875,
      "logps/rejected": -578.6034545898438,
      "loss": 0.2482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3933165073394775,
      "rewards/margins": 1.7601653337478638,
      "rewards/rejected": -3.153481960296631,
      "step": 11494
    },
    {
      "epoch": 1.5041956326163408,
      "grad_norm": 4.093397617340088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2768821120262146,
      "logits/rejected": -0.2858216166496277,
      "logps/chosen": -359.3768005371094,
      "logps/rejected": -339.03985595703125,
      "loss": 0.2394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.136470079421997,
      "rewards/margins": 1.8502166271209717,
      "rewards/rejected": -2.9866864681243896,
      "step": 11495
    },
    {
      "epoch": 1.5043264905536926,
      "grad_norm": 11.199247360229492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08231654763221741,
      "logits/rejected": -0.2186538577079773,
      "logps/chosen": -556.5380859375,
      "logps/rejected": -490.2154846191406,
      "loss": 0.5675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9232981204986572,
      "rewards/margins": 0.45055899024009705,
      "rewards/rejected": -2.373857021331787,
      "step": 11496
    },
    {
      "epoch": 1.5044573484910444,
      "grad_norm": 11.05552864074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22202062606811523,
      "logits/rejected": -0.023740746080875397,
      "logps/chosen": -606.0730590820312,
      "logps/rejected": -638.9999389648438,
      "loss": 0.4738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5174015760421753,
      "rewards/margins": 1.438248872756958,
      "rewards/rejected": -2.9556503295898438,
      "step": 11497
    },
    {
      "epoch": 1.5045882064283962,
      "grad_norm": 6.843719959259033,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46731072664260864,
      "logits/rejected": -0.32337766885757446,
      "logps/chosen": -480.9826965332031,
      "logps/rejected": -424.1787414550781,
      "loss": 0.3241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.043893814086914,
      "rewards/margins": 1.6839473247528076,
      "rewards/rejected": -3.7278411388397217,
      "step": 11498
    },
    {
      "epoch": 1.504719064365748,
      "grad_norm": 6.903085708618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06559739261865616,
      "logits/rejected": -0.027248471975326538,
      "logps/chosen": -384.1566467285156,
      "logps/rejected": -497.4890441894531,
      "loss": 0.294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8128485679626465,
      "rewards/margins": 1.4970827102661133,
      "rewards/rejected": -3.309931516647339,
      "step": 11499
    },
    {
      "epoch": 1.5048499223030998,
      "grad_norm": 8.220294952392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3825417459011078,
      "logits/rejected": -0.46644753217697144,
      "logps/chosen": -314.35565185546875,
      "logps/rejected": -371.031494140625,
      "loss": 0.4506,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1848044395446777,
      "rewards/margins": 1.4390227794647217,
      "rewards/rejected": -3.6238274574279785,
      "step": 11500
    },
    {
      "epoch": 1.5049807802404516,
      "grad_norm": 4.922706604003906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049382079392671585,
      "logits/rejected": 0.07500670850276947,
      "logps/chosen": -475.6695251464844,
      "logps/rejected": -530.874755859375,
      "loss": 0.1962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1872442960739136,
      "rewards/margins": 2.7203986644744873,
      "rewards/rejected": -3.9076430797576904,
      "step": 11501
    },
    {
      "epoch": 1.5051116381778034,
      "grad_norm": 7.20278787612915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2000158429145813,
      "logits/rejected": -0.35347408056259155,
      "logps/chosen": -535.143310546875,
      "logps/rejected": -522.34619140625,
      "loss": 0.5352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.530707597732544,
      "rewards/margins": 0.7995014190673828,
      "rewards/rejected": -2.330209255218506,
      "step": 11502
    },
    {
      "epoch": 1.5052424961151551,
      "grad_norm": 7.370347023010254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22565077245235443,
      "logits/rejected": -0.08700507879257202,
      "logps/chosen": -637.4940185546875,
      "logps/rejected": -571.8687133789062,
      "loss": 0.3947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0894849300384521,
      "rewards/margins": 1.3550655841827393,
      "rewards/rejected": -2.4445505142211914,
      "step": 11503
    },
    {
      "epoch": 1.5053733540525067,
      "grad_norm": 7.408362865447998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2991552948951721,
      "logits/rejected": -0.09906870126724243,
      "logps/chosen": -480.1555480957031,
      "logps/rejected": -387.5795593261719,
      "loss": 0.3693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5413399934768677,
      "rewards/margins": 1.1382378339767456,
      "rewards/rejected": -2.679577589035034,
      "step": 11504
    },
    {
      "epoch": 1.5055042119898585,
      "grad_norm": 7.592654705047607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2205933779478073,
      "logits/rejected": 0.07023479789495468,
      "logps/chosen": -456.6312561035156,
      "logps/rejected": -409.13397216796875,
      "loss": 0.3418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0088951587677002,
      "rewards/margins": 1.5236577987670898,
      "rewards/rejected": -2.53255295753479,
      "step": 11505
    },
    {
      "epoch": 1.5056350699272103,
      "grad_norm": 8.744768142700195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2888511121273041,
      "logits/rejected": -0.24820522964000702,
      "logps/chosen": -506.2476806640625,
      "logps/rejected": -410.9569091796875,
      "loss": 0.5806,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9663103818893433,
      "rewards/margins": 0.6654505133628845,
      "rewards/rejected": -2.631760835647583,
      "step": 11506
    },
    {
      "epoch": 1.5057659278645619,
      "grad_norm": 9.88427734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0008629262447357178,
      "logits/rejected": -0.032991424202919006,
      "logps/chosen": -475.90924072265625,
      "logps/rejected": -470.0997314453125,
      "loss": 0.5033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2644760608673096,
      "rewards/margins": 1.176631212234497,
      "rewards/rejected": -2.4411072731018066,
      "step": 11507
    },
    {
      "epoch": 1.5058967858019137,
      "grad_norm": 6.664958953857422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46698322892189026,
      "logits/rejected": -0.4655405879020691,
      "logps/chosen": -653.334228515625,
      "logps/rejected": -510.46124267578125,
      "loss": 0.3121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16774116456508636,
      "rewards/margins": 1.6828653812408447,
      "rewards/rejected": -1.8506065607070923,
      "step": 11508
    },
    {
      "epoch": 1.5060276437392655,
      "grad_norm": 6.400010108947754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35474884510040283,
      "logits/rejected": -0.2347806841135025,
      "logps/chosen": -727.564697265625,
      "logps/rejected": -539.0296020507812,
      "loss": 0.345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6579237580299377,
      "rewards/margins": 1.5394130945205688,
      "rewards/rejected": -2.1973366737365723,
      "step": 11509
    },
    {
      "epoch": 1.5061585016766172,
      "grad_norm": 3.9757986068725586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17129769921302795,
      "logits/rejected": -0.36718809604644775,
      "logps/chosen": -571.2376708984375,
      "logps/rejected": -526.2816162109375,
      "loss": 0.1494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.428257942199707,
      "rewards/margins": 2.7301366329193115,
      "rewards/rejected": -4.1583943367004395,
      "step": 11510
    },
    {
      "epoch": 1.506289359613969,
      "grad_norm": 8.709755897521973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13129779696464539,
      "logits/rejected": -0.17598389089107513,
      "logps/chosen": -337.52008056640625,
      "logps/rejected": -335.5408630371094,
      "loss": 0.5175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9900918006896973,
      "rewards/margins": 0.8913594484329224,
      "rewards/rejected": -1.8814512491226196,
      "step": 11511
    },
    {
      "epoch": 1.5064202175513208,
      "grad_norm": 9.39218807220459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4288631081581116,
      "logits/rejected": -0.2666069269180298,
      "logps/chosen": -470.05474853515625,
      "logps/rejected": -433.3060302734375,
      "loss": 0.4729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.811269760131836,
      "rewards/margins": 0.5926796197891235,
      "rewards/rejected": -2.403949499130249,
      "step": 11512
    },
    {
      "epoch": 1.5065510754886726,
      "grad_norm": 10.051342964172363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11300681531429291,
      "logits/rejected": -0.1627819538116455,
      "logps/chosen": -292.46954345703125,
      "logps/rejected": -487.61224365234375,
      "loss": 0.6759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2437528371810913,
      "rewards/margins": 0.8893519639968872,
      "rewards/rejected": -2.1331048011779785,
      "step": 11513
    },
    {
      "epoch": 1.5066819334260244,
      "grad_norm": 6.878689289093018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4643709063529968,
      "logits/rejected": -0.3779054880142212,
      "logps/chosen": -395.5696716308594,
      "logps/rejected": -362.16180419921875,
      "loss": 0.3016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8158707022666931,
      "rewards/margins": 1.7406487464904785,
      "rewards/rejected": -2.5565195083618164,
      "step": 11514
    },
    {
      "epoch": 1.5068127913633762,
      "grad_norm": 6.693984031677246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17956802248954773,
      "logits/rejected": -0.3248101472854614,
      "logps/chosen": -430.1387939453125,
      "logps/rejected": -612.3598022460938,
      "loss": 0.3158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.864404320716858,
      "rewards/margins": 2.0426523685455322,
      "rewards/rejected": -3.9070568084716797,
      "step": 11515
    },
    {
      "epoch": 1.506943649300728,
      "grad_norm": 11.003409385681152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46740108728408813,
      "logits/rejected": -0.4320090413093567,
      "logps/chosen": -399.78179931640625,
      "logps/rejected": -416.6298828125,
      "loss": 0.5463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.0320122241973877,
      "rewards/margins": 0.8110191822052002,
      "rewards/rejected": -3.843031406402588,
      "step": 11516
    },
    {
      "epoch": 1.5070745072380798,
      "grad_norm": 3.112109661102295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43945199251174927,
      "logits/rejected": -0.28746938705444336,
      "logps/chosen": -392.5386047363281,
      "logps/rejected": -399.967529296875,
      "loss": 0.165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5912203192710876,
      "rewards/margins": 2.593947649002075,
      "rewards/rejected": -3.1851682662963867,
      "step": 11517
    },
    {
      "epoch": 1.5072053651754316,
      "grad_norm": 2.796912908554077,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4123936891555786,
      "logits/rejected": -0.4224484860897064,
      "logps/chosen": -550.3480834960938,
      "logps/rejected": -502.845947265625,
      "loss": 0.0974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27073538303375244,
      "rewards/margins": 3.8318843841552734,
      "rewards/rejected": -4.102619647979736,
      "step": 11518
    },
    {
      "epoch": 1.5073362231127831,
      "grad_norm": 5.688229084014893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21064504981040955,
      "logits/rejected": -0.24475637078285217,
      "logps/chosen": -464.74658203125,
      "logps/rejected": -407.6033020019531,
      "loss": 0.2615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2429993152618408,
      "rewards/margins": 1.8077328205108643,
      "rewards/rejected": -3.050731658935547,
      "step": 11519
    },
    {
      "epoch": 1.507467081050135,
      "grad_norm": 5.063912391662598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2567812204360962,
      "logits/rejected": -0.16854746639728546,
      "logps/chosen": -423.4007263183594,
      "logps/rejected": -327.0936584472656,
      "loss": 0.2587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.913179636001587,
      "rewards/margins": 1.5527249574661255,
      "rewards/rejected": -3.465904474258423,
      "step": 11520
    },
    {
      "epoch": 1.5075979389874867,
      "grad_norm": 9.863826751708984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13353362679481506,
      "logits/rejected": -0.38140466809272766,
      "logps/chosen": -679.9803466796875,
      "logps/rejected": -464.4771728515625,
      "loss": 0.5332,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8066015243530273,
      "rewards/margins": 0.8270214796066284,
      "rewards/rejected": -2.633622884750366,
      "step": 11521
    },
    {
      "epoch": 1.5077287969248385,
      "grad_norm": 9.569765090942383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.001031387597322464,
      "logits/rejected": -0.044063322246074677,
      "logps/chosen": -470.67645263671875,
      "logps/rejected": -501.810302734375,
      "loss": 0.4603,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1240489482879639,
      "rewards/margins": 1.1851112842559814,
      "rewards/rejected": -2.3091602325439453,
      "step": 11522
    },
    {
      "epoch": 1.50785965486219,
      "grad_norm": 6.589390277862549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33906447887420654,
      "logits/rejected": -0.4687436819076538,
      "logps/chosen": -377.30841064453125,
      "logps/rejected": -330.4200134277344,
      "loss": 0.3971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1759636402130127,
      "rewards/margins": 0.8208636045455933,
      "rewards/rejected": -2.9968268871307373,
      "step": 11523
    },
    {
      "epoch": 1.5079905127995419,
      "grad_norm": 7.558935642242432,
      "learning_rate": 5e-06,
      "logits/chosen": -0.565229058265686,
      "logits/rejected": -0.33521562814712524,
      "logps/chosen": -297.5075378417969,
      "logps/rejected": -550.2877807617188,
      "loss": 0.3359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4666800498962402,
      "rewards/margins": 1.8945635557174683,
      "rewards/rejected": -4.36124324798584,
      "step": 11524
    },
    {
      "epoch": 1.5081213707368937,
      "grad_norm": 5.495724201202393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23163685202598572,
      "logits/rejected": -0.5473998785018921,
      "logps/chosen": -517.6441040039062,
      "logps/rejected": -408.34637451171875,
      "loss": 0.2414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3395518064498901,
      "rewards/margins": 1.8838011026382446,
      "rewards/rejected": -3.2233529090881348,
      "step": 11525
    },
    {
      "epoch": 1.5082522286742455,
      "grad_norm": 9.852718353271484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22528883814811707,
      "logits/rejected": 0.13117331266403198,
      "logps/chosen": -493.5496520996094,
      "logps/rejected": -518.1979370117188,
      "loss": 0.4287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6753233671188354,
      "rewards/margins": 1.467406153678894,
      "rewards/rejected": -3.1427295207977295,
      "step": 11526
    },
    {
      "epoch": 1.5083830866115973,
      "grad_norm": 8.294559478759766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03916686773300171,
      "logits/rejected": -0.30031725764274597,
      "logps/chosen": -659.464599609375,
      "logps/rejected": -480.5662841796875,
      "loss": 0.3533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.455430507659912,
      "rewards/margins": 1.6667392253875732,
      "rewards/rejected": -4.1221699714660645,
      "step": 11527
    },
    {
      "epoch": 1.508513944548949,
      "grad_norm": 8.168275833129883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23729580640792847,
      "logits/rejected": -0.2635256350040436,
      "logps/chosen": -326.4250793457031,
      "logps/rejected": -308.6880798339844,
      "loss": 0.4382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.149902820587158,
      "rewards/margins": 1.0408291816711426,
      "rewards/rejected": -3.190732002258301,
      "step": 11528
    },
    {
      "epoch": 1.5086448024863008,
      "grad_norm": 7.282752990722656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07931704819202423,
      "logits/rejected": -0.2078683078289032,
      "logps/chosen": -395.4018859863281,
      "logps/rejected": -325.2878112792969,
      "loss": 0.3768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7107318639755249,
      "rewards/margins": 1.883954405784607,
      "rewards/rejected": -2.594686269760132,
      "step": 11529
    },
    {
      "epoch": 1.5087756604236526,
      "grad_norm": 2.6574974060058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3376949429512024,
      "logits/rejected": -0.4033671021461487,
      "logps/chosen": -510.50091552734375,
      "logps/rejected": -472.3594665527344,
      "loss": 0.0676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49898090958595276,
      "rewards/margins": 3.5800304412841797,
      "rewards/rejected": -4.0790114402771,
      "step": 11530
    },
    {
      "epoch": 1.5089065183610044,
      "grad_norm": 8.140792846679688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0893135666847229,
      "logits/rejected": -0.08139105141162872,
      "logps/chosen": -582.8656616210938,
      "logps/rejected": -408.2237548828125,
      "loss": 0.3402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9988752603530884,
      "rewards/margins": 1.7185431718826294,
      "rewards/rejected": -2.7174184322357178,
      "step": 11531
    },
    {
      "epoch": 1.5090373762983562,
      "grad_norm": 15.74174690246582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.504548966884613,
      "logits/rejected": -0.3058597147464752,
      "logps/chosen": -408.76190185546875,
      "logps/rejected": -498.64605712890625,
      "loss": 0.8539,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7320853471755981,
      "rewards/margins": 0.5393800139427185,
      "rewards/rejected": -2.271465301513672,
      "step": 11532
    },
    {
      "epoch": 1.509168234235708,
      "grad_norm": 13.119294166564941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29499274492263794,
      "logits/rejected": -0.48186343908309937,
      "logps/chosen": -441.9545593261719,
      "logps/rejected": -346.02069091796875,
      "loss": 0.5896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.7682652473449707,
      "rewards/margins": 0.7512383460998535,
      "rewards/rejected": -3.5195038318634033,
      "step": 11533
    },
    {
      "epoch": 1.5092990921730596,
      "grad_norm": 10.911718368530273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1338133066892624,
      "logits/rejected": -0.13420212268829346,
      "logps/chosen": -610.971923828125,
      "logps/rejected": -428.26239013671875,
      "loss": 0.57,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5182416439056396,
      "rewards/margins": 0.8533836603164673,
      "rewards/rejected": -2.3716251850128174,
      "step": 11534
    },
    {
      "epoch": 1.5094299501104114,
      "grad_norm": 8.410970687866211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44620516896247864,
      "logits/rejected": -0.3420032858848572,
      "logps/chosen": -361.2368469238281,
      "logps/rejected": -447.8865661621094,
      "loss": 0.3635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9864492416381836,
      "rewards/margins": 2.154684543609619,
      "rewards/rejected": -4.141134262084961,
      "step": 11535
    },
    {
      "epoch": 1.5095608080477632,
      "grad_norm": 12.947470664978027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024588692933321,
      "logits/rejected": 0.15138457715511322,
      "logps/chosen": -395.46868896484375,
      "logps/rejected": -343.7423095703125,
      "loss": 0.9565,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7209147214889526,
      "rewards/margins": 0.03785611689090729,
      "rewards/rejected": -1.7587708234786987,
      "step": 11536
    },
    {
      "epoch": 1.509691665985115,
      "grad_norm": 8.20694351196289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.227635458111763,
      "logits/rejected": -0.3955135941505432,
      "logps/chosen": -399.3139953613281,
      "logps/rejected": -378.2746276855469,
      "loss": 0.4821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3181251287460327,
      "rewards/margins": 0.7713310718536377,
      "rewards/rejected": -2.08945631980896,
      "step": 11537
    },
    {
      "epoch": 1.5098225239224665,
      "grad_norm": 5.360501289367676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21272879838943481,
      "logits/rejected": -0.5080737471580505,
      "logps/chosen": -500.69122314453125,
      "logps/rejected": -605.0031127929688,
      "loss": 0.1975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5667344927787781,
      "rewards/margins": 2.3003478050231934,
      "rewards/rejected": -2.867082357406616,
      "step": 11538
    },
    {
      "epoch": 1.5099533818598183,
      "grad_norm": 6.339574337005615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1830163598060608,
      "logits/rejected": -0.18318121135234833,
      "logps/chosen": -631.9810180664062,
      "logps/rejected": -584.2171630859375,
      "loss": 0.2778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.053209662437439,
      "rewards/margins": 2.0713677406311035,
      "rewards/rejected": -3.124577522277832,
      "step": 11539
    },
    {
      "epoch": 1.51008423979717,
      "grad_norm": 4.708370208740234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14229394495487213,
      "logits/rejected": -0.22380192577838898,
      "logps/chosen": -458.7774963378906,
      "logps/rejected": -438.3275451660156,
      "loss": 0.2332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.331733226776123,
      "rewards/margins": 1.6017284393310547,
      "rewards/rejected": -2.9334616661071777,
      "step": 11540
    },
    {
      "epoch": 1.510215097734522,
      "grad_norm": 6.079980373382568,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01661480963230133,
      "logits/rejected": 0.055697329342365265,
      "logps/chosen": -389.35205078125,
      "logps/rejected": -406.1666259765625,
      "loss": 0.2312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5400398969650269,
      "rewards/margins": 1.9221868515014648,
      "rewards/rejected": -3.4622271060943604,
      "step": 11541
    },
    {
      "epoch": 1.5103459556718737,
      "grad_norm": 17.20125961303711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25371092557907104,
      "logits/rejected": -0.18360289931297302,
      "logps/chosen": -492.8887023925781,
      "logps/rejected": -421.9970703125,
      "loss": 0.8959,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.472956657409668,
      "rewards/margins": 0.4612722098827362,
      "rewards/rejected": -2.9342288970947266,
      "step": 11542
    },
    {
      "epoch": 1.5104768136092255,
      "grad_norm": 8.193913459777832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18528050184249878,
      "logits/rejected": -0.272420197725296,
      "logps/chosen": -438.0616760253906,
      "logps/rejected": -482.3023986816406,
      "loss": 0.4541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6070542335510254,
      "rewards/margins": 0.7768263220787048,
      "rewards/rejected": -2.383880138397217,
      "step": 11543
    },
    {
      "epoch": 1.5106076715465773,
      "grad_norm": 4.099424839019775,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18480494618415833,
      "logits/rejected": 0.0226006880402565,
      "logps/chosen": -517.564697265625,
      "logps/rejected": -461.6722717285156,
      "loss": 0.1733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19641171395778656,
      "rewards/margins": 2.6059205532073975,
      "rewards/rejected": -2.8023321628570557,
      "step": 11544
    },
    {
      "epoch": 1.510738529483929,
      "grad_norm": 12.757022857666016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24954649806022644,
      "logits/rejected": -0.48715877532958984,
      "logps/chosen": -395.7186279296875,
      "logps/rejected": -413.0088195800781,
      "loss": 0.8131,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.653162956237793,
      "rewards/margins": 0.9969942569732666,
      "rewards/rejected": -3.6501569747924805,
      "step": 11545
    },
    {
      "epoch": 1.5108693874212809,
      "grad_norm": 9.834603309631348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04464579373598099,
      "logits/rejected": -0.25991836190223694,
      "logps/chosen": -548.8343505859375,
      "logps/rejected": -405.7674255371094,
      "loss": 0.3968,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2271621227264404,
      "rewards/margins": 1.52462637424469,
      "rewards/rejected": -2.75178861618042,
      "step": 11546
    },
    {
      "epoch": 1.5110002453586326,
      "grad_norm": 4.126076698303223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40576040744781494,
      "logits/rejected": -0.4766203463077545,
      "logps/chosen": -523.11279296875,
      "logps/rejected": -558.5819702148438,
      "loss": 0.1447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8246480226516724,
      "rewards/margins": 2.544602870941162,
      "rewards/rejected": -3.369251251220703,
      "step": 11547
    },
    {
      "epoch": 1.5111311032959844,
      "grad_norm": 8.845991134643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11078383773565292,
      "logits/rejected": -0.04339928179979324,
      "logps/chosen": -498.21185302734375,
      "logps/rejected": -452.6002502441406,
      "loss": 0.4575,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4434900283813477,
      "rewards/margins": 1.1399741172790527,
      "rewards/rejected": -2.5834639072418213,
      "step": 11548
    },
    {
      "epoch": 1.511261961233336,
      "grad_norm": 12.759713172912598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12390896677970886,
      "logits/rejected": -0.3077881634235382,
      "logps/chosen": -441.0368957519531,
      "logps/rejected": -428.6590270996094,
      "loss": 0.5677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.50635826587677,
      "rewards/margins": 1.7732856273651123,
      "rewards/rejected": -3.279644012451172,
      "step": 11549
    },
    {
      "epoch": 1.5113928191706878,
      "grad_norm": 6.473954677581787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22978676855564117,
      "logits/rejected": -0.32245194911956787,
      "logps/chosen": -363.4620361328125,
      "logps/rejected": -416.5837097167969,
      "loss": 0.2894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4442915916442871,
      "rewards/margins": 1.8949304819107056,
      "rewards/rejected": -2.339221954345703,
      "step": 11550
    },
    {
      "epoch": 1.5115236771080396,
      "grad_norm": 9.404784202575684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3590376377105713,
      "logits/rejected": -0.19206394255161285,
      "logps/chosen": -357.7938537597656,
      "logps/rejected": -345.0734558105469,
      "loss": 0.5551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1876049041748047,
      "rewards/margins": 0.7316864728927612,
      "rewards/rejected": -2.9192914962768555,
      "step": 11551
    },
    {
      "epoch": 1.5116545350453914,
      "grad_norm": 10.168729782104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0463697612285614,
      "logits/rejected": -0.09720001369714737,
      "logps/chosen": -338.63262939453125,
      "logps/rejected": -431.03302001953125,
      "loss": 0.7021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2867151498794556,
      "rewards/margins": 0.5007431507110596,
      "rewards/rejected": -1.7874584197998047,
      "step": 11552
    },
    {
      "epoch": 1.511785392982743,
      "grad_norm": 3.815091133117676,
      "learning_rate": 5e-06,
      "logits/chosen": 0.268798291683197,
      "logits/rejected": 0.4086940884590149,
      "logps/chosen": -526.9990234375,
      "logps/rejected": -426.1440124511719,
      "loss": 0.1442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.87357097864151,
      "rewards/margins": 2.4451122283935547,
      "rewards/rejected": -3.318683385848999,
      "step": 11553
    },
    {
      "epoch": 1.5119162509200947,
      "grad_norm": 8.873344421386719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16728626191616058,
      "logits/rejected": -0.3478756844997406,
      "logps/chosen": -524.4222412109375,
      "logps/rejected": -395.76751708984375,
      "loss": 0.5031,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.104670763015747,
      "rewards/margins": 0.853385865688324,
      "rewards/rejected": -1.9580563306808472,
      "step": 11554
    },
    {
      "epoch": 1.5120471088574465,
      "grad_norm": 2.8839290142059326,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10041990131139755,
      "logits/rejected": -0.2330743819475174,
      "logps/chosen": -573.8847045898438,
      "logps/rejected": -575.5707397460938,
      "loss": 0.1067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7355446815490723,
      "rewards/margins": 2.6778392791748047,
      "rewards/rejected": -4.413383483886719,
      "step": 11555
    },
    {
      "epoch": 1.5121779667947983,
      "grad_norm": 6.8172078132629395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12588219344615936,
      "logits/rejected": -0.04062424227595329,
      "logps/chosen": -445.2228088378906,
      "logps/rejected": -383.99481201171875,
      "loss": 0.2974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.841657817363739,
      "rewards/margins": 1.595531702041626,
      "rewards/rejected": -2.4371895790100098,
      "step": 11556
    },
    {
      "epoch": 1.5123088247321501,
      "grad_norm": 9.8660888671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25275740027427673,
      "logits/rejected": -0.3083377182483673,
      "logps/chosen": -435.265380859375,
      "logps/rejected": -472.6634216308594,
      "loss": 0.4067,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.340348243713379,
      "rewards/margins": 1.1247445344924927,
      "rewards/rejected": -2.465092658996582,
      "step": 11557
    },
    {
      "epoch": 1.512439682669502,
      "grad_norm": 9.220945358276367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10637491196393967,
      "logits/rejected": -0.10131864249706268,
      "logps/chosen": -504.61328125,
      "logps/rejected": -424.6081237792969,
      "loss": 0.6339,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.427730679512024,
      "rewards/margins": 0.5666660070419312,
      "rewards/rejected": -1.9943965673446655,
      "step": 11558
    },
    {
      "epoch": 1.5125705406068537,
      "grad_norm": 5.594213485717773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27093541622161865,
      "logits/rejected": -0.24283333122730255,
      "logps/chosen": -478.9502868652344,
      "logps/rejected": -623.5232543945312,
      "loss": 0.325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2265795469284058,
      "rewards/margins": 1.7807998657226562,
      "rewards/rejected": -3.0073792934417725,
      "step": 11559
    },
    {
      "epoch": 1.5127013985442055,
      "grad_norm": 7.211307048797607,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07259821891784668,
      "logits/rejected": 0.10302411764860153,
      "logps/chosen": -367.24481201171875,
      "logps/rejected": -443.5723571777344,
      "loss": 0.3316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7070278525352478,
      "rewards/margins": 1.4026196002960205,
      "rewards/rejected": -2.109647274017334,
      "step": 11560
    },
    {
      "epoch": 1.5128322564815573,
      "grad_norm": 4.79427433013916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0793842151761055,
      "logits/rejected": -0.1576876938343048,
      "logps/chosen": -424.9028015136719,
      "logps/rejected": -398.7962646484375,
      "loss": 0.3076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.066412329673767,
      "rewards/margins": 1.5607104301452637,
      "rewards/rejected": -2.6271228790283203,
      "step": 11561
    },
    {
      "epoch": 1.512963114418909,
      "grad_norm": 3.6033132076263428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03465419262647629,
      "logits/rejected": -0.0020153336226940155,
      "logps/chosen": -406.6921081542969,
      "logps/rejected": -447.6513671875,
      "loss": 0.1426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8439067602157593,
      "rewards/margins": 2.583040237426758,
      "rewards/rejected": -3.4269473552703857,
      "step": 11562
    },
    {
      "epoch": 1.5130939723562609,
      "grad_norm": 7.141214370727539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05469899624586105,
      "logits/rejected": -0.06402213871479034,
      "logps/chosen": -371.29083251953125,
      "logps/rejected": -343.01495361328125,
      "loss": 0.3631,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1485954225063324,
      "rewards/margins": 1.1417415142059326,
      "rewards/rejected": -1.2903369665145874,
      "step": 11563
    },
    {
      "epoch": 1.5132248302936127,
      "grad_norm": 14.939275741577148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29359036684036255,
      "logits/rejected": -0.3792739510536194,
      "logps/chosen": -584.6924438476562,
      "logps/rejected": -480.7255554199219,
      "loss": 0.4287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9360207319259644,
      "rewards/margins": 1.6088411808013916,
      "rewards/rejected": -2.5448617935180664,
      "step": 11564
    },
    {
      "epoch": 1.5133556882309642,
      "grad_norm": 7.927530288696289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1277642399072647,
      "logits/rejected": -0.015644267201423645,
      "logps/chosen": -460.8179931640625,
      "logps/rejected": -473.5428161621094,
      "loss": 0.3747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.824798822402954,
      "rewards/margins": 1.3143038749694824,
      "rewards/rejected": -3.1391026973724365,
      "step": 11565
    },
    {
      "epoch": 1.513486546168316,
      "grad_norm": 13.603403091430664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03181782364845276,
      "logits/rejected": -0.13213765621185303,
      "logps/chosen": -458.3779602050781,
      "logps/rejected": -498.27642822265625,
      "loss": 0.6322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6772555112838745,
      "rewards/margins": 1.4457006454467773,
      "rewards/rejected": -3.1229560375213623,
      "step": 11566
    },
    {
      "epoch": 1.5136174041056678,
      "grad_norm": 11.760107040405273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2550591826438904,
      "logits/rejected": -0.2767922282218933,
      "logps/chosen": -503.38818359375,
      "logps/rejected": -560.3845825195312,
      "loss": 0.6002,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.284606695175171,
      "rewards/margins": 0.8438999056816101,
      "rewards/rejected": -2.1285064220428467,
      "step": 11567
    },
    {
      "epoch": 1.5137482620430196,
      "grad_norm": 7.008642673492432,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22580446302890778,
      "logits/rejected": -0.4025143086910248,
      "logps/chosen": -502.0853271484375,
      "logps/rejected": -578.12890625,
      "loss": 0.359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4561914205551147,
      "rewards/margins": 2.4023547172546387,
      "rewards/rejected": -3.858546257019043,
      "step": 11568
    },
    {
      "epoch": 1.5138791199803712,
      "grad_norm": 10.839888572692871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17288216948509216,
      "logits/rejected": -0.07501257956027985,
      "logps/chosen": -544.9290771484375,
      "logps/rejected": -422.6341552734375,
      "loss": 0.3874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7762847542762756,
      "rewards/margins": 1.8197777271270752,
      "rewards/rejected": -2.596062421798706,
      "step": 11569
    },
    {
      "epoch": 1.514009977917723,
      "grad_norm": 5.137597560882568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32737475633621216,
      "logits/rejected": -0.20901921391487122,
      "logps/chosen": -407.6257629394531,
      "logps/rejected": -376.40313720703125,
      "loss": 0.3216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6085781455039978,
      "rewards/margins": 1.6098597049713135,
      "rewards/rejected": -2.218438148498535,
      "step": 11570
    },
    {
      "epoch": 1.5141408358550748,
      "grad_norm": 4.662816047668457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12942500412464142,
      "logits/rejected": -0.061210714280605316,
      "logps/chosen": -724.6126708984375,
      "logps/rejected": -654.4057006835938,
      "loss": 0.2018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9512336850166321,
      "rewards/margins": 2.3597006797790527,
      "rewards/rejected": -3.310934543609619,
      "step": 11571
    },
    {
      "epoch": 1.5142716937924265,
      "grad_norm": 17.260833740234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2696971595287323,
      "logits/rejected": -0.4632572531700134,
      "logps/chosen": -459.087890625,
      "logps/rejected": -388.7892150878906,
      "loss": 1.3118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1311464309692383,
      "rewards/margins": 0.39959028363227844,
      "rewards/rejected": -2.5307364463806152,
      "step": 11572
    },
    {
      "epoch": 1.5144025517297783,
      "grad_norm": 8.268881797790527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07581306248903275,
      "logits/rejected": -0.1647450029850006,
      "logps/chosen": -340.5205383300781,
      "logps/rejected": -402.2974853515625,
      "loss": 0.336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6548587679862976,
      "rewards/margins": 2.6021029949188232,
      "rewards/rejected": -3.2569615840911865,
      "step": 11573
    },
    {
      "epoch": 1.5145334096671301,
      "grad_norm": 3.3549933433532715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4045071005821228,
      "logits/rejected": -0.5769897699356079,
      "logps/chosen": -488.4924621582031,
      "logps/rejected": -469.0589294433594,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6758935451507568,
      "rewards/margins": 3.154305934906006,
      "rewards/rejected": -3.830199718475342,
      "step": 11574
    },
    {
      "epoch": 1.514664267604482,
      "grad_norm": 11.814156532287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3683242201805115,
      "logits/rejected": -0.37147238850593567,
      "logps/chosen": -452.8259582519531,
      "logps/rejected": -510.5912780761719,
      "loss": 0.649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.104133129119873,
      "rewards/margins": 0.7637622356414795,
      "rewards/rejected": -1.867895483970642,
      "step": 11575
    },
    {
      "epoch": 1.5147951255418337,
      "grad_norm": 10.805167198181152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24915069341659546,
      "logits/rejected": -0.27483779191970825,
      "logps/chosen": -292.804443359375,
      "logps/rejected": -375.90460205078125,
      "loss": 0.5007,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0930311679840088,
      "rewards/margins": 1.3692330121994019,
      "rewards/rejected": -2.462264060974121,
      "step": 11576
    },
    {
      "epoch": 1.5149259834791855,
      "grad_norm": 9.957842826843262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3072042167186737,
      "logits/rejected": -0.34591400623321533,
      "logps/chosen": -463.0655517578125,
      "logps/rejected": -508.2350158691406,
      "loss": 0.6567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7014894485473633,
      "rewards/margins": 1.1889679431915283,
      "rewards/rejected": -1.8904573917388916,
      "step": 11577
    },
    {
      "epoch": 1.5150568414165373,
      "grad_norm": 10.883834838867188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3253377377986908,
      "logits/rejected": -0.12367507070302963,
      "logps/chosen": -357.4618835449219,
      "logps/rejected": -560.2349853515625,
      "loss": 0.8218,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1104261875152588,
      "rewards/margins": 0.9102510213851929,
      "rewards/rejected": -2.020677328109741,
      "step": 11578
    },
    {
      "epoch": 1.515187699353889,
      "grad_norm": 12.703873634338379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14991846680641174,
      "logits/rejected": -0.1168687641620636,
      "logps/chosen": -595.9620971679688,
      "logps/rejected": -530.576171875,
      "loss": 0.7865,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5896637439727783,
      "rewards/margins": 0.5928890109062195,
      "rewards/rejected": -2.1825525760650635,
      "step": 11579
    },
    {
      "epoch": 1.5153185572912407,
      "grad_norm": 7.2736053466796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10920418798923492,
      "logits/rejected": -0.11086256802082062,
      "logps/chosen": -612.770263671875,
      "logps/rejected": -442.1575927734375,
      "loss": 0.3141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2136988639831543,
      "rewards/margins": 1.3654052019119263,
      "rewards/rejected": -3.57910418510437,
      "step": 11580
    },
    {
      "epoch": 1.5154494152285924,
      "grad_norm": 6.936080455780029,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01608806475996971,
      "logits/rejected": -0.14324511587619781,
      "logps/chosen": -354.8138122558594,
      "logps/rejected": -530.3594360351562,
      "loss": 0.3943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6949374675750732,
      "rewards/margins": 1.6747558116912842,
      "rewards/rejected": -3.3696932792663574,
      "step": 11581
    },
    {
      "epoch": 1.5155802731659442,
      "grad_norm": 4.660006999969482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1847352683544159,
      "logits/rejected": -0.25188812613487244,
      "logps/chosen": -485.0091552734375,
      "logps/rejected": -420.17919921875,
      "loss": 0.298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9978108406066895,
      "rewards/margins": 1.3076571226119995,
      "rewards/rejected": -2.3054678440093994,
      "step": 11582
    },
    {
      "epoch": 1.515711131103296,
      "grad_norm": 2.4371585845947266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3198890686035156,
      "logits/rejected": -0.23671625554561615,
      "logps/chosen": -580.565673828125,
      "logps/rejected": -441.1921081542969,
      "loss": 0.1031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6612125635147095,
      "rewards/margins": 2.4695122241973877,
      "rewards/rejected": -3.1307246685028076,
      "step": 11583
    },
    {
      "epoch": 1.5158419890406476,
      "grad_norm": 8.315898895263672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21717654168605804,
      "logits/rejected": -0.3474079668521881,
      "logps/chosen": -403.2279357910156,
      "logps/rejected": -328.39483642578125,
      "loss": 0.4304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1635122299194336,
      "rewards/margins": 0.8865640759468079,
      "rewards/rejected": -2.050076484680176,
      "step": 11584
    },
    {
      "epoch": 1.5159728469779994,
      "grad_norm": 7.999938488006592,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07309877127408981,
      "logits/rejected": 0.15648004412651062,
      "logps/chosen": -393.19927978515625,
      "logps/rejected": -408.9424133300781,
      "loss": 0.4419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6369249820709229,
      "rewards/margins": 1.512593388557434,
      "rewards/rejected": -3.1495184898376465,
      "step": 11585
    },
    {
      "epoch": 1.5161037049153512,
      "grad_norm": 5.5850958824157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31510576605796814,
      "logits/rejected": -0.25792214274406433,
      "logps/chosen": -389.2647705078125,
      "logps/rejected": -646.6985473632812,
      "loss": 0.1969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.091975450515747,
      "rewards/margins": 2.1431171894073486,
      "rewards/rejected": -3.2350924015045166,
      "step": 11586
    },
    {
      "epoch": 1.516234562852703,
      "grad_norm": 5.059214115142822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16363847255706787,
      "logits/rejected": -0.2236836850643158,
      "logps/chosen": -327.63818359375,
      "logps/rejected": -285.54010009765625,
      "loss": 0.3915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2158074378967285,
      "rewards/margins": 1.4175134897232056,
      "rewards/rejected": -2.6333205699920654,
      "step": 11587
    },
    {
      "epoch": 1.5163654207900548,
      "grad_norm": 6.853480339050293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2265864759683609,
      "logits/rejected": -0.3241347074508667,
      "logps/chosen": -465.7779541015625,
      "logps/rejected": -450.425048828125,
      "loss": 0.3052,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4300885200500488,
      "rewards/margins": 1.505574107170105,
      "rewards/rejected": -2.935662269592285,
      "step": 11588
    },
    {
      "epoch": 1.5164962787274066,
      "grad_norm": 7.908463478088379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2346535325050354,
      "logits/rejected": -0.19151727855205536,
      "logps/chosen": -314.807373046875,
      "logps/rejected": -533.5631713867188,
      "loss": 0.2737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1658999919891357,
      "rewards/margins": 1.9988229274749756,
      "rewards/rejected": -3.1647233963012695,
      "step": 11589
    },
    {
      "epoch": 1.5166271366647583,
      "grad_norm": 11.807780265808105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4655974209308624,
      "logits/rejected": -0.4305911362171173,
      "logps/chosen": -416.4886779785156,
      "logps/rejected": -646.8051147460938,
      "loss": 0.6785,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2774362564086914,
      "rewards/margins": 1.61020827293396,
      "rewards/rejected": -3.8876442909240723,
      "step": 11590
    },
    {
      "epoch": 1.5167579946021101,
      "grad_norm": 12.840363502502441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49627336859703064,
      "logits/rejected": -0.40894824266433716,
      "logps/chosen": -444.5448913574219,
      "logps/rejected": -425.8049011230469,
      "loss": 0.6794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.973048210144043,
      "rewards/margins": 0.9336813688278198,
      "rewards/rejected": -2.9067294597625732,
      "step": 11591
    },
    {
      "epoch": 1.516888852539462,
      "grad_norm": 9.2308931350708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37181633710861206,
      "logits/rejected": -0.21728795766830444,
      "logps/chosen": -273.608154296875,
      "logps/rejected": -293.8839416503906,
      "loss": 0.6369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.618026614189148,
      "rewards/margins": 0.4074057340621948,
      "rewards/rejected": -2.0254323482513428,
      "step": 11592
    },
    {
      "epoch": 1.5170197104768137,
      "grad_norm": 24.646841049194336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10472951084375381,
      "logits/rejected": -0.026049695909023285,
      "logps/chosen": -591.5075073242188,
      "logps/rejected": -522.853515625,
      "loss": 0.5015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.037738800048828,
      "rewards/margins": 2.094482421875,
      "rewards/rejected": -4.132221221923828,
      "step": 11593
    },
    {
      "epoch": 1.5171505684141655,
      "grad_norm": 5.020682334899902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2560749650001526,
      "logits/rejected": -0.46245574951171875,
      "logps/chosen": -418.6518249511719,
      "logps/rejected": -691.3853759765625,
      "loss": 0.1716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1037920713424683,
      "rewards/margins": 2.3060083389282227,
      "rewards/rejected": -3.4098005294799805,
      "step": 11594
    },
    {
      "epoch": 1.517281426351517,
      "grad_norm": 8.410591125488281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3559878468513489,
      "logits/rejected": -0.3338892459869385,
      "logps/chosen": -488.3014221191406,
      "logps/rejected": -454.09967041015625,
      "loss": 0.3265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7312343120574951,
      "rewards/margins": 1.6451449394226074,
      "rewards/rejected": -3.3763792514801025,
      "step": 11595
    },
    {
      "epoch": 1.5174122842888689,
      "grad_norm": 8.553176879882812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07308606058359146,
      "logits/rejected": -0.1489010453224182,
      "logps/chosen": -510.6564636230469,
      "logps/rejected": -444.5498962402344,
      "loss": 0.3688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9895985126495361,
      "rewards/margins": 0.9684041738510132,
      "rewards/rejected": -1.9580026865005493,
      "step": 11596
    },
    {
      "epoch": 1.5175431422262207,
      "grad_norm": 6.973296165466309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3251081109046936,
      "logits/rejected": -0.4336954951286316,
      "logps/chosen": -407.4450378417969,
      "logps/rejected": -549.2908935546875,
      "loss": 0.4023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6771849393844604,
      "rewards/margins": 1.551365852355957,
      "rewards/rejected": -3.228550672531128,
      "step": 11597
    },
    {
      "epoch": 1.5176740001635725,
      "grad_norm": 13.583393096923828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2199992686510086,
      "logits/rejected": -0.226210355758667,
      "logps/chosen": -415.31280517578125,
      "logps/rejected": -270.4432678222656,
      "loss": 0.8884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3802785873413086,
      "rewards/margins": 0.19596827030181885,
      "rewards/rejected": -1.5762468576431274,
      "step": 11598
    },
    {
      "epoch": 1.517804858100924,
      "grad_norm": 8.025522232055664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10840576887130737,
      "logits/rejected": -0.14256668090820312,
      "logps/chosen": -332.6073303222656,
      "logps/rejected": -412.4351806640625,
      "loss": 0.2842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.335842251777649,
      "rewards/margins": 1.4816838502883911,
      "rewards/rejected": -2.81752610206604,
      "step": 11599
    },
    {
      "epoch": 1.5179357160382758,
      "grad_norm": 9.65321159362793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1467939168214798,
      "logits/rejected": -0.044138066470623016,
      "logps/chosen": -591.765869140625,
      "logps/rejected": -426.5936279296875,
      "loss": 0.4916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49227064847946167,
      "rewards/margins": 1.4096450805664062,
      "rewards/rejected": -1.9019157886505127,
      "step": 11600
    },
    {
      "epoch": 1.5180665739756276,
      "grad_norm": 7.378326892852783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13691726326942444,
      "logits/rejected": 0.17103801667690277,
      "logps/chosen": -331.7367248535156,
      "logps/rejected": -315.0265808105469,
      "loss": 0.4625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1308494806289673,
      "rewards/margins": 1.0701009035110474,
      "rewards/rejected": -2.2009503841400146,
      "step": 11601
    },
    {
      "epoch": 1.5181974319129794,
      "grad_norm": 2.746203660964966,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1856173574924469,
      "logits/rejected": -0.4127696454524994,
      "logps/chosen": -327.3147277832031,
      "logps/rejected": -454.5227355957031,
      "loss": 0.1781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8478269577026367,
      "rewards/margins": 2.5048251152038574,
      "rewards/rejected": -3.352651834487915,
      "step": 11602
    },
    {
      "epoch": 1.5183282898503312,
      "grad_norm": 7.021607398986816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1666942983865738,
      "logits/rejected": -0.4276014566421509,
      "logps/chosen": -443.6376647949219,
      "logps/rejected": -522.9775390625,
      "loss": 0.2815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7837644815444946,
      "rewards/margins": 1.8541216850280762,
      "rewards/rejected": -2.637885808944702,
      "step": 11603
    },
    {
      "epoch": 1.518459147787683,
      "grad_norm": 10.57255744934082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24954012036323547,
      "logits/rejected": -0.4189019203186035,
      "logps/chosen": -265.29010009765625,
      "logps/rejected": -251.20742797851562,
      "loss": 0.7843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7741057872772217,
      "rewards/margins": 0.1713065803050995,
      "rewards/rejected": -1.9454121589660645,
      "step": 11604
    },
    {
      "epoch": 1.5185900057250348,
      "grad_norm": 7.118106842041016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007207863032817841,
      "logits/rejected": -0.23458991944789886,
      "logps/chosen": -410.142822265625,
      "logps/rejected": -449.1396179199219,
      "loss": 0.3216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3221043348312378,
      "rewards/margins": 1.637700080871582,
      "rewards/rejected": -2.9598042964935303,
      "step": 11605
    },
    {
      "epoch": 1.5187208636623866,
      "grad_norm": 10.193068504333496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14195390045642853,
      "logits/rejected": -0.0557764396071434,
      "logps/chosen": -385.35272216796875,
      "logps/rejected": -518.986572265625,
      "loss": 0.5908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4305955171585083,
      "rewards/margins": 1.4022636413574219,
      "rewards/rejected": -2.8328592777252197,
      "step": 11606
    },
    {
      "epoch": 1.5188517215997384,
      "grad_norm": 8.675373077392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02770763635635376,
      "logits/rejected": -0.3218516409397125,
      "logps/chosen": -645.5957641601562,
      "logps/rejected": -516.8419189453125,
      "loss": 0.3992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0455783605575562,
      "rewards/margins": 0.9251392483711243,
      "rewards/rejected": -1.9707176685333252,
      "step": 11607
    },
    {
      "epoch": 1.5189825795370901,
      "grad_norm": 14.245865821838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2305467426776886,
      "logits/rejected": -0.21233582496643066,
      "logps/chosen": -608.0390014648438,
      "logps/rejected": -614.8888549804688,
      "loss": 0.835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0139329433441162,
      "rewards/margins": 0.7946971654891968,
      "rewards/rejected": -1.8086299896240234,
      "step": 11608
    },
    {
      "epoch": 1.519113437474442,
      "grad_norm": 14.627765655517578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11927363276481628,
      "logits/rejected": -0.3520244359970093,
      "logps/chosen": -406.86181640625,
      "logps/rejected": -459.9793701171875,
      "loss": 0.8061,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.391739845275879,
      "rewards/margins": 0.31417086720466614,
      "rewards/rejected": -1.7059106826782227,
      "step": 11609
    },
    {
      "epoch": 1.5192442954117937,
      "grad_norm": 9.535314559936523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2899145185947418,
      "logits/rejected": -0.12365863472223282,
      "logps/chosen": -394.48443603515625,
      "logps/rejected": -423.134765625,
      "loss": 0.4948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0865628719329834,
      "rewards/margins": 0.853302001953125,
      "rewards/rejected": -1.9398648738861084,
      "step": 11610
    },
    {
      "epoch": 1.5193751533491453,
      "grad_norm": 10.696799278259277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0908561572432518,
      "logits/rejected": -0.2830592691898346,
      "logps/chosen": -385.4300842285156,
      "logps/rejected": -403.3178405761719,
      "loss": 0.6514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2423734664916992,
      "rewards/margins": 1.651465892791748,
      "rewards/rejected": -2.8938393592834473,
      "step": 11611
    },
    {
      "epoch": 1.519506011286497,
      "grad_norm": 8.548991203308105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2616065442562103,
      "logits/rejected": -0.20130199193954468,
      "logps/chosen": -520.9073486328125,
      "logps/rejected": -524.3125610351562,
      "loss": 0.3393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2380131483078003,
      "rewards/margins": 2.341453790664673,
      "rewards/rejected": -3.579467296600342,
      "step": 11612
    },
    {
      "epoch": 1.5196368692238489,
      "grad_norm": 4.559396266937256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.187652125954628,
      "logits/rejected": -0.246965691447258,
      "logps/chosen": -436.350830078125,
      "logps/rejected": -469.3136291503906,
      "loss": 0.252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3346933126449585,
      "rewards/margins": 1.858681559562683,
      "rewards/rejected": -3.1933748722076416,
      "step": 11613
    },
    {
      "epoch": 1.5197677271612007,
      "grad_norm": 13.383589744567871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41015639901161194,
      "logits/rejected": -0.4062160849571228,
      "logps/chosen": -565.3029174804688,
      "logps/rejected": -479.56329345703125,
      "loss": 0.614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2757583856582642,
      "rewards/margins": 1.2340943813323975,
      "rewards/rejected": -2.509852886199951,
      "step": 11614
    },
    {
      "epoch": 1.5198985850985522,
      "grad_norm": 12.731146812438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12353929132223129,
      "logits/rejected": -0.004522811621427536,
      "logps/chosen": -616.198486328125,
      "logps/rejected": -590.7637939453125,
      "loss": 0.504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3910036087036133,
      "rewards/margins": 1.0717939138412476,
      "rewards/rejected": -2.4627974033355713,
      "step": 11615
    },
    {
      "epoch": 1.520029443035904,
      "grad_norm": 5.457464218139648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07748299837112427,
      "logits/rejected": -0.23712575435638428,
      "logps/chosen": -404.986328125,
      "logps/rejected": -398.6688232421875,
      "loss": 0.2203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.749333381652832,
      "rewards/margins": 2.4693498611450195,
      "rewards/rejected": -4.218683242797852,
      "step": 11616
    },
    {
      "epoch": 1.5201603009732558,
      "grad_norm": 4.409395217895508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5172581672668457,
      "logits/rejected": -0.6178304553031921,
      "logps/chosen": -395.053955078125,
      "logps/rejected": -373.63995361328125,
      "loss": 0.3383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0629302263259888,
      "rewards/margins": 1.4539060592651367,
      "rewards/rejected": -2.516836166381836,
      "step": 11617
    },
    {
      "epoch": 1.5202911589106076,
      "grad_norm": 3.945187568664551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20406314730644226,
      "logits/rejected": -0.2589252293109894,
      "logps/chosen": -436.71075439453125,
      "logps/rejected": -492.964599609375,
      "loss": 0.1377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4536135196685791,
      "rewards/margins": 2.58715558052063,
      "rewards/rejected": -3.040769100189209,
      "step": 11618
    },
    {
      "epoch": 1.5204220168479594,
      "grad_norm": 14.546304702758789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09618324041366577,
      "logits/rejected": -0.2882595360279083,
      "logps/chosen": -410.0332946777344,
      "logps/rejected": -563.5987548828125,
      "loss": 0.874,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4249998331069946,
      "rewards/margins": 0.8185802698135376,
      "rewards/rejected": -2.2435801029205322,
      "step": 11619
    },
    {
      "epoch": 1.5205528747853112,
      "grad_norm": 10.41886043548584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015590665861964226,
      "logits/rejected": -0.06975159049034119,
      "logps/chosen": -556.0169067382812,
      "logps/rejected": -543.8190307617188,
      "loss": 0.5969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.458278775215149,
      "rewards/margins": 0.894195556640625,
      "rewards/rejected": -2.3524744510650635,
      "step": 11620
    },
    {
      "epoch": 1.520683732722663,
      "grad_norm": 9.47163200378418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028094682842493057,
      "logits/rejected": -0.29881468415260315,
      "logps/chosen": -505.12945556640625,
      "logps/rejected": -405.08734130859375,
      "loss": 0.3895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.025521993637085,
      "rewards/margins": 1.279039978981018,
      "rewards/rejected": -3.3045618534088135,
      "step": 11621
    },
    {
      "epoch": 1.5208145906600148,
      "grad_norm": 8.593862533569336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23896515369415283,
      "logits/rejected": -0.26346829533576965,
      "logps/chosen": -408.6180419921875,
      "logps/rejected": -360.83831787109375,
      "loss": 0.5013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1710478067398071,
      "rewards/margins": 0.7962599992752075,
      "rewards/rejected": -1.9673078060150146,
      "step": 11622
    },
    {
      "epoch": 1.5209454485973666,
      "grad_norm": 6.452659606933594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08167319744825363,
      "logits/rejected": -0.1303611695766449,
      "logps/chosen": -622.5157470703125,
      "logps/rejected": -428.93115234375,
      "loss": 0.2418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0884803533554077,
      "rewards/margins": 1.5785831212997437,
      "rewards/rejected": -2.6670634746551514,
      "step": 11623
    },
    {
      "epoch": 1.5210763065347184,
      "grad_norm": 7.724681377410889,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01775318756699562,
      "logits/rejected": 0.12084060907363892,
      "logps/chosen": -448.5091552734375,
      "logps/rejected": -514.6036987304688,
      "loss": 0.3567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4589937925338745,
      "rewards/margins": 1.3199210166931152,
      "rewards/rejected": -2.7789149284362793,
      "step": 11624
    },
    {
      "epoch": 1.5212071644720702,
      "grad_norm": 8.557339668273926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1814776360988617,
      "logits/rejected": -0.1653350591659546,
      "logps/chosen": -453.165283203125,
      "logps/rejected": -483.31927490234375,
      "loss": 0.4528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7455894351005554,
      "rewards/margins": 1.086424708366394,
      "rewards/rejected": -1.8320140838623047,
      "step": 11625
    },
    {
      "epoch": 1.5213380224094217,
      "grad_norm": 4.050281524658203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.213217094540596,
      "logits/rejected": 0.010743186809122562,
      "logps/chosen": -553.6129760742188,
      "logps/rejected": -477.5369873046875,
      "loss": 0.1131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.028715789318084717,
      "rewards/margins": 3.0973398685455322,
      "rewards/rejected": -3.1260554790496826,
      "step": 11626
    },
    {
      "epoch": 1.5214688803467735,
      "grad_norm": 8.045454025268555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022496569901704788,
      "logits/rejected": -0.1309192031621933,
      "logps/chosen": -359.1578674316406,
      "logps/rejected": -324.5547790527344,
      "loss": 0.421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6019524335861206,
      "rewards/margins": 2.358945846557617,
      "rewards/rejected": -2.9608983993530273,
      "step": 11627
    },
    {
      "epoch": 1.5215997382841253,
      "grad_norm": 7.515609264373779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4011279046535492,
      "logits/rejected": -0.2803153395652771,
      "logps/chosen": -380.96051025390625,
      "logps/rejected": -354.0189514160156,
      "loss": 0.4228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2094109058380127,
      "rewards/margins": 1.1797113418579102,
      "rewards/rejected": -2.3891220092773438,
      "step": 11628
    },
    {
      "epoch": 1.521730596221477,
      "grad_norm": 5.678808689117432,
      "learning_rate": 5e-06,
      "logits/chosen": -0.050252459943294525,
      "logits/rejected": -0.04654398560523987,
      "logps/chosen": -582.951171875,
      "logps/rejected": -562.5374755859375,
      "loss": 0.2084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7463588118553162,
      "rewards/margins": 2.0819008350372314,
      "rewards/rejected": -2.8282599449157715,
      "step": 11629
    },
    {
      "epoch": 1.5218614541588287,
      "grad_norm": 11.379485130310059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2661546468734741,
      "logits/rejected": -0.36747124791145325,
      "logps/chosen": -624.8558349609375,
      "logps/rejected": -552.3350219726562,
      "loss": 0.6169,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.92584228515625,
      "rewards/margins": 1.2203397750854492,
      "rewards/rejected": -2.146182060241699,
      "step": 11630
    },
    {
      "epoch": 1.5219923120961805,
      "grad_norm": 11.955865859985352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2838805913925171,
      "logits/rejected": -0.03474077582359314,
      "logps/chosen": -421.33026123046875,
      "logps/rejected": -574.1820068359375,
      "loss": 0.5712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8136990070343018,
      "rewards/margins": 1.133748173713684,
      "rewards/rejected": -1.9474470615386963,
      "step": 11631
    },
    {
      "epoch": 1.5221231700335323,
      "grad_norm": 8.951156616210938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2907150089740753,
      "logits/rejected": -0.29021161794662476,
      "logps/chosen": -403.28826904296875,
      "logps/rejected": -415.5268249511719,
      "loss": 0.421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0873308181762695,
      "rewards/margins": 0.9511907696723938,
      "rewards/rejected": -2.0385217666625977,
      "step": 11632
    },
    {
      "epoch": 1.522254027970884,
      "grad_norm": 4.8474345207214355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09098672866821289,
      "logits/rejected": 0.1135527640581131,
      "logps/chosen": -532.50146484375,
      "logps/rejected": -423.38421630859375,
      "loss": 0.1754,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6357101798057556,
      "rewards/margins": 2.9916446208953857,
      "rewards/rejected": -3.627354621887207,
      "step": 11633
    },
    {
      "epoch": 1.5223848859082358,
      "grad_norm": 6.346500396728516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25712987780570984,
      "logits/rejected": -0.3091203272342682,
      "logps/chosen": -379.6253662109375,
      "logps/rejected": -417.0770568847656,
      "loss": 0.3698,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7132904529571533,
      "rewards/margins": 1.559093713760376,
      "rewards/rejected": -3.2723841667175293,
      "step": 11634
    },
    {
      "epoch": 1.5225157438455876,
      "grad_norm": 7.628127098083496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.213893324136734,
      "logits/rejected": -0.11807574331760406,
      "logps/chosen": -435.2280578613281,
      "logps/rejected": -443.97833251953125,
      "loss": 0.4474,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3076425790786743,
      "rewards/margins": 0.9267934560775757,
      "rewards/rejected": -2.23443603515625,
      "step": 11635
    },
    {
      "epoch": 1.5226466017829394,
      "grad_norm": 14.18162727355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1720239371061325,
      "logits/rejected": -0.4649219214916229,
      "logps/chosen": -375.41949462890625,
      "logps/rejected": -359.51177978515625,
      "loss": 0.8216,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5612744092941284,
      "rewards/margins": 0.4320828318595886,
      "rewards/rejected": -1.9933569431304932,
      "step": 11636
    },
    {
      "epoch": 1.5227774597202912,
      "grad_norm": 11.75759506225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27824878692626953,
      "logits/rejected": -0.036710940301418304,
      "logps/chosen": -376.76611328125,
      "logps/rejected": -433.6868896484375,
      "loss": 0.8205,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9817975759506226,
      "rewards/margins": 0.29160386323928833,
      "rewards/rejected": -1.2734013795852661,
      "step": 11637
    },
    {
      "epoch": 1.522908317657643,
      "grad_norm": 5.016489505767822,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009513083845376968,
      "logits/rejected": -0.11723043769598007,
      "logps/chosen": -703.4068603515625,
      "logps/rejected": -588.5174560546875,
      "loss": 0.222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5497573018074036,
      "rewards/margins": 2.0126545429229736,
      "rewards/rejected": -2.5624117851257324,
      "step": 11638
    },
    {
      "epoch": 1.5230391755949948,
      "grad_norm": 4.266682147979736,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2765023410320282,
      "logits/rejected": -0.2963007688522339,
      "logps/chosen": -499.620849609375,
      "logps/rejected": -359.7917175292969,
      "loss": 0.1741,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.227087140083313,
      "rewards/margins": 2.330463409423828,
      "rewards/rejected": -3.5575504302978516,
      "step": 11639
    },
    {
      "epoch": 1.5231700335323466,
      "grad_norm": 4.930787563323975,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11830355226993561,
      "logits/rejected": -0.02385162003338337,
      "logps/chosen": -639.8306274414062,
      "logps/rejected": -553.374755859375,
      "loss": 0.1799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08591973781585693,
      "rewards/margins": 2.5761682987213135,
      "rewards/rejected": -2.490248203277588,
      "step": 11640
    },
    {
      "epoch": 1.5233008914696982,
      "grad_norm": 4.9036455154418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039003148674964905,
      "logits/rejected": -0.19843138754367828,
      "logps/chosen": -442.4114074707031,
      "logps/rejected": -335.3669738769531,
      "loss": 0.1688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36410006880760193,
      "rewards/margins": 2.286393642425537,
      "rewards/rejected": -2.6504933834075928,
      "step": 11641
    },
    {
      "epoch": 1.52343174940705,
      "grad_norm": 4.308365821838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1252482682466507,
      "logits/rejected": 0.05241728201508522,
      "logps/chosen": -487.715576171875,
      "logps/rejected": -463.164306640625,
      "loss": 0.1683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09750103205442429,
      "rewards/margins": 2.6251511573791504,
      "rewards/rejected": -2.7226521968841553,
      "step": 11642
    },
    {
      "epoch": 1.5235626073444017,
      "grad_norm": 6.431029796600342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09396016597747803,
      "logits/rejected": 0.05627932399511337,
      "logps/chosen": -484.3490295410156,
      "logps/rejected": -490.88720703125,
      "loss": 0.3125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.020748496055603,
      "rewards/margins": 1.407447338104248,
      "rewards/rejected": -2.4281959533691406,
      "step": 11643
    },
    {
      "epoch": 1.5236934652817535,
      "grad_norm": 7.064042091369629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0847805067896843,
      "logits/rejected": -0.08450385928153992,
      "logps/chosen": -417.13861083984375,
      "logps/rejected": -551.208740234375,
      "loss": 0.2328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6768396496772766,
      "rewards/margins": 1.9997485876083374,
      "rewards/rejected": -2.676588296890259,
      "step": 11644
    },
    {
      "epoch": 1.523824323219105,
      "grad_norm": 9.57882308959961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0380842462182045,
      "logits/rejected": -0.13128811120986938,
      "logps/chosen": -433.88372802734375,
      "logps/rejected": -418.955322265625,
      "loss": 0.5845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2757691144943237,
      "rewards/margins": 0.73943030834198,
      "rewards/rejected": -2.015199661254883,
      "step": 11645
    },
    {
      "epoch": 1.523955181156457,
      "grad_norm": 6.6751627922058105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2610609531402588,
      "logits/rejected": -0.3800576329231262,
      "logps/chosen": -538.568603515625,
      "logps/rejected": -504.29083251953125,
      "loss": 0.3896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2541046142578125,
      "rewards/margins": 1.5177569389343262,
      "rewards/rejected": -1.7718614339828491,
      "step": 11646
    },
    {
      "epoch": 1.5240860390938087,
      "grad_norm": 7.2483930587768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16901488602161407,
      "logits/rejected": -0.2695735991001129,
      "logps/chosen": -409.63671875,
      "logps/rejected": -364.5445556640625,
      "loss": 0.4463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.778018832206726,
      "rewards/margins": 1.2695212364196777,
      "rewards/rejected": -3.0475399494171143,
      "step": 11647
    },
    {
      "epoch": 1.5242168970311605,
      "grad_norm": 6.692533493041992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05918479338288307,
      "logits/rejected": 0.11414319276809692,
      "logps/chosen": -551.10888671875,
      "logps/rejected": -515.6290283203125,
      "loss": 0.2912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5843524932861328,
      "rewards/margins": 1.2526241540908813,
      "rewards/rejected": -1.8369767665863037,
      "step": 11648
    },
    {
      "epoch": 1.5243477549685123,
      "grad_norm": 6.542456150054932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5716781616210938,
      "logits/rejected": -0.6890736818313599,
      "logps/chosen": -650.65478515625,
      "logps/rejected": -762.681396484375,
      "loss": 0.275,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.57984459400177,
      "rewards/margins": 1.6693031787872314,
      "rewards/rejected": -3.249147415161133,
      "step": 11649
    },
    {
      "epoch": 1.524478612905864,
      "grad_norm": 4.8900041580200195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28005778789520264,
      "logits/rejected": -0.1764882206916809,
      "logps/chosen": -404.07666015625,
      "logps/rejected": -287.4151611328125,
      "loss": 0.3352,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4994717836380005,
      "rewards/margins": 1.2323002815246582,
      "rewards/rejected": -2.7317721843719482,
      "step": 11650
    },
    {
      "epoch": 1.5246094708432159,
      "grad_norm": 11.577648162841797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26457804441452026,
      "logits/rejected": -0.26774951815605164,
      "logps/chosen": -519.5936889648438,
      "logps/rejected": -555.5422973632812,
      "loss": 0.7699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0758332014083862,
      "rewards/margins": 0.6566482186317444,
      "rewards/rejected": -1.7324814796447754,
      "step": 11651
    },
    {
      "epoch": 1.5247403287805676,
      "grad_norm": 7.230193614959717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3418671786785126,
      "logits/rejected": -0.17085856199264526,
      "logps/chosen": -337.9305419921875,
      "logps/rejected": -358.82513427734375,
      "loss": 0.5581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5777639150619507,
      "rewards/margins": 1.0899081230163574,
      "rewards/rejected": -1.6676721572875977,
      "step": 11652
    },
    {
      "epoch": 1.5248711867179194,
      "grad_norm": 9.243947982788086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05481170117855072,
      "logits/rejected": -0.2802068293094635,
      "logps/chosen": -388.58123779296875,
      "logps/rejected": -510.412109375,
      "loss": 0.4122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3663654327392578,
      "rewards/margins": 2.2065176963806152,
      "rewards/rejected": -3.5728836059570312,
      "step": 11653
    },
    {
      "epoch": 1.5250020446552712,
      "grad_norm": 5.455781936645508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25539451837539673,
      "logits/rejected": -0.41527363657951355,
      "logps/chosen": -292.929443359375,
      "logps/rejected": -408.2078857421875,
      "loss": 0.2404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5386921763420105,
      "rewards/margins": 2.087867021560669,
      "rewards/rejected": -2.626559257507324,
      "step": 11654
    },
    {
      "epoch": 1.525132902592623,
      "grad_norm": 4.637876987457275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012920500710606575,
      "logits/rejected": -0.14295007288455963,
      "logps/chosen": -441.26617431640625,
      "logps/rejected": -495.1387939453125,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6031641364097595,
      "rewards/margins": 2.490020275115967,
      "rewards/rejected": -3.09318470954895,
      "step": 11655
    },
    {
      "epoch": 1.5252637605299748,
      "grad_norm": 7.674811363220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24701571464538574,
      "logits/rejected": -0.26889967918395996,
      "logps/chosen": -574.730224609375,
      "logps/rejected": -482.44586181640625,
      "loss": 0.3348,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7832236289978027,
      "rewards/margins": 1.5179731845855713,
      "rewards/rejected": -2.301196813583374,
      "step": 11656
    },
    {
      "epoch": 1.5253946184673264,
      "grad_norm": 9.481855392456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29120996594429016,
      "logits/rejected": -0.22715714573860168,
      "logps/chosen": -637.93408203125,
      "logps/rejected": -643.9674682617188,
      "loss": 0.4279,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9881629943847656,
      "rewards/margins": 1.2660832405090332,
      "rewards/rejected": -2.254246234893799,
      "step": 11657
    },
    {
      "epoch": 1.5255254764046782,
      "grad_norm": 7.70974063873291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2390623688697815,
      "logits/rejected": -0.28839871287345886,
      "logps/chosen": -429.5081787109375,
      "logps/rejected": -383.5540771484375,
      "loss": 0.3031,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9497846961021423,
      "rewards/margins": 1.6614634990692139,
      "rewards/rejected": -2.611248254776001,
      "step": 11658
    },
    {
      "epoch": 1.52565633434203,
      "grad_norm": 5.9585442543029785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20463186502456665,
      "logits/rejected": -0.26967212557792664,
      "logps/chosen": -589.6681518554688,
      "logps/rejected": -464.89691162109375,
      "loss": 0.2398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8827051520347595,
      "rewards/margins": 1.679935097694397,
      "rewards/rejected": -2.5626401901245117,
      "step": 11659
    },
    {
      "epoch": 1.5257871922793815,
      "grad_norm": 9.238998413085938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31623655557632446,
      "logits/rejected": -0.30706310272216797,
      "logps/chosen": -638.8130493164062,
      "logps/rejected": -719.31884765625,
      "loss": 0.4061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.875397801399231,
      "rewards/margins": 2.1062707901000977,
      "rewards/rejected": -3.981667995452881,
      "step": 11660
    },
    {
      "epoch": 1.5259180502167333,
      "grad_norm": 7.451336860656738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3274594247341156,
      "logits/rejected": -0.39723071455955505,
      "logps/chosen": -487.2154541015625,
      "logps/rejected": -450.1246032714844,
      "loss": 0.3584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1514840126037598,
      "rewards/margins": 1.2708067893981934,
      "rewards/rejected": -3.4222910404205322,
      "step": 11661
    },
    {
      "epoch": 1.5260489081540851,
      "grad_norm": 7.74888277053833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5545211434364319,
      "logits/rejected": -0.38039430975914,
      "logps/chosen": -477.0392150878906,
      "logps/rejected": -468.81048583984375,
      "loss": 0.3459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8584842085838318,
      "rewards/margins": 1.5484132766723633,
      "rewards/rejected": -2.40689754486084,
      "step": 11662
    },
    {
      "epoch": 1.526179766091437,
      "grad_norm": 8.933562278747559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14663255214691162,
      "logits/rejected": -0.15326642990112305,
      "logps/chosen": -411.8425598144531,
      "logps/rejected": -416.4308776855469,
      "loss": 0.4634,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8284246921539307,
      "rewards/margins": 1.2784067392349243,
      "rewards/rejected": -2.1068313121795654,
      "step": 11663
    },
    {
      "epoch": 1.5263106240287887,
      "grad_norm": 6.026881217956543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18112193048000336,
      "logits/rejected": -0.26822996139526367,
      "logps/chosen": -386.9277648925781,
      "logps/rejected": -309.9642333984375,
      "loss": 0.2011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.508984386920929,
      "rewards/margins": 2.098939895629883,
      "rewards/rejected": -2.607924222946167,
      "step": 11664
    },
    {
      "epoch": 1.5264414819661405,
      "grad_norm": 13.239901542663574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021320633590221405,
      "logits/rejected": -0.32216954231262207,
      "logps/chosen": -569.7052612304688,
      "logps/rejected": -369.8999938964844,
      "loss": 0.5416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45897939801216125,
      "rewards/margins": 0.7433023452758789,
      "rewards/rejected": -1.2022818326950073,
      "step": 11665
    },
    {
      "epoch": 1.5265723399034923,
      "grad_norm": 9.2393217086792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31105124950408936,
      "logits/rejected": -0.46928486227989197,
      "logps/chosen": -305.1448059082031,
      "logps/rejected": -450.4202575683594,
      "loss": 0.4457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4641772508621216,
      "rewards/margins": 2.127502679824829,
      "rewards/rejected": -3.591679811477661,
      "step": 11666
    },
    {
      "epoch": 1.526703197840844,
      "grad_norm": 10.699264526367188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1137194111943245,
      "logits/rejected": -0.30420148372650146,
      "logps/chosen": -302.37579345703125,
      "logps/rejected": -520.5293579101562,
      "loss": 0.4529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4027979373931885,
      "rewards/margins": 0.9891149401664734,
      "rewards/rejected": -3.3919129371643066,
      "step": 11667
    },
    {
      "epoch": 1.5268340557781959,
      "grad_norm": 11.331076622009277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32275402545928955,
      "logits/rejected": -0.1159953624010086,
      "logps/chosen": -448.92626953125,
      "logps/rejected": -471.6510314941406,
      "loss": 1.1851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.643670678138733,
      "rewards/margins": 0.9203349947929382,
      "rewards/rejected": -2.5640058517456055,
      "step": 11668
    },
    {
      "epoch": 1.5269649137155477,
      "grad_norm": 5.853076934814453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05678228288888931,
      "logits/rejected": -0.3421464264392853,
      "logps/chosen": -505.499755859375,
      "logps/rejected": -413.32110595703125,
      "loss": 0.3147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6975052356719971,
      "rewards/margins": 1.2565425634384155,
      "rewards/rejected": -1.9540477991104126,
      "step": 11669
    },
    {
      "epoch": 1.5270957716528994,
      "grad_norm": 4.73095703125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28188976645469666,
      "logits/rejected": -0.2818862497806549,
      "logps/chosen": -350.6243896484375,
      "logps/rejected": -489.94671630859375,
      "loss": 0.1603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8392863273620605,
      "rewards/margins": 2.9181008338928223,
      "rewards/rejected": -3.757387161254883,
      "step": 11670
    },
    {
      "epoch": 1.5272266295902512,
      "grad_norm": 3.972405195236206,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0719711109995842,
      "logits/rejected": -0.12909580767154694,
      "logps/chosen": -338.8024597167969,
      "logps/rejected": -504.9127502441406,
      "loss": 0.1073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13315603137016296,
      "rewards/margins": 3.2314419746398926,
      "rewards/rejected": -3.364597797393799,
      "step": 11671
    },
    {
      "epoch": 1.5273574875276028,
      "grad_norm": 11.78339958190918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.110780730843544,
      "logits/rejected": -0.04857310652732849,
      "logps/chosen": -725.076904296875,
      "logps/rejected": -503.7948303222656,
      "loss": 0.3984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5559257864952087,
      "rewards/margins": 2.1269850730895996,
      "rewards/rejected": -2.682910680770874,
      "step": 11672
    },
    {
      "epoch": 1.5274883454649546,
      "grad_norm": 6.578361988067627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15594738721847534,
      "logits/rejected": -0.07887747138738632,
      "logps/chosen": -380.3992614746094,
      "logps/rejected": -345.56280517578125,
      "loss": 0.4657,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3966838121414185,
      "rewards/margins": 1.3015414476394653,
      "rewards/rejected": -2.698225259780884,
      "step": 11673
    },
    {
      "epoch": 1.5276192034023064,
      "grad_norm": 11.414966583251953,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22259533405303955,
      "logits/rejected": 0.03603534400463104,
      "logps/chosen": -433.56353759765625,
      "logps/rejected": -654.656494140625,
      "loss": 0.7722,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1241481304168701,
      "rewards/margins": 0.8852896690368652,
      "rewards/rejected": -2.0094380378723145,
      "step": 11674
    },
    {
      "epoch": 1.5277500613396582,
      "grad_norm": 6.752772808074951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013565972447395325,
      "logits/rejected": -0.10906863957643509,
      "logps/chosen": -473.43353271484375,
      "logps/rejected": -398.15460205078125,
      "loss": 0.2701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9975689053535461,
      "rewards/margins": 2.1087937355041504,
      "rewards/rejected": -3.1063625812530518,
      "step": 11675
    },
    {
      "epoch": 1.5278809192770098,
      "grad_norm": 13.083702087402344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16381245851516724,
      "logits/rejected": -0.47945237159729004,
      "logps/chosen": -495.13336181640625,
      "logps/rejected": -394.7532043457031,
      "loss": 0.6691,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.6047868728637695,
      "rewards/margins": 1.1235735416412354,
      "rewards/rejected": -2.728360176086426,
      "step": 11676
    },
    {
      "epoch": 1.5280117772143615,
      "grad_norm": 20.889846801757812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4760742485523224,
      "logits/rejected": -0.467448890209198,
      "logps/chosen": -469.528076171875,
      "logps/rejected": -451.620361328125,
      "loss": 0.9616,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9040910005569458,
      "rewards/margins": 0.015187263488769531,
      "rewards/rejected": -1.9192782640457153,
      "step": 11677
    },
    {
      "epoch": 1.5281426351517133,
      "grad_norm": 9.017851829528809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3784469962120056,
      "logits/rejected": -0.25741472840309143,
      "logps/chosen": -615.03857421875,
      "logps/rejected": -468.5893249511719,
      "loss": 0.4362,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2976150512695312,
      "rewards/margins": 0.8421371579170227,
      "rewards/rejected": -2.139752149581909,
      "step": 11678
    },
    {
      "epoch": 1.5282734930890651,
      "grad_norm": 4.4589338302612305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12169287353754044,
      "logits/rejected": -0.005301378667354584,
      "logps/chosen": -349.36785888671875,
      "logps/rejected": -255.54359436035156,
      "loss": 0.1688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1269972324371338,
      "rewards/margins": 2.164773941040039,
      "rewards/rejected": -3.2917709350585938,
      "step": 11679
    },
    {
      "epoch": 1.528404351026417,
      "grad_norm": 12.65214729309082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3275443911552429,
      "logits/rejected": -0.14425979554653168,
      "logps/chosen": -427.09564208984375,
      "logps/rejected": -429.85955810546875,
      "loss": 0.7308,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9852949380874634,
      "rewards/margins": 0.9486365914344788,
      "rewards/rejected": -2.933931589126587,
      "step": 11680
    },
    {
      "epoch": 1.5285352089637687,
      "grad_norm": 8.060555458068848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5310560464859009,
      "logits/rejected": -0.7243474721908569,
      "logps/chosen": -280.2105712890625,
      "logps/rejected": -284.8414306640625,
      "loss": 0.7708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2404640913009644,
      "rewards/margins": 0.3402324318885803,
      "rewards/rejected": -1.5806965827941895,
      "step": 11681
    },
    {
      "epoch": 1.5286660669011205,
      "grad_norm": 13.340287208557129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008344275876879692,
      "logits/rejected": -0.23439040780067444,
      "logps/chosen": -719.0069580078125,
      "logps/rejected": -658.586669921875,
      "loss": 0.9245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3122167587280273,
      "rewards/margins": 0.35712453722953796,
      "rewards/rejected": -2.669341564178467,
      "step": 11682
    },
    {
      "epoch": 1.5287969248384723,
      "grad_norm": 12.543481826782227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11173420399427414,
      "logits/rejected": -0.22311189770698547,
      "logps/chosen": -602.9403076171875,
      "logps/rejected": -750.26220703125,
      "loss": 1.022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0268596410751343,
      "rewards/margins": 1.6132266521453857,
      "rewards/rejected": -2.6400861740112305,
      "step": 11683
    },
    {
      "epoch": 1.528927782775824,
      "grad_norm": 9.680418014526367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5392118096351624,
      "logits/rejected": -0.20874737203121185,
      "logps/chosen": -716.0206909179688,
      "logps/rejected": -312.33746337890625,
      "loss": 0.4947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3947280645370483,
      "rewards/margins": 0.7960639595985413,
      "rewards/rejected": -2.1907918453216553,
      "step": 11684
    },
    {
      "epoch": 1.5290586407131759,
      "grad_norm": 5.742737293243408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4242551624774933,
      "logits/rejected": -0.5210704803466797,
      "logps/chosen": -508.8821105957031,
      "logps/rejected": -428.288330078125,
      "loss": 0.2258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46858999133110046,
      "rewards/margins": 2.258333206176758,
      "rewards/rejected": -2.7269232273101807,
      "step": 11685
    },
    {
      "epoch": 1.5291894986505277,
      "grad_norm": 10.981985092163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.395931601524353,
      "logits/rejected": -0.30932435393333435,
      "logps/chosen": -315.69903564453125,
      "logps/rejected": -358.649169921875,
      "loss": 0.7042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.958566427230835,
      "rewards/margins": 1.0085090398788452,
      "rewards/rejected": -1.9670757055282593,
      "step": 11686
    },
    {
      "epoch": 1.5293203565878792,
      "grad_norm": 5.3675103187561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2785791754722595,
      "logits/rejected": -0.3633502423763275,
      "logps/chosen": -475.1021728515625,
      "logps/rejected": -460.5859680175781,
      "loss": 0.2534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4159221351146698,
      "rewards/margins": 1.8112661838531494,
      "rewards/rejected": -2.2271883487701416,
      "step": 11687
    },
    {
      "epoch": 1.529451214525231,
      "grad_norm": 8.383501052856445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09120765328407288,
      "logits/rejected": -0.04610238969326019,
      "logps/chosen": -362.8800964355469,
      "logps/rejected": -337.8891296386719,
      "loss": 0.2752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1687111109495163,
      "rewards/margins": 1.758469820022583,
      "rewards/rejected": -1.9271810054779053,
      "step": 11688
    },
    {
      "epoch": 1.5295820724625828,
      "grad_norm": 12.100500106811523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06474655866622925,
      "logits/rejected": -0.1500425785779953,
      "logps/chosen": -560.8062133789062,
      "logps/rejected": -490.2322998046875,
      "loss": 0.6134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1773830652236938,
      "rewards/margins": 1.519644021987915,
      "rewards/rejected": -2.6970272064208984,
      "step": 11689
    },
    {
      "epoch": 1.5297129303999346,
      "grad_norm": 8.674007415771484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41993868350982666,
      "logits/rejected": -0.5596591234207153,
      "logps/chosen": -484.43353271484375,
      "logps/rejected": -519.0894775390625,
      "loss": 0.5141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3165765702724457,
      "rewards/margins": 2.1504573822021484,
      "rewards/rejected": -1.8338813781738281,
      "step": 11690
    },
    {
      "epoch": 1.5298437883372862,
      "grad_norm": 8.171247482299805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00998114887624979,
      "logits/rejected": 0.06792574375867844,
      "logps/chosen": -427.4339904785156,
      "logps/rejected": -494.1333312988281,
      "loss": 0.4995,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13445349037647247,
      "rewards/margins": 0.660162091255188,
      "rewards/rejected": -0.794615626335144,
      "step": 11691
    },
    {
      "epoch": 1.529974646274638,
      "grad_norm": 9.866683006286621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058084938675165176,
      "logits/rejected": -0.12307238578796387,
      "logps/chosen": -401.65447998046875,
      "logps/rejected": -496.5112609863281,
      "loss": 0.5127,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5546920895576477,
      "rewards/margins": 1.951775074005127,
      "rewards/rejected": -2.50646710395813,
      "step": 11692
    },
    {
      "epoch": 1.5301055042119898,
      "grad_norm": 9.97280216217041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.208426833152771,
      "logits/rejected": -0.16716468334197998,
      "logps/chosen": -590.626220703125,
      "logps/rejected": -466.462646484375,
      "loss": 0.5295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5665450692176819,
      "rewards/margins": 1.3955442905426025,
      "rewards/rejected": -1.9620895385742188,
      "step": 11693
    },
    {
      "epoch": 1.5302363621493416,
      "grad_norm": 10.769543647766113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2869681119918823,
      "logits/rejected": -0.6762620210647583,
      "logps/chosen": -524.8912963867188,
      "logps/rejected": -470.76190185546875,
      "loss": 0.5901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.418176531791687,
      "rewards/margins": 0.9814941883087158,
      "rewards/rejected": -2.3996706008911133,
      "step": 11694
    },
    {
      "epoch": 1.5303672200866933,
      "grad_norm": 4.937402248382568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09385325014591217,
      "logits/rejected": -0.16577960550785065,
      "logps/chosen": -567.0108642578125,
      "logps/rejected": -576.1178588867188,
      "loss": 0.1994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17295590043067932,
      "rewards/margins": 2.2028086185455322,
      "rewards/rejected": -2.3757643699645996,
      "step": 11695
    },
    {
      "epoch": 1.5304980780240451,
      "grad_norm": 7.3766703605651855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2394338697195053,
      "logits/rejected": -0.22199060022830963,
      "logps/chosen": -340.75396728515625,
      "logps/rejected": -358.8590393066406,
      "loss": 0.4831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5234681963920593,
      "rewards/margins": 1.189916968345642,
      "rewards/rejected": -1.713384985923767,
      "step": 11696
    },
    {
      "epoch": 1.530628935961397,
      "grad_norm": 5.1458964347839355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2512780427932739,
      "logits/rejected": 0.1466008871793747,
      "logps/chosen": -454.6017150878906,
      "logps/rejected": -315.8762512207031,
      "loss": 0.3037,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.312139093875885,
      "rewards/margins": 1.7477405071258545,
      "rewards/rejected": -1.4356014728546143,
      "step": 11697
    },
    {
      "epoch": 1.5307597938987487,
      "grad_norm": 5.399857521057129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27241572737693787,
      "logits/rejected": -0.24254216253757477,
      "logps/chosen": -389.92877197265625,
      "logps/rejected": -513.1027221679688,
      "loss": 0.3025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24676840007305145,
      "rewards/margins": 2.488471746444702,
      "rewards/rejected": -2.7352404594421387,
      "step": 11698
    },
    {
      "epoch": 1.5308906518361005,
      "grad_norm": 8.162531852722168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0872480571269989,
      "logits/rejected": -0.263827919960022,
      "logps/chosen": -436.3380126953125,
      "logps/rejected": -383.8140869140625,
      "loss": 0.4438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.86784428358078,
      "rewards/margins": 1.131412386894226,
      "rewards/rejected": -1.9992566108703613,
      "step": 11699
    },
    {
      "epoch": 1.5310215097734523,
      "grad_norm": 3.5832717418670654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3638136684894562,
      "logits/rejected": -0.21506333351135254,
      "logps/chosen": -635.2418212890625,
      "logps/rejected": -476.8369140625,
      "loss": 0.1447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5535978078842163,
      "rewards/margins": 3.4046483039855957,
      "rewards/rejected": -3.9582459926605225,
      "step": 11700
    },
    {
      "epoch": 1.531152367710804,
      "grad_norm": 6.324071407318115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21544703841209412,
      "logits/rejected": -0.02472933754324913,
      "logps/chosen": -442.3494567871094,
      "logps/rejected": -459.9480285644531,
      "loss": 0.3076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9022039175033569,
      "rewards/margins": 1.3951457738876343,
      "rewards/rejected": -2.297349691390991,
      "step": 11701
    },
    {
      "epoch": 1.5312832256481557,
      "grad_norm": 4.515686511993408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14834357798099518,
      "logits/rejected": -0.2460232377052307,
      "logps/chosen": -555.5922241210938,
      "logps/rejected": -470.83026123046875,
      "loss": 0.1765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3826805353164673,
      "rewards/margins": 2.571146011352539,
      "rewards/rejected": -2.953826427459717,
      "step": 11702
    },
    {
      "epoch": 1.5314140835855075,
      "grad_norm": 8.431645393371582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.135369211435318,
      "logits/rejected": -0.17195023596286774,
      "logps/chosen": -340.2322998046875,
      "logps/rejected": -426.6734924316406,
      "loss": 0.633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.200668454170227,
      "rewards/margins": 0.6274999380111694,
      "rewards/rejected": -1.8281686305999756,
      "step": 11703
    },
    {
      "epoch": 1.5315449415228592,
      "grad_norm": 11.357558250427246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3223605155944824,
      "logits/rejected": -0.24587751924991608,
      "logps/chosen": -326.1517639160156,
      "logps/rejected": -450.96307373046875,
      "loss": 0.8467,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.094166874885559,
      "rewards/margins": 0.26633143424987793,
      "rewards/rejected": -1.3604984283447266,
      "step": 11704
    },
    {
      "epoch": 1.531675799460211,
      "grad_norm": 7.0953850746154785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.191024512052536,
      "logits/rejected": -0.07175848633050919,
      "logps/chosen": -305.95086669921875,
      "logps/rejected": -444.2128601074219,
      "loss": 0.3918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48216477036476135,
      "rewards/margins": 1.2943577766418457,
      "rewards/rejected": -1.7765226364135742,
      "step": 11705
    },
    {
      "epoch": 1.5318066573975626,
      "grad_norm": 7.308772563934326,
      "learning_rate": 5e-06,
      "logits/chosen": -0.083281509578228,
      "logits/rejected": -0.38459542393684387,
      "logps/chosen": -387.2540283203125,
      "logps/rejected": -364.7740173339844,
      "loss": 0.3386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7100271582603455,
      "rewards/margins": 2.2714803218841553,
      "rewards/rejected": -2.9815073013305664,
      "step": 11706
    },
    {
      "epoch": 1.5319375153349144,
      "grad_norm": 8.981707572937012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3965327739715576,
      "logits/rejected": -0.46298927068710327,
      "logps/chosen": -531.0758056640625,
      "logps/rejected": -531.1889038085938,
      "loss": 0.4751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45858341455459595,
      "rewards/margins": 1.0241022109985352,
      "rewards/rejected": -1.4826856851577759,
      "step": 11707
    },
    {
      "epoch": 1.5320683732722662,
      "grad_norm": 4.527109622955322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18229809403419495,
      "logits/rejected": -0.18082834780216217,
      "logps/chosen": -482.9383239746094,
      "logps/rejected": -416.3524169921875,
      "loss": 0.2398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8767582178115845,
      "rewards/margins": 1.6665680408477783,
      "rewards/rejected": -2.5433261394500732,
      "step": 11708
    },
    {
      "epoch": 1.532199231209618,
      "grad_norm": 7.6326704025268555,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0720454752445221,
      "logits/rejected": -0.132775217294693,
      "logps/chosen": -591.1841430664062,
      "logps/rejected": -527.55078125,
      "loss": 0.4083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3128834068775177,
      "rewards/margins": 1.1030523777008057,
      "rewards/rejected": -1.4159356355667114,
      "step": 11709
    },
    {
      "epoch": 1.5323300891469698,
      "grad_norm": 8.407702445983887,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08476606756448746,
      "logits/rejected": -0.09370424598455429,
      "logps/chosen": -494.6979675292969,
      "logps/rejected": -517.8687133789062,
      "loss": 0.4249,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9771372079849243,
      "rewards/margins": 1.3914731740951538,
      "rewards/rejected": -2.3686106204986572,
      "step": 11710
    },
    {
      "epoch": 1.5324609470843216,
      "grad_norm": 7.233341217041016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24656738340854645,
      "logits/rejected": -0.18801221251487732,
      "logps/chosen": -460.0653076171875,
      "logps/rejected": -424.548828125,
      "loss": 0.284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.986475944519043,
      "rewards/margins": 1.7156345844268799,
      "rewards/rejected": -2.702110528945923,
      "step": 11711
    },
    {
      "epoch": 1.5325918050216734,
      "grad_norm": 9.247567176818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20859520137310028,
      "logits/rejected": -0.13253578543663025,
      "logps/chosen": -551.286865234375,
      "logps/rejected": -387.34539794921875,
      "loss": 0.3965,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.661887526512146,
      "rewards/margins": 1.315349817276001,
      "rewards/rejected": -1.977237343788147,
      "step": 11712
    },
    {
      "epoch": 1.5327226629590252,
      "grad_norm": 6.284893989562988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4102012813091278,
      "logits/rejected": -0.19994622468948364,
      "logps/chosen": -582.635986328125,
      "logps/rejected": -644.087158203125,
      "loss": 0.2501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8622365593910217,
      "rewards/margins": 1.7411847114562988,
      "rewards/rejected": -2.603421449661255,
      "step": 11713
    },
    {
      "epoch": 1.532853520896377,
      "grad_norm": 8.125398635864258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4444940984249115,
      "logits/rejected": -0.2327948808670044,
      "logps/chosen": -439.15032958984375,
      "logps/rejected": -398.0836181640625,
      "loss": 0.485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.018182396888733,
      "rewards/margins": 0.7899183630943298,
      "rewards/rejected": -1.808100700378418,
      "step": 11714
    },
    {
      "epoch": 1.5329843788337287,
      "grad_norm": 8.750040054321289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08942873775959015,
      "logits/rejected": 0.017258308827877045,
      "logps/chosen": -476.31329345703125,
      "logps/rejected": -494.8585510253906,
      "loss": 0.5556,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6686261296272278,
      "rewards/margins": 0.5957340002059937,
      "rewards/rejected": -1.2643600702285767,
      "step": 11715
    },
    {
      "epoch": 1.5331152367710805,
      "grad_norm": 10.30484676361084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4340120553970337,
      "logits/rejected": -0.3714110553264618,
      "logps/chosen": -371.1014709472656,
      "logps/rejected": -267.2774658203125,
      "loss": 0.3619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1799774169921875,
      "rewards/margins": 1.297810435295105,
      "rewards/rejected": -2.477787971496582,
      "step": 11716
    },
    {
      "epoch": 1.5332460947084323,
      "grad_norm": 7.470711708068848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16185933351516724,
      "logits/rejected": -0.3712214231491089,
      "logps/chosen": -457.4581298828125,
      "logps/rejected": -413.0547180175781,
      "loss": 0.4649,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6026039123535156,
      "rewards/margins": 1.2810579538345337,
      "rewards/rejected": -1.8836618661880493,
      "step": 11717
    },
    {
      "epoch": 1.5333769526457839,
      "grad_norm": 9.67237377166748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2107376754283905,
      "logits/rejected": -0.2324126660823822,
      "logps/chosen": -729.4273681640625,
      "logps/rejected": -597.33984375,
      "loss": 0.5112,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.013024896383285522,
      "rewards/margins": 1.0646648406982422,
      "rewards/rejected": -1.07768976688385,
      "step": 11718
    },
    {
      "epoch": 1.5335078105831357,
      "grad_norm": 5.291820049285889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19502677023410797,
      "logits/rejected": -0.19616037607192993,
      "logps/chosen": -414.6407775878906,
      "logps/rejected": -403.0809631347656,
      "loss": 0.266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3644660711288452,
      "rewards/margins": 2.0994300842285156,
      "rewards/rejected": -2.4638962745666504,
      "step": 11719
    },
    {
      "epoch": 1.5336386685204875,
      "grad_norm": 22.937728881835938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3825502395629883,
      "logits/rejected": -0.3737961947917938,
      "logps/chosen": -465.65155029296875,
      "logps/rejected": -636.9132080078125,
      "loss": 0.476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.174620509147644,
      "rewards/margins": 0.8631396889686584,
      "rewards/rejected": -2.0377602577209473,
      "step": 11720
    },
    {
      "epoch": 1.5337695264578393,
      "grad_norm": 6.7404303550720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34775981307029724,
      "logits/rejected": -0.3921184837818146,
      "logps/chosen": -453.54180908203125,
      "logps/rejected": -435.5439453125,
      "loss": 0.3495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8671423196792603,
      "rewards/margins": 0.970768928527832,
      "rewards/rejected": -1.8379112482070923,
      "step": 11721
    },
    {
      "epoch": 1.5339003843951908,
      "grad_norm": 11.172672271728516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4234025180339813,
      "logits/rejected": -0.5558242201805115,
      "logps/chosen": -458.5283508300781,
      "logps/rejected": -418.8704833984375,
      "loss": 0.5871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5158756971359253,
      "rewards/margins": 0.8007393479347229,
      "rewards/rejected": -2.316614866256714,
      "step": 11722
    },
    {
      "epoch": 1.5340312423325426,
      "grad_norm": 7.754715442657471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06730642169713974,
      "logits/rejected": -0.30356383323669434,
      "logps/chosen": -289.43994140625,
      "logps/rejected": -303.03826904296875,
      "loss": 0.4523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5441963076591492,
      "rewards/margins": 0.8548624515533447,
      "rewards/rejected": -1.3990588188171387,
      "step": 11723
    },
    {
      "epoch": 1.5341621002698944,
      "grad_norm": 8.310308456420898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4278167188167572,
      "logits/rejected": -0.15626923739910126,
      "logps/chosen": -407.5396728515625,
      "logps/rejected": -416.7176513671875,
      "loss": 0.3767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6781067848205566,
      "rewards/margins": 1.1750378608703613,
      "rewards/rejected": -1.8531447649002075,
      "step": 11724
    },
    {
      "epoch": 1.5342929582072462,
      "grad_norm": 3.8337979316711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08398082107305527,
      "logits/rejected": 0.05654236301779747,
      "logps/chosen": -559.2573852539062,
      "logps/rejected": -756.4417114257812,
      "loss": 0.1899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5250129699707031,
      "rewards/margins": 2.439826250076294,
      "rewards/rejected": -2.964839220046997,
      "step": 11725
    },
    {
      "epoch": 1.534423816144598,
      "grad_norm": 9.326667785644531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02497090771794319,
      "logits/rejected": -0.07038640975952148,
      "logps/chosen": -402.8654479980469,
      "logps/rejected": -508.282470703125,
      "loss": 0.6619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20914456248283386,
      "rewards/margins": 1.808263897895813,
      "rewards/rejected": -2.0174083709716797,
      "step": 11726
    },
    {
      "epoch": 1.5345546740819498,
      "grad_norm": 8.078024864196777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07527976483106613,
      "logits/rejected": -0.10571293532848358,
      "logps/chosen": -497.5137634277344,
      "logps/rejected": -487.13458251953125,
      "loss": 0.5193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.172790765762329,
      "rewards/margins": 0.49822407960891724,
      "rewards/rejected": -1.6710149049758911,
      "step": 11727
    },
    {
      "epoch": 1.5346855320193016,
      "grad_norm": 7.910406112670898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.521612823009491,
      "logits/rejected": -0.3990321457386017,
      "logps/chosen": -402.6905822753906,
      "logps/rejected": -360.19378662109375,
      "loss": 0.3913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7351337671279907,
      "rewards/margins": 1.5752012729644775,
      "rewards/rejected": -2.3103349208831787,
      "step": 11728
    },
    {
      "epoch": 1.5348163899566534,
      "grad_norm": 5.945125579833984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4425627589225769,
      "logits/rejected": -0.4399210810661316,
      "logps/chosen": -296.83184814453125,
      "logps/rejected": -419.115478515625,
      "loss": 0.439,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0927976369857788,
      "rewards/margins": 1.4834920167922974,
      "rewards/rejected": -2.576289653778076,
      "step": 11729
    },
    {
      "epoch": 1.5349472478940052,
      "grad_norm": 12.327468872070312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36550700664520264,
      "logits/rejected": -0.331218421459198,
      "logps/chosen": -474.18719482421875,
      "logps/rejected": -477.0678405761719,
      "loss": 0.7305,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5692522525787354,
      "rewards/margins": 0.5877116918563843,
      "rewards/rejected": -2.156964063644409,
      "step": 11730
    },
    {
      "epoch": 1.535078105831357,
      "grad_norm": 9.56724739074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14018109440803528,
      "logits/rejected": -0.21497665345668793,
      "logps/chosen": -678.4012451171875,
      "logps/rejected": -665.35595703125,
      "loss": 0.584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49148696660995483,
      "rewards/margins": 0.5333049297332764,
      "rewards/rejected": -1.0247918367385864,
      "step": 11731
    },
    {
      "epoch": 1.5352089637687087,
      "grad_norm": 9.878536224365234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4145696759223938,
      "logits/rejected": -0.5211607217788696,
      "logps/chosen": -491.23773193359375,
      "logps/rejected": -598.6763916015625,
      "loss": 0.51,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9026655554771423,
      "rewards/margins": 1.657586932182312,
      "rewards/rejected": -2.5602524280548096,
      "step": 11732
    },
    {
      "epoch": 1.5353398217060603,
      "grad_norm": 10.951112747192383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0495273619890213,
      "logits/rejected": -0.03078736737370491,
      "logps/chosen": -503.5447082519531,
      "logps/rejected": -541.8206176757812,
      "loss": 0.6085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45072558522224426,
      "rewards/margins": 0.6074546575546265,
      "rewards/rejected": -1.0581802129745483,
      "step": 11733
    },
    {
      "epoch": 1.535470679643412,
      "grad_norm": 4.615641117095947,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40387284755706787,
      "logits/rejected": -0.36233294010162354,
      "logps/chosen": -294.0870056152344,
      "logps/rejected": -370.3270263671875,
      "loss": 0.2146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.679328203201294,
      "rewards/margins": 2.1158223152160645,
      "rewards/rejected": -2.7951502799987793,
      "step": 11734
    },
    {
      "epoch": 1.535601537580764,
      "grad_norm": 14.086557388305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14529913663864136,
      "logits/rejected": -0.10157611966133118,
      "logps/chosen": -668.3478393554688,
      "logps/rejected": -769.501708984375,
      "loss": 1.1746,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5644454956054688,
      "rewards/margins": 0.25238490104675293,
      "rewards/rejected": -1.8168307542800903,
      "step": 11735
    },
    {
      "epoch": 1.5357323955181157,
      "grad_norm": 5.156350135803223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0995689108967781,
      "logits/rejected": -0.2566946744918823,
      "logps/chosen": -535.2828369140625,
      "logps/rejected": -465.096923828125,
      "loss": 0.223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32390516996383667,
      "rewards/margins": 2.190962553024292,
      "rewards/rejected": -2.5148677825927734,
      "step": 11736
    },
    {
      "epoch": 1.5358632534554673,
      "grad_norm": 9.938359260559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2217472493648529,
      "logits/rejected": -0.19419649243354797,
      "logps/chosen": -457.1036376953125,
      "logps/rejected": -354.2338562011719,
      "loss": 0.4132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2925684452056885,
      "rewards/margins": 1.2603387832641602,
      "rewards/rejected": -2.5529072284698486,
      "step": 11737
    },
    {
      "epoch": 1.535994111392819,
      "grad_norm": 11.557052612304688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2038290798664093,
      "logits/rejected": -0.1249326691031456,
      "logps/chosen": -609.3709716796875,
      "logps/rejected": -458.00067138671875,
      "loss": 0.6345,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1306140422821045,
      "rewards/margins": 0.6550062298774719,
      "rewards/rejected": -1.7856203317642212,
      "step": 11738
    },
    {
      "epoch": 1.5361249693301708,
      "grad_norm": 10.629020690917969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23262609541416168,
      "logits/rejected": -0.13374869525432587,
      "logps/chosen": -313.4949951171875,
      "logps/rejected": -343.7950744628906,
      "loss": 0.8053,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3418447971343994,
      "rewards/margins": 0.15001901984214783,
      "rewards/rejected": -1.4918638467788696,
      "step": 11739
    },
    {
      "epoch": 1.5362558272675226,
      "grad_norm": 10.058067321777344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15023450553417206,
      "logits/rejected": 0.009600445628166199,
      "logps/chosen": -417.3816833496094,
      "logps/rejected": -362.8586120605469,
      "loss": 0.4357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8620333075523376,
      "rewards/margins": 0.9800065755844116,
      "rewards/rejected": -1.842039942741394,
      "step": 11740
    },
    {
      "epoch": 1.5363866852048744,
      "grad_norm": 9.731598854064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46246442198753357,
      "logits/rejected": -0.47054341435432434,
      "logps/chosen": -530.83056640625,
      "logps/rejected": -475.59356689453125,
      "loss": 0.5013,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6698379516601562,
      "rewards/margins": 1.3032429218292236,
      "rewards/rejected": -1.9730809926986694,
      "step": 11741
    },
    {
      "epoch": 1.5365175431422262,
      "grad_norm": 9.506853103637695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19341303408145905,
      "logits/rejected": -0.23296232521533966,
      "logps/chosen": -475.1679382324219,
      "logps/rejected": -407.83367919921875,
      "loss": 0.5024,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9782062768936157,
      "rewards/margins": 1.4392364025115967,
      "rewards/rejected": -2.417442798614502,
      "step": 11742
    },
    {
      "epoch": 1.536648401079578,
      "grad_norm": 5.438559532165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3402705192565918,
      "logits/rejected": -0.4735201895236969,
      "logps/chosen": -446.4043884277344,
      "logps/rejected": -344.34429931640625,
      "loss": 0.241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8585385680198669,
      "rewards/margins": 1.7873735427856445,
      "rewards/rejected": -2.645911931991577,
      "step": 11743
    },
    {
      "epoch": 1.5367792590169298,
      "grad_norm": 11.912961959838867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14868691563606262,
      "logits/rejected": -0.003220919519662857,
      "logps/chosen": -729.5996704101562,
      "logps/rejected": -724.5422973632812,
      "loss": 0.8492,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2324864864349365,
      "rewards/margins": 0.5640662908554077,
      "rewards/rejected": -1.7965527772903442,
      "step": 11744
    },
    {
      "epoch": 1.5369101169542816,
      "grad_norm": 6.671900749206543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3456083834171295,
      "logits/rejected": -0.21382278203964233,
      "logps/chosen": -292.6487121582031,
      "logps/rejected": -320.3306884765625,
      "loss": 0.4775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5708808898925781,
      "rewards/margins": 1.174642562866211,
      "rewards/rejected": -1.745523452758789,
      "step": 11745
    },
    {
      "epoch": 1.5370409748916334,
      "grad_norm": 13.043610572814941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012751162052154541,
      "logits/rejected": -0.1828082650899887,
      "logps/chosen": -496.4669189453125,
      "logps/rejected": -475.2511291503906,
      "loss": 0.9365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8391456604003906,
      "rewards/margins": 0.34782665967941284,
      "rewards/rejected": -1.1869722604751587,
      "step": 11746
    },
    {
      "epoch": 1.5371718328289852,
      "grad_norm": 3.498178243637085,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07847036421298981,
      "logits/rejected": -0.020091988146305084,
      "logps/chosen": -398.23065185546875,
      "logps/rejected": -393.0051574707031,
      "loss": 0.1485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18033868074417114,
      "rewards/margins": 2.185600757598877,
      "rewards/rejected": -2.0052621364593506,
      "step": 11747
    },
    {
      "epoch": 1.5373026907663367,
      "grad_norm": 8.92323112487793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2584376931190491,
      "logits/rejected": -0.3073120713233948,
      "logps/chosen": -535.2716674804688,
      "logps/rejected": -456.6484680175781,
      "loss": 0.6125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1540918350219727,
      "rewards/margins": 1.4014780521392822,
      "rewards/rejected": -2.555569648742676,
      "step": 11748
    },
    {
      "epoch": 1.5374335487036885,
      "grad_norm": 6.159215927124023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10124973207712173,
      "logits/rejected": -0.277902215719223,
      "logps/chosen": -613.8554077148438,
      "logps/rejected": -543.8582763671875,
      "loss": 0.3124,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1104872226715088,
      "rewards/margins": 1.4412113428115845,
      "rewards/rejected": -2.5516984462738037,
      "step": 11749
    },
    {
      "epoch": 1.5375644066410403,
      "grad_norm": 8.113093376159668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03513513505458832,
      "logits/rejected": -0.09307447820901871,
      "logps/chosen": -579.95263671875,
      "logps/rejected": -445.50396728515625,
      "loss": 0.3401,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4253332018852234,
      "rewards/margins": 1.504838466644287,
      "rewards/rejected": -1.9301716089248657,
      "step": 11750
    },
    {
      "epoch": 1.5376952645783921,
      "grad_norm": 6.264786720275879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3397100269794464,
      "logits/rejected": -0.3336833417415619,
      "logps/chosen": -343.07952880859375,
      "logps/rejected": -493.8343505859375,
      "loss": 0.3578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6086519360542297,
      "rewards/margins": 1.1380863189697266,
      "rewards/rejected": -1.746738314628601,
      "step": 11751
    },
    {
      "epoch": 1.5378261225157437,
      "grad_norm": 8.165822982788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21906954050064087,
      "logits/rejected": -0.27242615818977356,
      "logps/chosen": -390.59051513671875,
      "logps/rejected": -336.92724609375,
      "loss": 0.3876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33746564388275146,
      "rewards/margins": 1.817800521850586,
      "rewards/rejected": -2.155266284942627,
      "step": 11752
    },
    {
      "epoch": 1.5379569804530955,
      "grad_norm": 1.5166019201278687,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0812324658036232,
      "logits/rejected": -0.4150896668434143,
      "logps/chosen": -534.4842529296875,
      "logps/rejected": -488.8208923339844,
      "loss": 0.0402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.128190815448761,
      "rewards/margins": 4.127534866333008,
      "rewards/rejected": -4.255725383758545,
      "step": 11753
    },
    {
      "epoch": 1.5380878383904473,
      "grad_norm": 6.009057998657227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18030695617198944,
      "logits/rejected": -0.3366304636001587,
      "logps/chosen": -327.5458068847656,
      "logps/rejected": -429.3837585449219,
      "loss": 0.3152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20187567174434662,
      "rewards/margins": 1.7847154140472412,
      "rewards/rejected": -1.986591100692749,
      "step": 11754
    },
    {
      "epoch": 1.538218696327799,
      "grad_norm": 11.29564094543457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36039087176322937,
      "logits/rejected": -0.2611212432384491,
      "logps/chosen": -338.4638977050781,
      "logps/rejected": -325.7766418457031,
      "loss": 0.6629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0595225095748901,
      "rewards/margins": 0.7643435001373291,
      "rewards/rejected": -1.8238660097122192,
      "step": 11755
    },
    {
      "epoch": 1.5383495542651509,
      "grad_norm": 9.249007225036621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48012155294418335,
      "logits/rejected": -0.6036418080329895,
      "logps/chosen": -573.96435546875,
      "logps/rejected": -383.39129638671875,
      "loss": 0.4073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40647315979003906,
      "rewards/margins": 1.2097222805023193,
      "rewards/rejected": -1.6161953210830688,
      "step": 11756
    },
    {
      "epoch": 1.5384804122025026,
      "grad_norm": 8.833259582519531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3438912630081177,
      "logits/rejected": -0.4941357374191284,
      "logps/chosen": -272.3607482910156,
      "logps/rejected": -306.8870544433594,
      "loss": 0.5288,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0494611263275146,
      "rewards/margins": 1.4020137786865234,
      "rewards/rejected": -2.451475143432617,
      "step": 11757
    },
    {
      "epoch": 1.5386112701398544,
      "grad_norm": 11.261072158813477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28941670060157776,
      "logits/rejected": -0.3664443790912628,
      "logps/chosen": -526.0216674804688,
      "logps/rejected": -579.2703857421875,
      "loss": 0.629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5283797979354858,
      "rewards/margins": 1.1058789491653442,
      "rewards/rejected": -1.6342586278915405,
      "step": 11758
    },
    {
      "epoch": 1.5387421280772062,
      "grad_norm": 5.553977012634277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03348918631672859,
      "logits/rejected": 0.015051346272230148,
      "logps/chosen": -484.540283203125,
      "logps/rejected": -511.16546630859375,
      "loss": 0.2038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.563190221786499,
      "rewards/margins": 1.7904032468795776,
      "rewards/rejected": -2.353593587875366,
      "step": 11759
    },
    {
      "epoch": 1.538872986014558,
      "grad_norm": 13.624106407165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12251383066177368,
      "logits/rejected": -0.01429799199104309,
      "logps/chosen": -477.4931945800781,
      "logps/rejected": -444.966064453125,
      "loss": 0.8984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2108780145645142,
      "rewards/margins": 0.33712828159332275,
      "rewards/rejected": -1.548006296157837,
      "step": 11760
    },
    {
      "epoch": 1.5390038439519098,
      "grad_norm": 6.027246475219727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35392171144485474,
      "logits/rejected": -0.29554659128189087,
      "logps/chosen": -424.63262939453125,
      "logps/rejected": -434.95001220703125,
      "loss": 0.3572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45592498779296875,
      "rewards/margins": 1.071022868156433,
      "rewards/rejected": -1.5269478559494019,
      "step": 11761
    },
    {
      "epoch": 1.5391347018892616,
      "grad_norm": 12.261143684387207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36424869298934937,
      "logits/rejected": -0.3082128167152405,
      "logps/chosen": -304.336181640625,
      "logps/rejected": -552.5325317382812,
      "loss": 0.5776,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4102516174316406,
      "rewards/margins": 1.6859890222549438,
      "rewards/rejected": -2.096240758895874,
      "step": 11762
    },
    {
      "epoch": 1.5392655598266134,
      "grad_norm": 14.46611213684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0373319536447525,
      "logits/rejected": -0.029533803462982178,
      "logps/chosen": -343.81671142578125,
      "logps/rejected": -395.1336975097656,
      "loss": 0.78,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9126238226890564,
      "rewards/margins": 0.9258992671966553,
      "rewards/rejected": -1.838523030281067,
      "step": 11763
    },
    {
      "epoch": 1.539396417763965,
      "grad_norm": 8.15583324432373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07280707359313965,
      "logits/rejected": 0.09271153807640076,
      "logps/chosen": -461.53094482421875,
      "logps/rejected": -407.432373046875,
      "loss": 0.3977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08543853461742401,
      "rewards/margins": 1.2212131023406982,
      "rewards/rejected": -1.3066517114639282,
      "step": 11764
    },
    {
      "epoch": 1.5395272757013168,
      "grad_norm": 5.422792911529541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3854564130306244,
      "logits/rejected": -0.47207996249198914,
      "logps/chosen": -310.6376953125,
      "logps/rejected": -474.06414794921875,
      "loss": 0.2495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7772518992424011,
      "rewards/margins": 1.9597121477127075,
      "rewards/rejected": -2.736964225769043,
      "step": 11765
    },
    {
      "epoch": 1.5396581336386685,
      "grad_norm": 7.930381774902344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3693491518497467,
      "logits/rejected": -0.276762455701828,
      "logps/chosen": -470.66766357421875,
      "logps/rejected": -361.4349060058594,
      "loss": 0.4752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9153953194618225,
      "rewards/margins": 0.9477434754371643,
      "rewards/rejected": -1.8631389141082764,
      "step": 11766
    },
    {
      "epoch": 1.5397889915760203,
      "grad_norm": 8.742259979248047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01156916469335556,
      "logits/rejected": -0.08076023310422897,
      "logps/chosen": -439.22320556640625,
      "logps/rejected": -357.35809326171875,
      "loss": 0.5001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9422668218612671,
      "rewards/margins": 0.8473271131515503,
      "rewards/rejected": -1.789594054222107,
      "step": 11767
    },
    {
      "epoch": 1.539919849513372,
      "grad_norm": 5.541027069091797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2913268506526947,
      "logits/rejected": -0.07901091873645782,
      "logps/chosen": -248.6782989501953,
      "logps/rejected": -313.6529846191406,
      "loss": 0.3868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7348434329032898,
      "rewards/margins": 1.2476882934570312,
      "rewards/rejected": -1.9825316667556763,
      "step": 11768
    },
    {
      "epoch": 1.5400507074507237,
      "grad_norm": 10.197271347045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012880437076091766,
      "logits/rejected": -0.011375520378351212,
      "logps/chosen": -374.92333984375,
      "logps/rejected": -501.0150451660156,
      "loss": 0.5969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1279369592666626,
      "rewards/margins": 0.626905083656311,
      "rewards/rejected": -1.7548420429229736,
      "step": 11769
    },
    {
      "epoch": 1.5401815653880755,
      "grad_norm": 6.499518394470215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11096373200416565,
      "logits/rejected": -0.0664905458688736,
      "logps/chosen": -345.09173583984375,
      "logps/rejected": -376.6387634277344,
      "loss": 0.3533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4644155204296112,
      "rewards/margins": 1.409883737564087,
      "rewards/rejected": -1.8742992877960205,
      "step": 11770
    },
    {
      "epoch": 1.5403124233254273,
      "grad_norm": 9.373313903808594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24767956137657166,
      "logits/rejected": -0.30134767293930054,
      "logps/chosen": -423.395751953125,
      "logps/rejected": -531.6499633789062,
      "loss": 0.3274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7608201503753662,
      "rewards/margins": 1.975221872329712,
      "rewards/rejected": -2.7360422611236572,
      "step": 11771
    },
    {
      "epoch": 1.540443281262779,
      "grad_norm": 4.922432899475098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018847841769456863,
      "logits/rejected": -0.2557433247566223,
      "logps/chosen": -461.4790954589844,
      "logps/rejected": -412.572509765625,
      "loss": 0.2312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.019521713256836,
      "rewards/margins": 2.2606711387634277,
      "rewards/rejected": -3.2801928520202637,
      "step": 11772
    },
    {
      "epoch": 1.5405741392001309,
      "grad_norm": 8.232595443725586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15146596729755402,
      "logits/rejected": -0.445365309715271,
      "logps/chosen": -524.2896728515625,
      "logps/rejected": -446.3070373535156,
      "loss": 0.3864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8894189596176147,
      "rewards/margins": 1.5720151662826538,
      "rewards/rejected": -2.4614338874816895,
      "step": 11773
    },
    {
      "epoch": 1.5407049971374827,
      "grad_norm": 9.553354263305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28381896018981934,
      "logits/rejected": -0.05734247714281082,
      "logps/chosen": -366.2085266113281,
      "logps/rejected": -479.7220458984375,
      "loss": 0.6203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8339025974273682,
      "rewards/margins": 0.34898316860198975,
      "rewards/rejected": -2.1828856468200684,
      "step": 11774
    },
    {
      "epoch": 1.5408358550748344,
      "grad_norm": 7.304050445556641,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07058791816234589,
      "logits/rejected": 0.07114240527153015,
      "logps/chosen": -323.7639465332031,
      "logps/rejected": -389.24896240234375,
      "loss": 0.368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8280745148658752,
      "rewards/margins": 2.0695457458496094,
      "rewards/rejected": -2.89762020111084,
      "step": 11775
    },
    {
      "epoch": 1.5409667130121862,
      "grad_norm": 8.484997749328613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38739970326423645,
      "logits/rejected": -0.2598091959953308,
      "logps/chosen": -508.9452819824219,
      "logps/rejected": -412.1298828125,
      "loss": 0.4687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.537156343460083,
      "rewards/margins": 0.8079736828804016,
      "rewards/rejected": -2.345130205154419,
      "step": 11776
    },
    {
      "epoch": 1.541097570949538,
      "grad_norm": 6.636539459228516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41524451971054077,
      "logits/rejected": -0.10716450959444046,
      "logps/chosen": -547.9401245117188,
      "logps/rejected": -558.4658203125,
      "loss": 0.3416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9620965719223022,
      "rewards/margins": 1.1376469135284424,
      "rewards/rejected": -2.099743604660034,
      "step": 11777
    },
    {
      "epoch": 1.5412284288868898,
      "grad_norm": 6.316859245300293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17511361837387085,
      "logits/rejected": -0.2674728035926819,
      "logps/chosen": -400.4216003417969,
      "logps/rejected": -354.73419189453125,
      "loss": 0.4552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2852306365966797,
      "rewards/margins": 0.9463762044906616,
      "rewards/rejected": -1.2316068410873413,
      "step": 11778
    },
    {
      "epoch": 1.5413592868242414,
      "grad_norm": 13.616976737976074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0906810536980629,
      "logits/rejected": -0.31429117918014526,
      "logps/chosen": -433.50640869140625,
      "logps/rejected": -508.3980712890625,
      "loss": 0.7466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.420080304145813,
      "rewards/margins": 0.46986123919487,
      "rewards/rejected": -1.8899415731430054,
      "step": 11779
    },
    {
      "epoch": 1.5414901447615932,
      "grad_norm": 4.087602138519287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15426911413669586,
      "logits/rejected": -0.22535362839698792,
      "logps/chosen": -523.6172485351562,
      "logps/rejected": -464.2018127441406,
      "loss": 0.164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09171447157859802,
      "rewards/margins": 2.4906363487243652,
      "rewards/rejected": -2.582350730895996,
      "step": 11780
    },
    {
      "epoch": 1.541621002698945,
      "grad_norm": 6.761826038360596,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07237832248210907,
      "logits/rejected": -0.12392366677522659,
      "logps/chosen": -446.7874755859375,
      "logps/rejected": -572.76025390625,
      "loss": 0.3631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.842376708984375,
      "rewards/margins": 1.6882601976394653,
      "rewards/rejected": -2.53063702583313,
      "step": 11781
    },
    {
      "epoch": 1.5417518606362968,
      "grad_norm": 8.924034118652344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21503834426403046,
      "logits/rejected": -0.3724818527698517,
      "logps/chosen": -497.1584777832031,
      "logps/rejected": -442.6839294433594,
      "loss": 0.5246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4734207093715668,
      "rewards/margins": 0.7591197490692139,
      "rewards/rejected": -1.232540488243103,
      "step": 11782
    },
    {
      "epoch": 1.5418827185736483,
      "grad_norm": 11.227324485778809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4843335747718811,
      "logits/rejected": -0.5299147367477417,
      "logps/chosen": -430.2724304199219,
      "logps/rejected": -577.9021606445312,
      "loss": 0.562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8957868814468384,
      "rewards/margins": 0.4786894619464874,
      "rewards/rejected": -2.374476194381714,
      "step": 11783
    },
    {
      "epoch": 1.5420135765110001,
      "grad_norm": 6.361134052276611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09691626578569412,
      "logits/rejected": -0.32776975631713867,
      "logps/chosen": -389.69281005859375,
      "logps/rejected": -468.5010986328125,
      "loss": 0.4256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7498452663421631,
      "rewards/margins": 1.6684856414794922,
      "rewards/rejected": -2.4183309078216553,
      "step": 11784
    },
    {
      "epoch": 1.542144434448352,
      "grad_norm": 3.5696191787719727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05632784962654114,
      "logits/rejected": -0.5611593723297119,
      "logps/chosen": -295.44598388671875,
      "logps/rejected": -392.54632568359375,
      "loss": 0.2677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1434742957353592,
      "rewards/margins": 2.1584413051605225,
      "rewards/rejected": -2.0149669647216797,
      "step": 11785
    },
    {
      "epoch": 1.5422752923857037,
      "grad_norm": 7.088153839111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5833381414413452,
      "logits/rejected": -0.4355360269546509,
      "logps/chosen": -401.095703125,
      "logps/rejected": -477.0410461425781,
      "loss": 0.4885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5614349246025085,
      "rewards/margins": 0.9636993408203125,
      "rewards/rejected": -1.5251343250274658,
      "step": 11786
    },
    {
      "epoch": 1.5424061503230555,
      "grad_norm": 5.784818172454834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3811444938182831,
      "logits/rejected": -0.33959585428237915,
      "logps/chosen": -373.46875,
      "logps/rejected": -337.1436462402344,
      "loss": 0.2992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6683538556098938,
      "rewards/margins": 1.8641345500946045,
      "rewards/rejected": -2.5324885845184326,
      "step": 11787
    },
    {
      "epoch": 1.5425370082604073,
      "grad_norm": 14.244742393493652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1631484031677246,
      "logits/rejected": -0.26106125116348267,
      "logps/chosen": -287.405029296875,
      "logps/rejected": -369.4681396484375,
      "loss": 0.7903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6817600727081299,
      "rewards/margins": 0.3664487600326538,
      "rewards/rejected": -2.0482089519500732,
      "step": 11788
    },
    {
      "epoch": 1.542667866197759,
      "grad_norm": 13.570405006408691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022470414638519287,
      "logits/rejected": -0.22987724840641022,
      "logps/chosen": -634.92236328125,
      "logps/rejected": -403.7550048828125,
      "loss": 0.645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06295987963676453,
      "rewards/margins": 0.953340470790863,
      "rewards/rejected": -1.0163004398345947,
      "step": 11789
    },
    {
      "epoch": 1.5427987241351109,
      "grad_norm": 13.784459114074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2925986647605896,
      "logits/rejected": -0.5750119090080261,
      "logps/chosen": -620.0905151367188,
      "logps/rejected": -435.2958679199219,
      "loss": 0.7281,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1370480060577393,
      "rewards/margins": 0.463204562664032,
      "rewards/rejected": -1.600252628326416,
      "step": 11790
    },
    {
      "epoch": 1.5429295820724627,
      "grad_norm": 7.109170436859131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24725504219532013,
      "logits/rejected": -0.05704083293676376,
      "logps/chosen": -329.4464111328125,
      "logps/rejected": -462.9591064453125,
      "loss": 0.4249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5090436935424805,
      "rewards/margins": 1.260049819946289,
      "rewards/rejected": -2.7690932750701904,
      "step": 11791
    },
    {
      "epoch": 1.5430604400098145,
      "grad_norm": 6.43118143081665,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25207439064979553,
      "logits/rejected": -0.4931257367134094,
      "logps/chosen": -304.7176818847656,
      "logps/rejected": -281.2960205078125,
      "loss": 0.3185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7897443175315857,
      "rewards/margins": 1.2626960277557373,
      "rewards/rejected": -2.052440643310547,
      "step": 11792
    },
    {
      "epoch": 1.5431912979471663,
      "grad_norm": 9.320453643798828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19288432598114014,
      "logits/rejected": -0.20807307958602905,
      "logps/chosen": -408.9031982421875,
      "logps/rejected": -404.5052185058594,
      "loss": 0.4288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.230432152748108,
      "rewards/margins": 1.2751966714859009,
      "rewards/rejected": -2.5056285858154297,
      "step": 11793
    },
    {
      "epoch": 1.5433221558845178,
      "grad_norm": 7.489055633544922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0988364964723587,
      "logits/rejected": -0.26853737235069275,
      "logps/chosen": -404.0513916015625,
      "logps/rejected": -477.6540222167969,
      "loss": 0.3574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08133470267057419,
      "rewards/margins": 1.198657512664795,
      "rewards/rejected": -1.2799921035766602,
      "step": 11794
    },
    {
      "epoch": 1.5434530138218696,
      "grad_norm": 10.32402515411377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02638925611972809,
      "logits/rejected": -0.06537289917469025,
      "logps/chosen": -407.33758544921875,
      "logps/rejected": -417.22344970703125,
      "loss": 0.5067,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5124462246894836,
      "rewards/margins": 1.1105761528015137,
      "rewards/rejected": -1.623022437095642,
      "step": 11795
    },
    {
      "epoch": 1.5435838717592214,
      "grad_norm": 10.745455741882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.460740864276886,
      "logits/rejected": -0.4864920973777771,
      "logps/chosen": -478.35211181640625,
      "logps/rejected": -464.7675476074219,
      "loss": 0.5089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7528513669967651,
      "rewards/margins": 1.5678800344467163,
      "rewards/rejected": -2.3207314014434814,
      "step": 11796
    },
    {
      "epoch": 1.5437147296965732,
      "grad_norm": 4.942568778991699,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04145513474941254,
      "logits/rejected": -0.13715949654579163,
      "logps/chosen": -720.939453125,
      "logps/rejected": -591.3849487304688,
      "loss": 0.157,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15166814625263214,
      "rewards/margins": 2.453599214553833,
      "rewards/rejected": -2.605267286300659,
      "step": 11797
    },
    {
      "epoch": 1.5438455876339248,
      "grad_norm": 2.5846664905548096,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029329154640436172,
      "logits/rejected": -0.1719505488872528,
      "logps/chosen": -351.2010803222656,
      "logps/rejected": -454.8662414550781,
      "loss": 0.0653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6001749038696289,
      "rewards/margins": 3.258084297180176,
      "rewards/rejected": -2.6579089164733887,
      "step": 11798
    },
    {
      "epoch": 1.5439764455712766,
      "grad_norm": 12.642208099365234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05689218267798424,
      "logits/rejected": 0.11148504912853241,
      "logps/chosen": -775.9775390625,
      "logps/rejected": -643.04248046875,
      "loss": 0.5276,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3957222104072571,
      "rewards/margins": 1.066771388053894,
      "rewards/rejected": -1.4624935388565063,
      "step": 11799
    },
    {
      "epoch": 1.5441073035086283,
      "grad_norm": 5.329172611236572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1978752613067627,
      "logits/rejected": -0.20008543133735657,
      "logps/chosen": -395.2703857421875,
      "logps/rejected": -610.0342407226562,
      "loss": 0.1428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1532481908798218,
      "rewards/margins": 3.002549409866333,
      "rewards/rejected": -4.155797958374023,
      "step": 11800
    },
    {
      "epoch": 1.5442381614459801,
      "grad_norm": 7.848710536956787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21744881570339203,
      "logits/rejected": -0.211663618683815,
      "logps/chosen": -489.3565368652344,
      "logps/rejected": -420.4066162109375,
      "loss": 0.4488,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5092424750328064,
      "rewards/margins": 1.6277503967285156,
      "rewards/rejected": -2.136992931365967,
      "step": 11801
    },
    {
      "epoch": 1.544369019383332,
      "grad_norm": 7.182619571685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28000158071517944,
      "logits/rejected": -0.3327515721321106,
      "logps/chosen": -798.60595703125,
      "logps/rejected": -475.3197326660156,
      "loss": 0.4406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25504112243652344,
      "rewards/margins": 1.8636751174926758,
      "rewards/rejected": -2.1187164783477783,
      "step": 11802
    },
    {
      "epoch": 1.5444998773206837,
      "grad_norm": 13.045055389404297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1289096623659134,
      "logits/rejected": -0.17491775751113892,
      "logps/chosen": -560.1038818359375,
      "logps/rejected": -511.7728576660156,
      "loss": 0.4564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6657853126525879,
      "rewards/margins": 1.9383366107940674,
      "rewards/rejected": -2.6041221618652344,
      "step": 11803
    },
    {
      "epoch": 1.5446307352580355,
      "grad_norm": 8.741034507751465,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026881935074925423,
      "logits/rejected": 0.06688500195741653,
      "logps/chosen": -488.2901306152344,
      "logps/rejected": -548.2484130859375,
      "loss": 0.5537,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.42222559452056885,
      "rewards/margins": 0.8408819437026978,
      "rewards/rejected": -1.2631076574325562,
      "step": 11804
    },
    {
      "epoch": 1.5447615931953873,
      "grad_norm": 7.398310661315918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05950460582971573,
      "logits/rejected": 0.0035495024640113115,
      "logps/chosen": -366.9576416015625,
      "logps/rejected": -395.95184326171875,
      "loss": 0.3772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5972976684570312,
      "rewards/margins": 1.444064974784851,
      "rewards/rejected": -2.041362762451172,
      "step": 11805
    },
    {
      "epoch": 1.544892451132739,
      "grad_norm": 9.031303405761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10426216572523117,
      "logits/rejected": -0.06258987635374069,
      "logps/chosen": -505.2252197265625,
      "logps/rejected": -397.8844299316406,
      "loss": 0.5623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.056257620453834534,
      "rewards/margins": 1.160248041152954,
      "rewards/rejected": -1.2165056467056274,
      "step": 11806
    },
    {
      "epoch": 1.5450233090700909,
      "grad_norm": 9.469277381896973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.255057156085968,
      "logits/rejected": -0.13387048244476318,
      "logps/chosen": -468.8890686035156,
      "logps/rejected": -503.01666259765625,
      "loss": 0.5159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8607099652290344,
      "rewards/margins": 1.208423137664795,
      "rewards/rejected": -2.0691332817077637,
      "step": 11807
    },
    {
      "epoch": 1.5451541670074427,
      "grad_norm": 10.150310516357422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15806740522384644,
      "logits/rejected": -0.1267528533935547,
      "logps/chosen": -351.7972106933594,
      "logps/rejected": -452.92230224609375,
      "loss": 0.456,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6980103850364685,
      "rewards/margins": 1.6094324588775635,
      "rewards/rejected": -2.3074429035186768,
      "step": 11808
    },
    {
      "epoch": 1.5452850249447945,
      "grad_norm": 6.826744556427002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17657513916492462,
      "logits/rejected": -0.14026854932308197,
      "logps/chosen": -360.6452331542969,
      "logps/rejected": -519.0316772460938,
      "loss": 0.302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9055566787719727,
      "rewards/margins": 1.3902326822280884,
      "rewards/rejected": -2.2957894802093506,
      "step": 11809
    },
    {
      "epoch": 1.545415882882146,
      "grad_norm": 15.402013778686523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2877032458782196,
      "logits/rejected": -0.23373450338840485,
      "logps/chosen": -518.2908935546875,
      "logps/rejected": -492.4351806640625,
      "loss": 0.9953,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0077896118164062,
      "rewards/margins": 0.15827959775924683,
      "rewards/rejected": -1.1660692691802979,
      "step": 11810
    },
    {
      "epoch": 1.5455467408194978,
      "grad_norm": 5.477260112762451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11344592273235321,
      "logits/rejected": -0.22967541217803955,
      "logps/chosen": -479.6390686035156,
      "logps/rejected": -468.8987121582031,
      "loss": 0.2783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2271118015050888,
      "rewards/margins": 1.9087321758270264,
      "rewards/rejected": -1.6816203594207764,
      "step": 11811
    },
    {
      "epoch": 1.5456775987568496,
      "grad_norm": 9.03407096862793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.022610999643802643,
      "logits/rejected": 0.049467749893665314,
      "logps/chosen": -758.8860473632812,
      "logps/rejected": -789.8401489257812,
      "loss": 0.3092,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6940454244613647,
      "rewards/margins": 1.6288682222366333,
      "rewards/rejected": -2.322913646697998,
      "step": 11812
    },
    {
      "epoch": 1.5458084566942012,
      "grad_norm": 4.975639343261719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060661785304546356,
      "logits/rejected": -0.3026546537876129,
      "logps/chosen": -544.5433349609375,
      "logps/rejected": -629.5509033203125,
      "loss": 0.1673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0092085599899292,
      "rewards/margins": 2.814587116241455,
      "rewards/rejected": -3.823795795440674,
      "step": 11813
    },
    {
      "epoch": 1.545939314631553,
      "grad_norm": 11.033544540405273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19710254669189453,
      "logits/rejected": -0.3445395827293396,
      "logps/chosen": -542.9017333984375,
      "logps/rejected": -556.077392578125,
      "loss": 0.5379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5210264325141907,
      "rewards/margins": 0.7314373254776001,
      "rewards/rejected": -1.2524638175964355,
      "step": 11814
    },
    {
      "epoch": 1.5460701725689048,
      "grad_norm": 7.9636616706848145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3669124245643616,
      "logits/rejected": 0.1430739313364029,
      "logps/chosen": -389.1146240234375,
      "logps/rejected": -429.1841125488281,
      "loss": 0.3117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8361994028091431,
      "rewards/margins": 2.116211414337158,
      "rewards/rejected": -2.9524106979370117,
      "step": 11815
    },
    {
      "epoch": 1.5462010305062566,
      "grad_norm": 4.775080680847168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38511207699775696,
      "logits/rejected": -0.281222939491272,
      "logps/chosen": -551.1119384765625,
      "logps/rejected": -465.73486328125,
      "loss": 0.2203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35161933302879333,
      "rewards/margins": 1.5393905639648438,
      "rewards/rejected": -1.89100980758667,
      "step": 11816
    },
    {
      "epoch": 1.5463318884436084,
      "grad_norm": 14.381852149963379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3750591576099396,
      "logits/rejected": -0.332269549369812,
      "logps/chosen": -514.5289306640625,
      "logps/rejected": -494.59881591796875,
      "loss": 0.5363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7656774520874023,
      "rewards/margins": 1.0423471927642822,
      "rewards/rejected": -1.8080246448516846,
      "step": 11817
    },
    {
      "epoch": 1.5464627463809602,
      "grad_norm": 3.1173439025878906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1644340455532074,
      "logits/rejected": -0.0028741881251335144,
      "logps/chosen": -532.4426879882812,
      "logps/rejected": -471.95562744140625,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07594811171293259,
      "rewards/margins": 3.0473971366882324,
      "rewards/rejected": -2.9714486598968506,
      "step": 11818
    },
    {
      "epoch": 1.546593604318312,
      "grad_norm": 6.208540439605713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32472822070121765,
      "logits/rejected": -0.5586031675338745,
      "logps/chosen": -282.9635314941406,
      "logps/rejected": -324.2495422363281,
      "loss": 0.4036,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2477972507476807,
      "rewards/margins": 1.3003681898117065,
      "rewards/rejected": -2.5481650829315186,
      "step": 11819
    },
    {
      "epoch": 1.5467244622556637,
      "grad_norm": 5.721954822540283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2668524384498596,
      "logits/rejected": -0.24064895510673523,
      "logps/chosen": -437.94287109375,
      "logps/rejected": -562.7869873046875,
      "loss": 0.2815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34159624576568604,
      "rewards/margins": 2.236926555633545,
      "rewards/rejected": -2.5785224437713623,
      "step": 11820
    },
    {
      "epoch": 1.5468553201930155,
      "grad_norm": 12.432527542114258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08124378323554993,
      "logits/rejected": -0.2115892618894577,
      "logps/chosen": -559.2838745117188,
      "logps/rejected": -538.5942993164062,
      "loss": 0.7119,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5681279897689819,
      "rewards/margins": 0.7967026233673096,
      "rewards/rejected": -1.3648306131362915,
      "step": 11821
    },
    {
      "epoch": 1.5469861781303673,
      "grad_norm": 11.323040008544922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5386614203453064,
      "logits/rejected": -0.3731077313423157,
      "logps/chosen": -470.82025146484375,
      "logps/rejected": -721.7680053710938,
      "loss": 0.3743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6943438053131104,
      "rewards/margins": 1.415751338005066,
      "rewards/rejected": -3.110095262527466,
      "step": 11822
    },
    {
      "epoch": 1.547117036067719,
      "grad_norm": 10.27980899810791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18366935849189758,
      "logits/rejected": 0.041134100407361984,
      "logps/chosen": -449.36029052734375,
      "logps/rejected": -506.1306457519531,
      "loss": 0.4114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8288557529449463,
      "rewards/margins": 2.033782720565796,
      "rewards/rejected": -2.862638473510742,
      "step": 11823
    },
    {
      "epoch": 1.547247894005071,
      "grad_norm": 12.320860862731934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10622886568307877,
      "logits/rejected": -0.2710324823856354,
      "logps/chosen": -419.3905029296875,
      "logps/rejected": -412.552001953125,
      "loss": 0.6237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5557736158370972,
      "rewards/margins": 1.1266367435455322,
      "rewards/rejected": -1.6824102401733398,
      "step": 11824
    },
    {
      "epoch": 1.5473787519424225,
      "grad_norm": 8.368949890136719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25044381618499756,
      "logits/rejected": -0.2846711575984955,
      "logps/chosen": -468.03887939453125,
      "logps/rejected": -384.6031494140625,
      "loss": 0.3918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5811500549316406,
      "rewards/margins": 1.2035802602767944,
      "rewards/rejected": -1.7847304344177246,
      "step": 11825
    },
    {
      "epoch": 1.5475096098797743,
      "grad_norm": 8.276651382446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3519314229488373,
      "logits/rejected": -0.2707386612892151,
      "logps/chosen": -496.52716064453125,
      "logps/rejected": -445.0450744628906,
      "loss": 0.4415,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9646785855293274,
      "rewards/margins": 0.9239345788955688,
      "rewards/rejected": -1.888612985610962,
      "step": 11826
    },
    {
      "epoch": 1.547640467817126,
      "grad_norm": 7.798339366912842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48785024881362915,
      "logits/rejected": -0.27848508954048157,
      "logps/chosen": -368.36883544921875,
      "logps/rejected": -444.1563720703125,
      "loss": 0.2626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5233070254325867,
      "rewards/margins": 2.6792445182800293,
      "rewards/rejected": -3.2025511264801025,
      "step": 11827
    },
    {
      "epoch": 1.5477713257544778,
      "grad_norm": 7.775847434997559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33330509066581726,
      "logits/rejected": -0.3431198000907898,
      "logps/chosen": -471.9460144042969,
      "logps/rejected": -503.4376220703125,
      "loss": 0.2122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4574720561504364,
      "rewards/margins": 2.7795090675354004,
      "rewards/rejected": -2.3220372200012207,
      "step": 11828
    },
    {
      "epoch": 1.5479021836918294,
      "grad_norm": 10.651314735412598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34801262617111206,
      "logits/rejected": -0.3017929196357727,
      "logps/chosen": -437.7444152832031,
      "logps/rejected": -383.531005859375,
      "loss": 0.5244,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9057846665382385,
      "rewards/margins": 0.855573832988739,
      "rewards/rejected": -1.7613584995269775,
      "step": 11829
    },
    {
      "epoch": 1.5480330416291812,
      "grad_norm": 9.135900497436523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18512433767318726,
      "logits/rejected": -0.26457327604293823,
      "logps/chosen": -411.1954650878906,
      "logps/rejected": -451.77825927734375,
      "loss": 0.4866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3021855354309082,
      "rewards/margins": 1.5057597160339355,
      "rewards/rejected": -2.8079450130462646,
      "step": 11830
    },
    {
      "epoch": 1.548163899566533,
      "grad_norm": 5.17841911315918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08337685465812683,
      "logits/rejected": -0.04727349057793617,
      "logps/chosen": -407.6128845214844,
      "logps/rejected": -479.05462646484375,
      "loss": 0.1774,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.885042130947113,
      "rewards/margins": 2.767320156097412,
      "rewards/rejected": -3.65236234664917,
      "step": 11831
    },
    {
      "epoch": 1.5482947575038848,
      "grad_norm": 11.2608060836792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3246269226074219,
      "logits/rejected": -0.46527379751205444,
      "logps/chosen": -315.3450927734375,
      "logps/rejected": -375.9915466308594,
      "loss": 0.7685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.471657633781433,
      "rewards/margins": 1.5697994232177734,
      "rewards/rejected": -3.041457176208496,
      "step": 11832
    },
    {
      "epoch": 1.5484256154412366,
      "grad_norm": 12.308732986450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3532375693321228,
      "logits/rejected": -0.2042771279811859,
      "logps/chosen": -577.2870483398438,
      "logps/rejected": -570.6929931640625,
      "loss": 0.6331,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2218252420425415,
      "rewards/margins": 0.9491140842437744,
      "rewards/rejected": -2.1709394454956055,
      "step": 11833
    },
    {
      "epoch": 1.5485564733785884,
      "grad_norm": 13.344718933105469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0691853016614914,
      "logits/rejected": 0.22131720185279846,
      "logps/chosen": -383.89337158203125,
      "logps/rejected": -401.78570556640625,
      "loss": 0.7972,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4580944776535034,
      "rewards/margins": 0.20457784831523895,
      "rewards/rejected": -1.6626722812652588,
      "step": 11834
    },
    {
      "epoch": 1.5486873313159402,
      "grad_norm": 8.925596237182617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5471113920211792,
      "logits/rejected": -0.39567357301712036,
      "logps/chosen": -666.10986328125,
      "logps/rejected": -509.0091247558594,
      "loss": 0.4793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9883249998092651,
      "rewards/margins": 0.9538873434066772,
      "rewards/rejected": -1.9422123432159424,
      "step": 11835
    },
    {
      "epoch": 1.548818189253292,
      "grad_norm": 6.812777519226074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45443153381347656,
      "logits/rejected": -0.36699384450912476,
      "logps/chosen": -595.7928466796875,
      "logps/rejected": -542.9131469726562,
      "loss": 0.366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23568134009838104,
      "rewards/margins": 1.636994481086731,
      "rewards/rejected": -1.872675895690918,
      "step": 11836
    },
    {
      "epoch": 1.5489490471906437,
      "grad_norm": 14.283127784729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2574228048324585,
      "logits/rejected": -0.2901563048362732,
      "logps/chosen": -694.8050537109375,
      "logps/rejected": -560.76416015625,
      "loss": 0.81,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.058136224746704,
      "rewards/margins": 0.7080478668212891,
      "rewards/rejected": -1.7661839723587036,
      "step": 11837
    },
    {
      "epoch": 1.5490799051279955,
      "grad_norm": 8.982614517211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08580826222896576,
      "logits/rejected": 0.05475812032818794,
      "logps/chosen": -431.83624267578125,
      "logps/rejected": -551.4105834960938,
      "loss": 0.2141,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.054184332489967346,
      "rewards/margins": 2.528501033782959,
      "rewards/rejected": -2.5826852321624756,
      "step": 11838
    },
    {
      "epoch": 1.5492107630653473,
      "grad_norm": 10.127111434936523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25531643629074097,
      "logits/rejected": -0.11418326199054718,
      "logps/chosen": -366.7899169921875,
      "logps/rejected": -396.32330322265625,
      "loss": 0.4539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.97566157579422,
      "rewards/margins": 1.3016682863235474,
      "rewards/rejected": -2.277329444885254,
      "step": 11839
    },
    {
      "epoch": 1.549341621002699,
      "grad_norm": 8.649179458618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1773046851158142,
      "logits/rejected": -0.22558963298797607,
      "logps/chosen": -554.59130859375,
      "logps/rejected": -456.1493835449219,
      "loss": 0.4588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4774520993232727,
      "rewards/margins": 1.3000059127807617,
      "rewards/rejected": -1.7774579524993896,
      "step": 11840
    },
    {
      "epoch": 1.5494724789400507,
      "grad_norm": 6.0553460121154785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06772959232330322,
      "logits/rejected": -0.09237617254257202,
      "logps/chosen": -647.3595581054688,
      "logps/rejected": -428.6043701171875,
      "loss": 0.2788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18166354298591614,
      "rewards/margins": 2.40014386177063,
      "rewards/rejected": -2.5818073749542236,
      "step": 11841
    },
    {
      "epoch": 1.5496033368774025,
      "grad_norm": 7.802538871765137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.057669974863529205,
      "logits/rejected": -0.08476939797401428,
      "logps/chosen": -654.7880249023438,
      "logps/rejected": -508.9088439941406,
      "loss": 0.3845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1014656126499176,
      "rewards/margins": 1.7072803974151611,
      "rewards/rejected": -1.8087459802627563,
      "step": 11842
    },
    {
      "epoch": 1.5497341948147543,
      "grad_norm": 10.603132247924805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11016812920570374,
      "logits/rejected": -0.15757721662521362,
      "logps/chosen": -334.61126708984375,
      "logps/rejected": -295.55181884765625,
      "loss": 0.8092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2437355518341064,
      "rewards/margins": 0.9605162739753723,
      "rewards/rejected": -2.204251766204834,
      "step": 11843
    },
    {
      "epoch": 1.5498650527521058,
      "grad_norm": 4.0359601974487305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043387897312641144,
      "logits/rejected": -0.16645115613937378,
      "logps/chosen": -440.72625732421875,
      "logps/rejected": -418.8228759765625,
      "loss": 0.1145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.275040864944458,
      "rewards/margins": 2.5314722061157227,
      "rewards/rejected": -3.8065133094787598,
      "step": 11844
    },
    {
      "epoch": 1.5499959106894576,
      "grad_norm": 11.023159980773926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19323354959487915,
      "logits/rejected": 0.14310704171657562,
      "logps/chosen": -437.0790100097656,
      "logps/rejected": -470.030029296875,
      "loss": 0.3992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6586271524429321,
      "rewards/margins": 2.447831153869629,
      "rewards/rejected": -3.1064584255218506,
      "step": 11845
    },
    {
      "epoch": 1.5501267686268094,
      "grad_norm": 7.135905742645264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0849791020154953,
      "logits/rejected": -0.18847070634365082,
      "logps/chosen": -460.3893737792969,
      "logps/rejected": -421.8215026855469,
      "loss": 0.2948,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5987162590026855,
      "rewards/margins": 2.076958656311035,
      "rewards/rejected": -2.6756749153137207,
      "step": 11846
    },
    {
      "epoch": 1.5502576265641612,
      "grad_norm": 5.795318603515625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5129287242889404,
      "logits/rejected": -0.6462838649749756,
      "logps/chosen": -256.4830017089844,
      "logps/rejected": -367.0495300292969,
      "loss": 0.3218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1831103563308716,
      "rewards/margins": 1.9724799394607544,
      "rewards/rejected": -3.155590295791626,
      "step": 11847
    },
    {
      "epoch": 1.550388484501513,
      "grad_norm": 8.633419036865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3038369119167328,
      "logits/rejected": -0.2136758267879486,
      "logps/chosen": -436.62005615234375,
      "logps/rejected": -588.3025512695312,
      "loss": 0.4056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.376648187637329,
      "rewards/margins": 1.038297176361084,
      "rewards/rejected": -2.414945125579834,
      "step": 11848
    },
    {
      "epoch": 1.5505193424388648,
      "grad_norm": 10.826766014099121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17923201620578766,
      "logits/rejected": -0.2142009139060974,
      "logps/chosen": -527.7324829101562,
      "logps/rejected": -424.27386474609375,
      "loss": 0.5169,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4149236679077148,
      "rewards/margins": 1.2613203525543213,
      "rewards/rejected": -2.6762442588806152,
      "step": 11849
    },
    {
      "epoch": 1.5506502003762166,
      "grad_norm": 5.137434959411621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3784940540790558,
      "logits/rejected": -0.22495217621326447,
      "logps/chosen": -321.897216796875,
      "logps/rejected": -405.955078125,
      "loss": 0.2943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25013819336891174,
      "rewards/margins": 1.9371695518493652,
      "rewards/rejected": -2.187307834625244,
      "step": 11850
    },
    {
      "epoch": 1.5507810583135684,
      "grad_norm": 7.91735315322876,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5854889750480652,
      "logits/rejected": -0.3634740114212036,
      "logps/chosen": -330.31390380859375,
      "logps/rejected": -363.73443603515625,
      "loss": 0.5843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4650682210922241,
      "rewards/margins": 0.6426987051963806,
      "rewards/rejected": -2.107766628265381,
      "step": 11851
    },
    {
      "epoch": 1.5509119162509202,
      "grad_norm": 12.956809997558594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.289703369140625,
      "logits/rejected": -0.2849617004394531,
      "logps/chosen": -535.29443359375,
      "logps/rejected": -432.9600830078125,
      "loss": 0.575,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8161447048187256,
      "rewards/margins": 0.6805577278137207,
      "rewards/rejected": -1.4967024326324463,
      "step": 11852
    },
    {
      "epoch": 1.551042774188272,
      "grad_norm": 10.480276107788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2544044852256775,
      "logits/rejected": -0.23041942715644836,
      "logps/chosen": -593.4462280273438,
      "logps/rejected": -489.45831298828125,
      "loss": 0.3156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02115286886692047,
      "rewards/margins": 1.8998559713363647,
      "rewards/rejected": -1.8787033557891846,
      "step": 11853
    },
    {
      "epoch": 1.5511736321256238,
      "grad_norm": 12.581717491149902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33354708552360535,
      "logits/rejected": -0.15138179063796997,
      "logps/chosen": -452.0443420410156,
      "logps/rejected": -551.6170654296875,
      "loss": 0.652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.884979248046875,
      "rewards/margins": 0.8132677674293518,
      "rewards/rejected": -1.6982471942901611,
      "step": 11854
    },
    {
      "epoch": 1.5513044900629753,
      "grad_norm": 4.633265018463135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4017829895019531,
      "logits/rejected": -0.5177422761917114,
      "logps/chosen": -379.5216064453125,
      "logps/rejected": -432.6913146972656,
      "loss": 0.2706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03127002716064453,
      "rewards/margins": 2.2212870121002197,
      "rewards/rejected": -2.190016984939575,
      "step": 11855
    },
    {
      "epoch": 1.5514353480003271,
      "grad_norm": 11.786487579345703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49740463495254517,
      "logits/rejected": -0.39270538091659546,
      "logps/chosen": -436.3285217285156,
      "logps/rejected": -559.05322265625,
      "loss": 0.5662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.13199782371521,
      "rewards/margins": 0.9574499130249023,
      "rewards/rejected": -2.0894479751586914,
      "step": 11856
    },
    {
      "epoch": 1.551566205937679,
      "grad_norm": 7.12329626083374,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4205440878868103,
      "logits/rejected": -0.4237312078475952,
      "logps/chosen": -543.3856811523438,
      "logps/rejected": -459.75152587890625,
      "loss": 0.3316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09265746176242828,
      "rewards/margins": 1.3503600358963013,
      "rewards/rejected": -1.4430173635482788,
      "step": 11857
    },
    {
      "epoch": 1.5516970638750307,
      "grad_norm": 9.023384094238281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3316596746444702,
      "logits/rejected": -0.37551313638687134,
      "logps/chosen": -468.92852783203125,
      "logps/rejected": -397.5916748046875,
      "loss": 0.5178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.106095552444458,
      "rewards/margins": 0.9313517808914185,
      "rewards/rejected": -2.037447214126587,
      "step": 11858
    },
    {
      "epoch": 1.5518279218123823,
      "grad_norm": 4.164721488952637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12586486339569092,
      "logits/rejected": 0.20582927763462067,
      "logps/chosen": -301.90313720703125,
      "logps/rejected": -367.5335388183594,
      "loss": 0.1881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9352765679359436,
      "rewards/margins": 2.1446259021759033,
      "rewards/rejected": -3.079902410507202,
      "step": 11859
    },
    {
      "epoch": 1.551958779749734,
      "grad_norm": 9.09018325805664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0942978709936142,
      "logits/rejected": 0.11720924079418182,
      "logps/chosen": -432.62774658203125,
      "logps/rejected": -427.73138427734375,
      "loss": 0.3319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0967553853988647,
      "rewards/margins": 1.4627025127410889,
      "rewards/rejected": -2.559458017349243,
      "step": 11860
    },
    {
      "epoch": 1.5520896376870859,
      "grad_norm": 8.884352684020996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6120420694351196,
      "logits/rejected": -0.7270238399505615,
      "logps/chosen": -597.7791748046875,
      "logps/rejected": -532.0359497070312,
      "loss": 0.4454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2676022052764893,
      "rewards/margins": 1.474845051765442,
      "rewards/rejected": -2.7424476146698,
      "step": 11861
    },
    {
      "epoch": 1.5522204956244376,
      "grad_norm": 7.456673622131348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23141984641551971,
      "logits/rejected": -0.40022552013397217,
      "logps/chosen": -474.6331787109375,
      "logps/rejected": -393.2491149902344,
      "loss": 0.2917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.908812403678894,
      "rewards/margins": 2.0581724643707275,
      "rewards/rejected": -2.966984748840332,
      "step": 11862
    },
    {
      "epoch": 1.5523513535617894,
      "grad_norm": 8.693252563476562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13025271892547607,
      "logits/rejected": -0.23492005467414856,
      "logps/chosen": -377.90362548828125,
      "logps/rejected": -327.6646728515625,
      "loss": 0.6803,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8819915652275085,
      "rewards/margins": 0.24247132241725922,
      "rewards/rejected": -1.1244629621505737,
      "step": 11863
    },
    {
      "epoch": 1.5524822114991412,
      "grad_norm": 12.086291313171387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029196765273809433,
      "logits/rejected": -0.036042220890522,
      "logps/chosen": -629.8295288085938,
      "logps/rejected": -510.7327880859375,
      "loss": 0.5739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3108989000320435,
      "rewards/margins": 0.8200498819351196,
      "rewards/rejected": -2.130948543548584,
      "step": 11864
    },
    {
      "epoch": 1.552613069436493,
      "grad_norm": 7.12406587600708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08322281390428543,
      "logits/rejected": -0.24591653048992157,
      "logps/chosen": -407.1280517578125,
      "logps/rejected": -404.489013671875,
      "loss": 0.4462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9029781818389893,
      "rewards/margins": 1.858124852180481,
      "rewards/rejected": -2.7611026763916016,
      "step": 11865
    },
    {
      "epoch": 1.5527439273738448,
      "grad_norm": 10.266395568847656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3975178897380829,
      "logits/rejected": -0.4771432876586914,
      "logps/chosen": -327.2126159667969,
      "logps/rejected": -426.75927734375,
      "loss": 0.369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2411015033721924,
      "rewards/margins": 1.5369863510131836,
      "rewards/rejected": -2.778087615966797,
      "step": 11866
    },
    {
      "epoch": 1.5528747853111966,
      "grad_norm": 4.681978225708008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06440309435129166,
      "logits/rejected": -0.39288920164108276,
      "logps/chosen": -344.8052062988281,
      "logps/rejected": -345.9638977050781,
      "loss": 0.1745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.177324891090393,
      "rewards/margins": 2.4729902744293213,
      "rewards/rejected": -3.650315284729004,
      "step": 11867
    },
    {
      "epoch": 1.5530056432485484,
      "grad_norm": 13.195652961730957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2440914809703827,
      "logits/rejected": -0.379312664270401,
      "logps/chosen": -507.2090148925781,
      "logps/rejected": -485.905517578125,
      "loss": 0.681,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4545270204544067,
      "rewards/margins": 0.6666624546051025,
      "rewards/rejected": -2.121189594268799,
      "step": 11868
    },
    {
      "epoch": 1.5531365011859002,
      "grad_norm": 10.799479484558105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23460280895233154,
      "logits/rejected": -0.4950776696205139,
      "logps/chosen": -415.31768798828125,
      "logps/rejected": -607.0394897460938,
      "loss": 0.6835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1330680847167969,
      "rewards/margins": 1.2711782455444336,
      "rewards/rejected": -2.4042463302612305,
      "step": 11869
    },
    {
      "epoch": 1.553267359123252,
      "grad_norm": 10.027836799621582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1535426378250122,
      "logits/rejected": -0.14570944011211395,
      "logps/chosen": -483.28253173828125,
      "logps/rejected": -440.23687744140625,
      "loss": 0.4914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6558107137680054,
      "rewards/margins": 1.4769178628921509,
      "rewards/rejected": -3.1327285766601562,
      "step": 11870
    },
    {
      "epoch": 1.5533982170606035,
      "grad_norm": 13.234514236450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1477171629667282,
      "logits/rejected": -0.08260288089513779,
      "logps/chosen": -455.114013671875,
      "logps/rejected": -376.5851745605469,
      "loss": 0.7129,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.738790512084961,
      "rewards/margins": 0.4618866443634033,
      "rewards/rejected": -2.2006771564483643,
      "step": 11871
    },
    {
      "epoch": 1.5535290749979553,
      "grad_norm": 10.90539264678955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19570142030715942,
      "logits/rejected": -0.1626894772052765,
      "logps/chosen": -444.7918701171875,
      "logps/rejected": -448.7734069824219,
      "loss": 0.313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18629586696624756,
      "rewards/margins": 1.964457631111145,
      "rewards/rejected": -2.1507532596588135,
      "step": 11872
    },
    {
      "epoch": 1.5536599329353071,
      "grad_norm": 5.388717174530029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23214130103588104,
      "logits/rejected": -0.16994991898536682,
      "logps/chosen": -605.759033203125,
      "logps/rejected": -565.074462890625,
      "loss": 0.265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3771439790725708,
      "rewards/margins": 1.5306040048599243,
      "rewards/rejected": -2.9077484607696533,
      "step": 11873
    },
    {
      "epoch": 1.553790790872659,
      "grad_norm": 7.28400182723999,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4746285080909729,
      "logits/rejected": -0.4537387788295746,
      "logps/chosen": -430.7210388183594,
      "logps/rejected": -486.748779296875,
      "loss": 0.4233,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7211098670959473,
      "rewards/margins": 1.5570682287216187,
      "rewards/rejected": -2.2781782150268555,
      "step": 11874
    },
    {
      "epoch": 1.5539216488100105,
      "grad_norm": 11.503467559814453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05904492735862732,
      "logits/rejected": -0.1604098230600357,
      "logps/chosen": -482.6242370605469,
      "logps/rejected": -439.7002868652344,
      "loss": 0.3999,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5686250925064087,
      "rewards/margins": 1.7888548374176025,
      "rewards/rejected": -2.357480049133301,
      "step": 11875
    },
    {
      "epoch": 1.5540525067473623,
      "grad_norm": 6.213560581207275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4249522387981415,
      "logits/rejected": -0.2716153562068939,
      "logps/chosen": -343.8145751953125,
      "logps/rejected": -619.4158935546875,
      "loss": 0.2087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0555105209350586,
      "rewards/margins": 3.047307014465332,
      "rewards/rejected": -4.102817535400391,
      "step": 11876
    },
    {
      "epoch": 1.554183364684714,
      "grad_norm": 7.984168529510498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.368356853723526,
      "logits/rejected": -0.3959220349788666,
      "logps/chosen": -400.64404296875,
      "logps/rejected": -549.7556762695312,
      "loss": 0.3048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2612134218215942,
      "rewards/margins": 2.34478497505188,
      "rewards/rejected": -3.6059982776641846,
      "step": 11877
    },
    {
      "epoch": 1.5543142226220659,
      "grad_norm": 7.249446868896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28677085041999817,
      "logits/rejected": -0.38906416296958923,
      "logps/chosen": -422.2989501953125,
      "logps/rejected": -347.4339599609375,
      "loss": 0.4295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.449282705783844,
      "rewards/margins": 1.7684324979782104,
      "rewards/rejected": -2.217715263366699,
      "step": 11878
    },
    {
      "epoch": 1.5544450805594177,
      "grad_norm": 7.825953006744385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16550007462501526,
      "logits/rejected": -0.2271399199962616,
      "logps/chosen": -520.3515625,
      "logps/rejected": -451.3328552246094,
      "loss": 0.2532,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15933892130851746,
      "rewards/margins": 1.830901026725769,
      "rewards/rejected": -1.9902399778366089,
      "step": 11879
    },
    {
      "epoch": 1.5545759384967694,
      "grad_norm": 10.453543663024902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5125521421432495,
      "logits/rejected": -0.47195708751678467,
      "logps/chosen": -451.09722900390625,
      "logps/rejected": -458.1160888671875,
      "loss": 0.5791,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.684482753276825,
      "rewards/margins": 0.8981167078018188,
      "rewards/rejected": -1.5825995206832886,
      "step": 11880
    },
    {
      "epoch": 1.5547067964341212,
      "grad_norm": 11.082409858703613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021794207394123077,
      "logits/rejected": 0.06482117623090744,
      "logps/chosen": -429.8568115234375,
      "logps/rejected": -409.8432312011719,
      "loss": 0.6636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4798007011413574,
      "rewards/margins": 0.8760979175567627,
      "rewards/rejected": -2.35589861869812,
      "step": 11881
    },
    {
      "epoch": 1.554837654371473,
      "grad_norm": 9.698930740356445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41290849447250366,
      "logits/rejected": -0.246835857629776,
      "logps/chosen": -517.4791870117188,
      "logps/rejected": -415.0707702636719,
      "loss": 0.5076,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014390170574188232,
      "rewards/margins": 0.961387574672699,
      "rewards/rejected": -0.9469972252845764,
      "step": 11882
    },
    {
      "epoch": 1.5549685123088248,
      "grad_norm": 9.535948753356934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29673612117767334,
      "logits/rejected": -0.41618362069129944,
      "logps/chosen": -531.1926879882812,
      "logps/rejected": -478.8609313964844,
      "loss": 0.4056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9193135499954224,
      "rewards/margins": 1.0186829566955566,
      "rewards/rejected": -1.9379966259002686,
      "step": 11883
    },
    {
      "epoch": 1.5550993702461766,
      "grad_norm": 7.3406195640563965,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2505636215209961,
      "logits/rejected": 0.11691709607839584,
      "logps/chosen": -462.63232421875,
      "logps/rejected": -327.72198486328125,
      "loss": 0.3611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6811233162879944,
      "rewards/margins": 1.8179198503494263,
      "rewards/rejected": -2.4990432262420654,
      "step": 11884
    },
    {
      "epoch": 1.5552302281835284,
      "grad_norm": 8.047898292541504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29051879048347473,
      "logits/rejected": -0.733600378036499,
      "logps/chosen": -556.6395874023438,
      "logps/rejected": -479.978515625,
      "loss": 0.4807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6814768314361572,
      "rewards/margins": 1.67313814163208,
      "rewards/rejected": -3.3546149730682373,
      "step": 11885
    },
    {
      "epoch": 1.55536108612088,
      "grad_norm": 7.643648624420166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08322346210479736,
      "logits/rejected": -0.08708460628986359,
      "logps/chosen": -449.0267333984375,
      "logps/rejected": -605.1508178710938,
      "loss": 0.3866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14758893847465515,
      "rewards/margins": 2.3322911262512207,
      "rewards/rejected": -2.4798800945281982,
      "step": 11886
    },
    {
      "epoch": 1.5554919440582318,
      "grad_norm": 12.125065803527832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1876673400402069,
      "logits/rejected": -0.22502350807189941,
      "logps/chosen": -491.12744140625,
      "logps/rejected": -380.4921569824219,
      "loss": 0.5771,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6663007736206055,
      "rewards/margins": 0.9495817422866821,
      "rewards/rejected": -2.615882396697998,
      "step": 11887
    },
    {
      "epoch": 1.5556228019955836,
      "grad_norm": 11.907212257385254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1438407599925995,
      "logits/rejected": -0.18956239521503448,
      "logps/chosen": -562.3125610351562,
      "logps/rejected": -435.01361083984375,
      "loss": 0.6283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9510490894317627,
      "rewards/margins": 0.7530413269996643,
      "rewards/rejected": -2.7040903568267822,
      "step": 11888
    },
    {
      "epoch": 1.5557536599329354,
      "grad_norm": 12.22473430633545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10835254937410355,
      "logits/rejected": -0.31563252210617065,
      "logps/chosen": -654.13330078125,
      "logps/rejected": -610.5194702148438,
      "loss": 0.5565,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4015469551086426,
      "rewards/margins": 0.9938086271286011,
      "rewards/rejected": -2.395355701446533,
      "step": 11889
    },
    {
      "epoch": 1.555884517870287,
      "grad_norm": 5.845930099487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4695943295955658,
      "logits/rejected": -0.5558282136917114,
      "logps/chosen": -516.909423828125,
      "logps/rejected": -386.23944091796875,
      "loss": 0.3916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39619025588035583,
      "rewards/margins": 1.131020188331604,
      "rewards/rejected": -1.5272104740142822,
      "step": 11890
    },
    {
      "epoch": 1.5560153758076387,
      "grad_norm": 9.502150535583496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23309355974197388,
      "logits/rejected": -0.1136620044708252,
      "logps/chosen": -560.9503784179688,
      "logps/rejected": -503.9058837890625,
      "loss": 0.3937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013154223561286926,
      "rewards/margins": 1.6368858814239502,
      "rewards/rejected": -1.6237318515777588,
      "step": 11891
    },
    {
      "epoch": 1.5561462337449905,
      "grad_norm": 7.834319114685059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21479357779026031,
      "logits/rejected": -0.4228948950767517,
      "logps/chosen": -510.9090270996094,
      "logps/rejected": -426.38616943359375,
      "loss": 0.3118,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3798424005508423,
      "rewards/margins": 1.575513243675232,
      "rewards/rejected": -1.9553557634353638,
      "step": 11892
    },
    {
      "epoch": 1.5562770916823423,
      "grad_norm": 11.334568977355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36936110258102417,
      "logits/rejected": -0.6627393364906311,
      "logps/chosen": -453.82110595703125,
      "logps/rejected": -959.6570434570312,
      "loss": 0.5494,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.532912015914917,
      "rewards/margins": 1.4514764547348022,
      "rewards/rejected": -3.9843881130218506,
      "step": 11893
    },
    {
      "epoch": 1.556407949619694,
      "grad_norm": 9.451765060424805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33195000886917114,
      "logits/rejected": -0.3079041838645935,
      "logps/chosen": -645.0648193359375,
      "logps/rejected": -487.74969482421875,
      "loss": 0.5058,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6045697927474976,
      "rewards/margins": 1.0821888446807861,
      "rewards/rejected": -1.6867586374282837,
      "step": 11894
    },
    {
      "epoch": 1.5565388075570459,
      "grad_norm": 9.454721450805664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6046675443649292,
      "logits/rejected": -0.5046271085739136,
      "logps/chosen": -386.74444580078125,
      "logps/rejected": -473.0616455078125,
      "loss": 0.7124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3129225969314575,
      "rewards/margins": 0.3207588195800781,
      "rewards/rejected": -1.6336814165115356,
      "step": 11895
    },
    {
      "epoch": 1.5566696654943977,
      "grad_norm": 10.12004280090332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08927272260189056,
      "logits/rejected": -0.04148099198937416,
      "logps/chosen": -509.83978271484375,
      "logps/rejected": -640.252685546875,
      "loss": 0.3411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6924489140510559,
      "rewards/margins": 2.253744602203369,
      "rewards/rejected": -2.9461939334869385,
      "step": 11896
    },
    {
      "epoch": 1.5568005234317495,
      "grad_norm": 14.09264087677002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2506446838378906,
      "logits/rejected": -0.07122944295406342,
      "logps/chosen": -380.02734375,
      "logps/rejected": -365.5076599121094,
      "loss": 0.8533,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7252166271209717,
      "rewards/margins": -0.05336892604827881,
      "rewards/rejected": -1.6718475818634033,
      "step": 11897
    },
    {
      "epoch": 1.5569313813691013,
      "grad_norm": 5.016955375671387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15645647048950195,
      "logits/rejected": -0.24503356218338013,
      "logps/chosen": -354.5569152832031,
      "logps/rejected": -423.0774841308594,
      "loss": 0.274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8891924023628235,
      "rewards/margins": 1.484296441078186,
      "rewards/rejected": -2.3734889030456543,
      "step": 11898
    },
    {
      "epoch": 1.557062239306453,
      "grad_norm": 9.781901359558105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4625980854034424,
      "logits/rejected": -0.5053175091743469,
      "logps/chosen": -416.6539306640625,
      "logps/rejected": -470.054931640625,
      "loss": 0.5475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.827606201171875,
      "rewards/margins": 0.421395480632782,
      "rewards/rejected": -1.2490017414093018,
      "step": 11899
    },
    {
      "epoch": 1.5571930972438048,
      "grad_norm": 7.815624237060547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.316206157207489,
      "logits/rejected": -0.22407834231853485,
      "logps/chosen": -405.3623352050781,
      "logps/rejected": -420.1563415527344,
      "loss": 0.374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3236119747161865,
      "rewards/margins": 1.0942349433898926,
      "rewards/rejected": -2.417847156524658,
      "step": 11900
    },
    {
      "epoch": 1.5573239551811564,
      "grad_norm": 4.475099563598633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27676692605018616,
      "logits/rejected": -0.21308673918247223,
      "logps/chosen": -321.4048156738281,
      "logps/rejected": -352.38580322265625,
      "loss": 0.2716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3639410138130188,
      "rewards/margins": 1.5925410985946655,
      "rewards/rejected": -1.2286001443862915,
      "step": 11901
    },
    {
      "epoch": 1.5574548131185082,
      "grad_norm": 6.2946977615356445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5957562327384949,
      "logits/rejected": -0.3507160246372223,
      "logps/chosen": -489.7397766113281,
      "logps/rejected": -525.8776245117188,
      "loss": 0.3124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6046600937843323,
      "rewards/margins": 1.40468168258667,
      "rewards/rejected": -2.0093417167663574,
      "step": 11902
    },
    {
      "epoch": 1.55758567105586,
      "grad_norm": 7.92609977722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4154491424560547,
      "logits/rejected": -0.574155330657959,
      "logps/chosen": -586.0804443359375,
      "logps/rejected": -529.4639892578125,
      "loss": 0.3432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.014234930276870728,
      "rewards/margins": 2.025235414505005,
      "rewards/rejected": -2.011000394821167,
      "step": 11903
    },
    {
      "epoch": 1.5577165289932118,
      "grad_norm": 12.945089340209961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3923015594482422,
      "logits/rejected": -0.34000688791275024,
      "logps/chosen": -383.29425048828125,
      "logps/rejected": -456.76904296875,
      "loss": 0.8906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6092076301574707,
      "rewards/margins": -0.0033050626516342163,
      "rewards/rejected": -1.6059025526046753,
      "step": 11904
    },
    {
      "epoch": 1.5578473869305633,
      "grad_norm": 5.533908843994141,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030356496572494507,
      "logits/rejected": -0.07883919030427933,
      "logps/chosen": -391.1261901855469,
      "logps/rejected": -349.0812683105469,
      "loss": 0.3398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6051036715507507,
      "rewards/margins": 1.4338724613189697,
      "rewards/rejected": -2.0389761924743652,
      "step": 11905
    },
    {
      "epoch": 1.5579782448679151,
      "grad_norm": 9.206323623657227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4325696527957916,
      "logits/rejected": -0.41973036527633667,
      "logps/chosen": -499.98480224609375,
      "logps/rejected": -566.26806640625,
      "loss": 0.3404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7335270047187805,
      "rewards/margins": 1.6919888257980347,
      "rewards/rejected": -2.425515651702881,
      "step": 11906
    },
    {
      "epoch": 1.558109102805267,
      "grad_norm": 8.797258377075195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.358087420463562,
      "logits/rejected": -0.3130844831466675,
      "logps/chosen": -386.24951171875,
      "logps/rejected": -384.0870666503906,
      "loss": 0.3958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.37775456905365,
      "rewards/margins": 1.3481882810592651,
      "rewards/rejected": -2.725942850112915,
      "step": 11907
    },
    {
      "epoch": 1.5582399607426187,
      "grad_norm": 15.627674102783203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.402670294046402,
      "logits/rejected": -0.3890523910522461,
      "logps/chosen": -478.3769226074219,
      "logps/rejected": -489.966796875,
      "loss": 0.5322,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4358758926391602,
      "rewards/margins": 1.3053779602050781,
      "rewards/rejected": -2.741253614425659,
      "step": 11908
    },
    {
      "epoch": 1.5583708186799705,
      "grad_norm": 8.126851081848145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1442129909992218,
      "logits/rejected": -0.12824617326259613,
      "logps/chosen": -654.3973388671875,
      "logps/rejected": -566.2112426757812,
      "loss": 0.3444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7165143489837646,
      "rewards/margins": 1.20284903049469,
      "rewards/rejected": -1.9193633794784546,
      "step": 11909
    },
    {
      "epoch": 1.5585016766173223,
      "grad_norm": 12.275605201721191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20861008763313293,
      "logits/rejected": -0.3104313910007477,
      "logps/chosen": -599.9137573242188,
      "logps/rejected": -623.6463012695312,
      "loss": 0.4798,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12731117010116577,
      "rewards/margins": 1.4623734951019287,
      "rewards/rejected": -1.5896847248077393,
      "step": 11910
    },
    {
      "epoch": 1.558632534554674,
      "grad_norm": 9.136167526245117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26100584864616394,
      "logits/rejected": -0.252803236246109,
      "logps/chosen": -375.14190673828125,
      "logps/rejected": -466.53826904296875,
      "loss": 0.5459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0420238971710205,
      "rewards/margins": 0.7259453535079956,
      "rewards/rejected": -1.7679691314697266,
      "step": 11911
    },
    {
      "epoch": 1.558763392492026,
      "grad_norm": 4.392529010772705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3792104423046112,
      "logits/rejected": -0.2895912528038025,
      "logps/chosen": -468.31622314453125,
      "logps/rejected": -549.66845703125,
      "loss": 0.1616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7981166839599609,
      "rewards/margins": 2.1064181327819824,
      "rewards/rejected": -2.9045348167419434,
      "step": 11912
    },
    {
      "epoch": 1.5588942504293777,
      "grad_norm": 19.877639770507812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2623257637023926,
      "logits/rejected": -0.07983561605215073,
      "logps/chosen": -423.936279296875,
      "logps/rejected": -479.7833251953125,
      "loss": 0.1946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06837084889411926,
      "rewards/margins": 2.7338008880615234,
      "rewards/rejected": -2.6654300689697266,
      "step": 11913
    },
    {
      "epoch": 1.5590251083667295,
      "grad_norm": 6.072092056274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25182998180389404,
      "logits/rejected": -0.2911510765552521,
      "logps/chosen": -490.12359619140625,
      "logps/rejected": -450.663818359375,
      "loss": 0.4254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.81879061460495,
      "rewards/margins": 1.4744319915771484,
      "rewards/rejected": -2.293222427368164,
      "step": 11914
    },
    {
      "epoch": 1.5591559663040813,
      "grad_norm": 9.965516090393066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1444508582353592,
      "logits/rejected": 0.011868476867675781,
      "logps/chosen": -413.1473388671875,
      "logps/rejected": -501.6561584472656,
      "loss": 0.491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4321919679641724,
      "rewards/margins": 1.204890489578247,
      "rewards/rejected": -2.63708233833313,
      "step": 11915
    },
    {
      "epoch": 1.559286824241433,
      "grad_norm": 13.763489723205566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38716986775398254,
      "logits/rejected": -0.22205112874507904,
      "logps/chosen": -681.8745727539062,
      "logps/rejected": -640.7137451171875,
      "loss": 0.4729,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5291092395782471,
      "rewards/margins": 1.3501125574111938,
      "rewards/rejected": -1.879221796989441,
      "step": 11916
    },
    {
      "epoch": 1.5594176821787846,
      "grad_norm": 11.32243824005127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34875190258026123,
      "logits/rejected": -0.15700507164001465,
      "logps/chosen": -488.84649658203125,
      "logps/rejected": -509.42596435546875,
      "loss": 0.6028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8248351216316223,
      "rewards/margins": 1.3828434944152832,
      "rewards/rejected": -2.20767879486084,
      "step": 11917
    },
    {
      "epoch": 1.5595485401161364,
      "grad_norm": 12.164552688598633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3523423671722412,
      "logits/rejected": -0.1946016550064087,
      "logps/chosen": -416.08245849609375,
      "logps/rejected": -368.231201171875,
      "loss": 0.7786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.299936056137085,
      "rewards/margins": 0.43206119537353516,
      "rewards/rejected": -2.73199725151062,
      "step": 11918
    },
    {
      "epoch": 1.5596793980534882,
      "grad_norm": 10.253737449645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3171911835670471,
      "logits/rejected": -0.4511590003967285,
      "logps/chosen": -433.25347900390625,
      "logps/rejected": -445.782958984375,
      "loss": 0.6703,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1893084049224854,
      "rewards/margins": 0.23072224855422974,
      "rewards/rejected": -1.4200305938720703,
      "step": 11919
    },
    {
      "epoch": 1.5598102559908398,
      "grad_norm": 5.834470748901367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004609592258930206,
      "logits/rejected": -0.18217387795448303,
      "logps/chosen": -490.0304870605469,
      "logps/rejected": -374.5407409667969,
      "loss": 0.2976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0260210037231445,
      "rewards/margins": 1.8281846046447754,
      "rewards/rejected": -2.854205846786499,
      "step": 11920
    },
    {
      "epoch": 1.5599411139281916,
      "grad_norm": 10.963497161865234,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04946470260620117,
      "logits/rejected": 0.0572766438126564,
      "logps/chosen": -593.66357421875,
      "logps/rejected": -464.7053527832031,
      "loss": 0.6057,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.061239242553711,
      "rewards/margins": 0.8860161304473877,
      "rewards/rejected": -1.9472556114196777,
      "step": 11921
    },
    {
      "epoch": 1.5600719718655434,
      "grad_norm": 8.491093635559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37793904542922974,
      "logits/rejected": -0.39129796624183655,
      "logps/chosen": -486.1076965332031,
      "logps/rejected": -519.18798828125,
      "loss": 0.4525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0075546503067017,
      "rewards/margins": 1.0835922956466675,
      "rewards/rejected": -2.091146945953369,
      "step": 11922
    },
    {
      "epoch": 1.5602028298028952,
      "grad_norm": 10.390381813049316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25705456733703613,
      "logits/rejected": -0.1314268261194229,
      "logps/chosen": -527.4688720703125,
      "logps/rejected": -444.95953369140625,
      "loss": 0.5287,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7720521688461304,
      "rewards/margins": 1.7233498096466064,
      "rewards/rejected": -2.4954018592834473,
      "step": 11923
    },
    {
      "epoch": 1.560333687740247,
      "grad_norm": 6.94118070602417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29009580612182617,
      "logits/rejected": -0.3283214867115021,
      "logps/chosen": -413.14520263671875,
      "logps/rejected": -392.889404296875,
      "loss": 0.486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7457002401351929,
      "rewards/margins": 0.761750340461731,
      "rewards/rejected": -1.5074505805969238,
      "step": 11924
    },
    {
      "epoch": 1.5604645456775987,
      "grad_norm": 6.2482194900512695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08359961211681366,
      "logits/rejected": 0.08968620002269745,
      "logps/chosen": -365.02276611328125,
      "logps/rejected": -503.47900390625,
      "loss": 0.2673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4306499660015106,
      "rewards/margins": 1.9483917951583862,
      "rewards/rejected": -2.379042148590088,
      "step": 11925
    },
    {
      "epoch": 1.5605954036149505,
      "grad_norm": 5.743717670440674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12156693637371063,
      "logits/rejected": -0.16496437788009644,
      "logps/chosen": -607.414306640625,
      "logps/rejected": -540.6187744140625,
      "loss": 0.1571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004410535097122192,
      "rewards/margins": 3.585193395614624,
      "rewards/rejected": -3.5896036624908447,
      "step": 11926
    },
    {
      "epoch": 1.5607262615523023,
      "grad_norm": 7.8309102058410645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37449926137924194,
      "logits/rejected": -0.16678071022033691,
      "logps/chosen": -410.28424072265625,
      "logps/rejected": -480.29168701171875,
      "loss": 0.3665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6926985383033752,
      "rewards/margins": 1.13372802734375,
      "rewards/rejected": -1.8264265060424805,
      "step": 11927
    },
    {
      "epoch": 1.560857119489654,
      "grad_norm": 9.74447250366211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3594468832015991,
      "logits/rejected": -0.4630078077316284,
      "logps/chosen": -339.79119873046875,
      "logps/rejected": -423.7151184082031,
      "loss": 0.5773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4988317489624023,
      "rewards/margins": 0.5656294226646423,
      "rewards/rejected": -2.0644612312316895,
      "step": 11928
    },
    {
      "epoch": 1.560987977427006,
      "grad_norm": 5.509284496307373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4782736897468567,
      "logits/rejected": -0.5224226713180542,
      "logps/chosen": -482.8887939453125,
      "logps/rejected": -445.46807861328125,
      "loss": 0.2475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5073080062866211,
      "rewards/margins": 2.8864986896514893,
      "rewards/rejected": -3.3938071727752686,
      "step": 11929
    },
    {
      "epoch": 1.5611188353643577,
      "grad_norm": 8.329837799072266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35574010014533997,
      "logits/rejected": -0.07539501786231995,
      "logps/chosen": -609.4159545898438,
      "logps/rejected": -635.4823608398438,
      "loss": 0.3647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4839569330215454,
      "rewards/margins": 1.9266200065612793,
      "rewards/rejected": -1.4426631927490234,
      "step": 11930
    },
    {
      "epoch": 1.5612496933017095,
      "grad_norm": 14.157974243164062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15739774703979492,
      "logits/rejected": -0.19873471558094025,
      "logps/chosen": -468.930908203125,
      "logps/rejected": -421.9720153808594,
      "loss": 0.9102,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.634564995765686,
      "rewards/margins": -0.13902488350868225,
      "rewards/rejected": -1.4955401420593262,
      "step": 11931
    },
    {
      "epoch": 1.561380551239061,
      "grad_norm": 6.663182258605957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08040472865104675,
      "logits/rejected": -0.42705848813056946,
      "logps/chosen": -727.435791015625,
      "logps/rejected": -619.7776489257812,
      "loss": 0.2749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6971132159233093,
      "rewards/margins": 2.5258541107177734,
      "rewards/rejected": -3.2229678630828857,
      "step": 11932
    },
    {
      "epoch": 1.5615114091764128,
      "grad_norm": 5.234447956085205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10781298577785492,
      "logits/rejected": -0.08627702295780182,
      "logps/chosen": -403.0802307128906,
      "logps/rejected": -373.75823974609375,
      "loss": 0.2188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32125282287597656,
      "rewards/margins": 2.0347018241882324,
      "rewards/rejected": -2.355954647064209,
      "step": 11933
    },
    {
      "epoch": 1.5616422671137646,
      "grad_norm": 9.019855499267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3110654652118683,
      "logits/rejected": -0.15204928815364838,
      "logps/chosen": -607.6023559570312,
      "logps/rejected": -708.1905517578125,
      "loss": 0.3537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3715111315250397,
      "rewards/margins": 1.5337649583816528,
      "rewards/rejected": -1.9052761793136597,
      "step": 11934
    },
    {
      "epoch": 1.5617731250511164,
      "grad_norm": 16.17531967163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6323783993721008,
      "logits/rejected": -0.5701416730880737,
      "logps/chosen": -371.271484375,
      "logps/rejected": -400.9276428222656,
      "loss": 0.7356,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4858348369598389,
      "rewards/margins": 0.9494369029998779,
      "rewards/rejected": -2.435271978378296,
      "step": 11935
    },
    {
      "epoch": 1.561903982988468,
      "grad_norm": 7.878781795501709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23496106266975403,
      "logits/rejected": -0.159833163022995,
      "logps/chosen": -334.04132080078125,
      "logps/rejected": -335.4701232910156,
      "loss": 0.4544,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1039981842041016,
      "rewards/margins": 1.7593913078308105,
      "rewards/rejected": -2.863389492034912,
      "step": 11936
    },
    {
      "epoch": 1.5620348409258198,
      "grad_norm": 8.585143089294434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39760226011276245,
      "logits/rejected": -0.38926172256469727,
      "logps/chosen": -541.2527465820312,
      "logps/rejected": -473.46282958984375,
      "loss": 0.4884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6568059921264648,
      "rewards/margins": 0.7921382188796997,
      "rewards/rejected": -1.4489442110061646,
      "step": 11937
    },
    {
      "epoch": 1.5621656988631716,
      "grad_norm": 6.3009443283081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3634982705116272,
      "logits/rejected": -0.4139135479927063,
      "logps/chosen": -388.99749755859375,
      "logps/rejected": -432.9788818359375,
      "loss": 0.2564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4413520097732544,
      "rewards/margins": 1.8616853952407837,
      "rewards/rejected": -3.303037405014038,
      "step": 11938
    },
    {
      "epoch": 1.5622965568005234,
      "grad_norm": 6.152241230010986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25684288144111633,
      "logits/rejected": -0.40447402000427246,
      "logps/chosen": -456.6639404296875,
      "logps/rejected": -569.9494018554688,
      "loss": 0.3028,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.236356258392334,
      "rewards/margins": 2.033017158508301,
      "rewards/rejected": -3.2693731784820557,
      "step": 11939
    },
    {
      "epoch": 1.5624274147378752,
      "grad_norm": 4.86115837097168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2952412962913513,
      "logits/rejected": -0.3444443643093109,
      "logps/chosen": -463.8627624511719,
      "logps/rejected": -420.5965576171875,
      "loss": 0.2106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22403185069561005,
      "rewards/margins": 2.243704319000244,
      "rewards/rejected": -2.467736005783081,
      "step": 11940
    },
    {
      "epoch": 1.562558272675227,
      "grad_norm": 14.02211856842041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1255941540002823,
      "logits/rejected": -0.4014509916305542,
      "logps/chosen": -242.50326538085938,
      "logps/rejected": -335.148681640625,
      "loss": 0.6857,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0306992530822754,
      "rewards/margins": 0.5405954122543335,
      "rewards/rejected": -2.5712947845458984,
      "step": 11941
    },
    {
      "epoch": 1.5626891306125787,
      "grad_norm": 4.468839168548584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6067734360694885,
      "logits/rejected": -0.7788759469985962,
      "logps/chosen": -408.26629638671875,
      "logps/rejected": -626.8936157226562,
      "loss": 0.1834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.161110520362854,
      "rewards/margins": 2.1076624393463135,
      "rewards/rejected": -3.268773317337036,
      "step": 11942
    },
    {
      "epoch": 1.5628199885499305,
      "grad_norm": 9.847517967224121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5785667300224304,
      "logits/rejected": -0.3153700828552246,
      "logps/chosen": -357.8089904785156,
      "logps/rejected": -502.84130859375,
      "loss": 0.5216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18835681676864624,
      "rewards/margins": 2.4713239669799805,
      "rewards/rejected": -2.6596806049346924,
      "step": 11943
    },
    {
      "epoch": 1.5629508464872823,
      "grad_norm": 13.536931037902832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28931424021720886,
      "logits/rejected": -0.4304840564727783,
      "logps/chosen": -381.26446533203125,
      "logps/rejected": -531.2227783203125,
      "loss": 0.6501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.642653226852417,
      "rewards/margins": 1.1698280572891235,
      "rewards/rejected": -3.81248140335083,
      "step": 11944
    },
    {
      "epoch": 1.5630817044246341,
      "grad_norm": 17.174039840698242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4142714738845825,
      "logits/rejected": -0.32582929730415344,
      "logps/chosen": -646.6774291992188,
      "logps/rejected": -469.5095520019531,
      "loss": 1.1949,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.5539329051971436,
      "rewards/margins": -0.29919856786727905,
      "rewards/rejected": -2.254734516143799,
      "step": 11945
    },
    {
      "epoch": 1.563212562361986,
      "grad_norm": 8.792545318603516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2463088184595108,
      "logits/rejected": -0.16533175110816956,
      "logps/chosen": -356.26666259765625,
      "logps/rejected": -369.95135498046875,
      "loss": 0.5083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8308944702148438,
      "rewards/margins": 0.9107828140258789,
      "rewards/rejected": -1.7416772842407227,
      "step": 11946
    },
    {
      "epoch": 1.5633434202993375,
      "grad_norm": 14.50934886932373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14579349756240845,
      "logits/rejected": -0.42023706436157227,
      "logps/chosen": -518.3250122070312,
      "logps/rejected": -412.8412780761719,
      "loss": 1.0193,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9924349784851074,
      "rewards/margins": -0.07174190878868103,
      "rewards/rejected": -0.9206931591033936,
      "step": 11947
    },
    {
      "epoch": 1.5634742782366893,
      "grad_norm": 7.675178050994873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39714330434799194,
      "logits/rejected": -0.5078766942024231,
      "logps/chosen": -512.8469848632812,
      "logps/rejected": -503.00738525390625,
      "loss": 0.4261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3955179452896118,
      "rewards/margins": 1.828355073928833,
      "rewards/rejected": -3.2238733768463135,
      "step": 11948
    },
    {
      "epoch": 1.563605136174041,
      "grad_norm": 6.768719673156738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18961265683174133,
      "logits/rejected": -0.2228180170059204,
      "logps/chosen": -357.929931640625,
      "logps/rejected": -438.004150390625,
      "loss": 0.3615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.606992244720459,
      "rewards/margins": 1.6854915618896484,
      "rewards/rejected": -2.292483329772949,
      "step": 11949
    },
    {
      "epoch": 1.5637359941113929,
      "grad_norm": 12.32555103302002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7562780380249023,
      "logits/rejected": -0.8168279528617859,
      "logps/chosen": -499.70220947265625,
      "logps/rejected": -405.6134338378906,
      "loss": 0.416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3832901418209076,
      "rewards/margins": 1.1281439065933228,
      "rewards/rejected": -1.5114339590072632,
      "step": 11950
    },
    {
      "epoch": 1.5638668520487444,
      "grad_norm": 13.430838584899902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08467486500740051,
      "logits/rejected": -0.048988983035087585,
      "logps/chosen": -573.627197265625,
      "logps/rejected": -552.6277465820312,
      "loss": 0.561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8474880456924438,
      "rewards/margins": 1.3464977741241455,
      "rewards/rejected": -2.193985939025879,
      "step": 11951
    },
    {
      "epoch": 1.5639977099860962,
      "grad_norm": 14.920461654663086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4551141858100891,
      "logits/rejected": -0.4686225652694702,
      "logps/chosen": -466.509033203125,
      "logps/rejected": -553.4567260742188,
      "loss": 0.8988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9728405475616455,
      "rewards/margins": 0.7497089505195618,
      "rewards/rejected": -2.7225494384765625,
      "step": 11952
    },
    {
      "epoch": 1.564128567923448,
      "grad_norm": 10.957109451293945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5416574478149414,
      "logits/rejected": -0.3630681335926056,
      "logps/chosen": -490.7256774902344,
      "logps/rejected": -518.2562255859375,
      "loss": 0.5412,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2691526412963867,
      "rewards/margins": 1.0246285200119019,
      "rewards/rejected": -1.2937811613082886,
      "step": 11953
    },
    {
      "epoch": 1.5642594258607998,
      "grad_norm": 4.167499542236328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19393298029899597,
      "logits/rejected": -0.4183042645454407,
      "logps/chosen": -419.07684326171875,
      "logps/rejected": -433.74346923828125,
      "loss": 0.1919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5492717027664185,
      "rewards/margins": 1.9054592847824097,
      "rewards/rejected": -3.454730987548828,
      "step": 11954
    },
    {
      "epoch": 1.5643902837981516,
      "grad_norm": 13.693262100219727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.169235959649086,
      "logits/rejected": -0.1809062510728836,
      "logps/chosen": -587.0146484375,
      "logps/rejected": -519.3885498046875,
      "loss": 1.1902,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.7503962516784668,
      "rewards/margins": -0.5120891332626343,
      "rewards/rejected": -1.238306999206543,
      "step": 11955
    },
    {
      "epoch": 1.5645211417355034,
      "grad_norm": 9.34342098236084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3332301080226898,
      "logits/rejected": -0.5486907362937927,
      "logps/chosen": -458.0240478515625,
      "logps/rejected": -406.4312744140625,
      "loss": 0.505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8793450593948364,
      "rewards/margins": 1.1922521591186523,
      "rewards/rejected": -3.0715973377227783,
      "step": 11956
    },
    {
      "epoch": 1.5646519996728552,
      "grad_norm": 7.355253219604492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20482483506202698,
      "logits/rejected": -0.3042421042919159,
      "logps/chosen": -461.4176940917969,
      "logps/rejected": -521.4397583007812,
      "loss": 0.3663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9912593960762024,
      "rewards/margins": 1.5146803855895996,
      "rewards/rejected": -2.505939483642578,
      "step": 11957
    },
    {
      "epoch": 1.564782857610207,
      "grad_norm": 10.528122901916504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4720723628997803,
      "logits/rejected": -0.23680181801319122,
      "logps/chosen": -661.6680908203125,
      "logps/rejected": -520.6001586914062,
      "loss": 0.3984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5219476819038391,
      "rewards/margins": 1.5409653186798096,
      "rewards/rejected": -2.062912940979004,
      "step": 11958
    },
    {
      "epoch": 1.5649137155475588,
      "grad_norm": 17.013399124145508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09239358454942703,
      "logits/rejected": -0.49710217118263245,
      "logps/chosen": -448.517333984375,
      "logps/rejected": -618.1279296875,
      "loss": 0.6588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.826442003250122,
      "rewards/margins": 1.2970839738845825,
      "rewards/rejected": -3.123525857925415,
      "step": 11959
    },
    {
      "epoch": 1.5650445734849106,
      "grad_norm": 9.206465721130371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2733316719532013,
      "logits/rejected": -0.45589330792427063,
      "logps/chosen": -394.78277587890625,
      "logps/rejected": -439.08856201171875,
      "loss": 0.4305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.392247200012207,
      "rewards/margins": 2.0026581287384033,
      "rewards/rejected": -3.3949053287506104,
      "step": 11960
    },
    {
      "epoch": 1.5651754314222623,
      "grad_norm": 7.920356750488281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25519391894340515,
      "logits/rejected": -0.18686753511428833,
      "logps/chosen": -353.5771789550781,
      "logps/rejected": -460.4093322753906,
      "loss": 0.452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9497607946395874,
      "rewards/margins": 1.2311359643936157,
      "rewards/rejected": -2.180896520614624,
      "step": 11961
    },
    {
      "epoch": 1.5653062893596141,
      "grad_norm": 3.924302339553833,
      "learning_rate": 5e-06,
      "logits/chosen": 0.043136127293109894,
      "logits/rejected": -0.16067296266555786,
      "logps/chosen": -556.5189208984375,
      "logps/rejected": -449.2331237792969,
      "loss": 0.1537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5557209253311157,
      "rewards/margins": 2.438416004180908,
      "rewards/rejected": -1.882694959640503,
      "step": 11962
    },
    {
      "epoch": 1.5654371472969657,
      "grad_norm": 10.452717781066895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3158608078956604,
      "logits/rejected": -0.10093602538108826,
      "logps/chosen": -332.2078857421875,
      "logps/rejected": -361.29638671875,
      "loss": 0.5569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8866658210754395,
      "rewards/margins": 1.08282470703125,
      "rewards/rejected": -1.9694901704788208,
      "step": 11963
    },
    {
      "epoch": 1.5655680052343175,
      "grad_norm": 12.791033744812012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1327035278081894,
      "logits/rejected": -0.2948337495326996,
      "logps/chosen": -480.3550720214844,
      "logps/rejected": -426.47314453125,
      "loss": 0.8396,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.645156741142273,
      "rewards/margins": -0.01819494366645813,
      "rewards/rejected": -1.626961588859558,
      "step": 11964
    },
    {
      "epoch": 1.5656988631716693,
      "grad_norm": 8.281952857971191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33440932631492615,
      "logits/rejected": -0.41805559396743774,
      "logps/chosen": -587.1663208007812,
      "logps/rejected": -505.75372314453125,
      "loss": 0.76,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3345192670822144,
      "rewards/margins": 0.6518519520759583,
      "rewards/rejected": -1.9863712787628174,
      "step": 11965
    },
    {
      "epoch": 1.5658297211090209,
      "grad_norm": 13.822722434997559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15356029570102692,
      "logits/rejected": -0.43732741475105286,
      "logps/chosen": -570.6268920898438,
      "logps/rejected": -541.7879638671875,
      "loss": 0.7822,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.104607343673706,
      "rewards/margins": 0.40129518508911133,
      "rewards/rejected": -2.5059025287628174,
      "step": 11966
    },
    {
      "epoch": 1.5659605790463726,
      "grad_norm": 11.082062721252441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6191397309303284,
      "logits/rejected": -0.4306429326534271,
      "logps/chosen": -442.4193420410156,
      "logps/rejected": -372.91973876953125,
      "loss": 0.4725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3303228616714478,
      "rewards/margins": 1.6152031421661377,
      "rewards/rejected": -2.945526123046875,
      "step": 11967
    },
    {
      "epoch": 1.5660914369837244,
      "grad_norm": 9.105846405029297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1474764347076416,
      "logits/rejected": -0.23400913178920746,
      "logps/chosen": -380.30999755859375,
      "logps/rejected": -504.7882080078125,
      "loss": 0.4462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3325648307800293,
      "rewards/margins": 1.5794521570205688,
      "rewards/rejected": -2.9120171070098877,
      "step": 11968
    },
    {
      "epoch": 1.5662222949210762,
      "grad_norm": 8.698299407958984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28076237440109253,
      "logits/rejected": -0.13914430141448975,
      "logps/chosen": -363.52783203125,
      "logps/rejected": -398.3785705566406,
      "loss": 0.4844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6327481269836426,
      "rewards/margins": 0.7636352777481079,
      "rewards/rejected": -1.396383285522461,
      "step": 11969
    },
    {
      "epoch": 1.566353152858428,
      "grad_norm": 9.68307113647461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02183697745203972,
      "logits/rejected": -0.09650710225105286,
      "logps/chosen": -660.4319458007812,
      "logps/rejected": -709.6688842773438,
      "loss": 0.3402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1875606775283813,
      "rewards/margins": 1.311673641204834,
      "rewards/rejected": -2.499234437942505,
      "step": 11970
    },
    {
      "epoch": 1.5664840107957798,
      "grad_norm": 8.68012809753418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32270556688308716,
      "logits/rejected": 0.016893982887268066,
      "logps/chosen": -478.02105712890625,
      "logps/rejected": -608.7113037109375,
      "loss": 0.3802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9800856113433838,
      "rewards/margins": 1.1981847286224365,
      "rewards/rejected": -2.178270101547241,
      "step": 11971
    },
    {
      "epoch": 1.5666148687331316,
      "grad_norm": 7.007179260253906,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1137978807091713,
      "logits/rejected": 0.02397165820002556,
      "logps/chosen": -623.506591796875,
      "logps/rejected": -509.6817626953125,
      "loss": 0.3821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07171174883842468,
      "rewards/margins": 1.420168161392212,
      "rewards/rejected": -1.4918798208236694,
      "step": 11972
    },
    {
      "epoch": 1.5667457266704834,
      "grad_norm": 8.935325622558594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3872922956943512,
      "logits/rejected": -0.39015892148017883,
      "logps/chosen": -468.13720703125,
      "logps/rejected": -409.879150390625,
      "loss": 0.3937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6149531602859497,
      "rewards/margins": 1.1674158573150635,
      "rewards/rejected": -1.7823690176010132,
      "step": 11973
    },
    {
      "epoch": 1.5668765846078352,
      "grad_norm": 7.392067909240723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3366631269454956,
      "logits/rejected": -0.4980637729167938,
      "logps/chosen": -387.63671875,
      "logps/rejected": -357.8710632324219,
      "loss": 0.3269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0905107259750366,
      "rewards/margins": 1.5063061714172363,
      "rewards/rejected": -2.5968170166015625,
      "step": 11974
    },
    {
      "epoch": 1.567007442545187,
      "grad_norm": 7.713892936706543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26361849904060364,
      "logits/rejected": -0.13856233656406403,
      "logps/chosen": -452.6147766113281,
      "logps/rejected": -506.4320373535156,
      "loss": 0.3293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3314809799194336,
      "rewards/margins": 1.9082685708999634,
      "rewards/rejected": -2.2397494316101074,
      "step": 11975
    },
    {
      "epoch": 1.5671383004825388,
      "grad_norm": 7.451228618621826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07761546969413757,
      "logits/rejected": -0.23565836250782013,
      "logps/chosen": -736.6925659179688,
      "logps/rejected": -643.7410888671875,
      "loss": 0.3458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5655964016914368,
      "rewards/margins": 1.2742712497711182,
      "rewards/rejected": -1.8398675918579102,
      "step": 11976
    },
    {
      "epoch": 1.5672691584198906,
      "grad_norm": 4.255537986755371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27279338240623474,
      "logits/rejected": -0.0897776260972023,
      "logps/chosen": -518.6213989257812,
      "logps/rejected": -565.1443481445312,
      "loss": 0.1673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3101591169834137,
      "rewards/margins": 2.653594732284546,
      "rewards/rejected": -2.9637539386749268,
      "step": 11977
    },
    {
      "epoch": 1.5674000163572421,
      "grad_norm": 6.515718460083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18115797638893127,
      "logits/rejected": 0.02214657887816429,
      "logps/chosen": -367.32745361328125,
      "logps/rejected": -436.5546875,
      "loss": 0.3814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0916143655776978,
      "rewards/margins": 1.441139578819275,
      "rewards/rejected": -2.5327539443969727,
      "step": 11978
    },
    {
      "epoch": 1.567530874294594,
      "grad_norm": 8.752527236938477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5104457139968872,
      "logits/rejected": -0.5111727714538574,
      "logps/chosen": -418.1269836425781,
      "logps/rejected": -305.2188720703125,
      "loss": 0.351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7124360799789429,
      "rewards/margins": 1.3191120624542236,
      "rewards/rejected": -3.031548023223877,
      "step": 11979
    },
    {
      "epoch": 1.5676617322319457,
      "grad_norm": 6.006295204162598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3414859473705292,
      "logits/rejected": -0.3990265429019928,
      "logps/chosen": -351.75457763671875,
      "logps/rejected": -427.3301086425781,
      "loss": 0.3077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4959337115287781,
      "rewards/margins": 1.1767525672912598,
      "rewards/rejected": -1.6726863384246826,
      "step": 11980
    },
    {
      "epoch": 1.5677925901692975,
      "grad_norm": 6.87315034866333,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5685390830039978,
      "logits/rejected": -0.388044536113739,
      "logps/chosen": -398.9170837402344,
      "logps/rejected": -409.27252197265625,
      "loss": 0.3727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6729784607887268,
      "rewards/margins": 1.1600944995880127,
      "rewards/rejected": -1.8330729007720947,
      "step": 11981
    },
    {
      "epoch": 1.567923448106649,
      "grad_norm": 4.473325729370117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1839946061372757,
      "logits/rejected": -0.22986525297164917,
      "logps/chosen": -515.3372802734375,
      "logps/rejected": -444.4173278808594,
      "loss": 0.2276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4459703266620636,
      "rewards/margins": 2.0314078330993652,
      "rewards/rejected": -1.5854376554489136,
      "step": 11982
    },
    {
      "epoch": 1.5680543060440009,
      "grad_norm": 7.4425482749938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01832764968276024,
      "logits/rejected": -0.13819622993469238,
      "logps/chosen": -567.7001953125,
      "logps/rejected": -602.92041015625,
      "loss": 0.3007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5666770935058594,
      "rewards/margins": 1.7199631929397583,
      "rewards/rejected": -3.2866406440734863,
      "step": 11983
    },
    {
      "epoch": 1.5681851639813527,
      "grad_norm": 13.895758628845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4670635461807251,
      "logits/rejected": -0.4125324487686157,
      "logps/chosen": -471.93212890625,
      "logps/rejected": -369.2657470703125,
      "loss": 0.5765,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0103641748428345,
      "rewards/margins": 1.1639622449874878,
      "rewards/rejected": -2.1743264198303223,
      "step": 11984
    },
    {
      "epoch": 1.5683160219187045,
      "grad_norm": 13.91291618347168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4814091920852661,
      "logits/rejected": -0.43062564730644226,
      "logps/chosen": -388.32940673828125,
      "logps/rejected": -491.76275634765625,
      "loss": 0.8718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.67049241065979,
      "rewards/margins": 0.19880558550357819,
      "rewards/rejected": -1.8692978620529175,
      "step": 11985
    },
    {
      "epoch": 1.5684468798560562,
      "grad_norm": 4.820681095123291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4676085412502289,
      "logits/rejected": -0.5534675717353821,
      "logps/chosen": -380.29534912109375,
      "logps/rejected": -533.37353515625,
      "loss": 0.2473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.38103771209716797,
      "rewards/margins": 1.8195675611495972,
      "rewards/rejected": -2.2006051540374756,
      "step": 11986
    },
    {
      "epoch": 1.568577737793408,
      "grad_norm": 8.488075256347656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44828706979751587,
      "logits/rejected": -0.5689207315444946,
      "logps/chosen": -518.9737548828125,
      "logps/rejected": -574.4129028320312,
      "loss": 0.2909,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3463386595249176,
      "rewards/margins": 1.8899344205856323,
      "rewards/rejected": -2.2362730503082275,
      "step": 11987
    },
    {
      "epoch": 1.5687085957307598,
      "grad_norm": 11.873167991638184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2342166006565094,
      "logits/rejected": -0.5251301527023315,
      "logps/chosen": -477.885009765625,
      "logps/rejected": -448.64019775390625,
      "loss": 0.4992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0113515853881836,
      "rewards/margins": 1.1462059020996094,
      "rewards/rejected": -2.157557487487793,
      "step": 11988
    },
    {
      "epoch": 1.5688394536681116,
      "grad_norm": 5.2369303703308105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17834563553333282,
      "logits/rejected": -0.21443310379981995,
      "logps/chosen": -547.4876708984375,
      "logps/rejected": -616.7349243164062,
      "loss": 0.2138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7222126126289368,
      "rewards/margins": 2.091397285461426,
      "rewards/rejected": -2.8136096000671387,
      "step": 11989
    },
    {
      "epoch": 1.5689703116054634,
      "grad_norm": 9.320204734802246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17404517531394958,
      "logits/rejected": -0.05356472730636597,
      "logps/chosen": -566.9742431640625,
      "logps/rejected": -541.4140625,
      "loss": 0.3491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7355095148086548,
      "rewards/margins": 1.5573734045028687,
      "rewards/rejected": -2.2928829193115234,
      "step": 11990
    },
    {
      "epoch": 1.5691011695428152,
      "grad_norm": 9.282511711120605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4772012233734131,
      "logits/rejected": -0.23077654838562012,
      "logps/chosen": -399.57147216796875,
      "logps/rejected": -478.8726501464844,
      "loss": 0.5997,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6714067459106445,
      "rewards/margins": 1.6292555332183838,
      "rewards/rejected": -2.3006622791290283,
      "step": 11991
    },
    {
      "epoch": 1.569232027480167,
      "grad_norm": 8.34882926940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4194027781486511,
      "logits/rejected": -0.3759804964065552,
      "logps/chosen": -464.4064636230469,
      "logps/rejected": -427.4342346191406,
      "loss": 0.4205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9877948760986328,
      "rewards/margins": 1.528745412826538,
      "rewards/rejected": -2.516540288925171,
      "step": 11992
    },
    {
      "epoch": 1.5693628854175186,
      "grad_norm": 17.05504035949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3067348897457123,
      "logits/rejected": -0.5469505190849304,
      "logps/chosen": -864.9855346679688,
      "logps/rejected": -551.5712890625,
      "loss": 0.9189,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0825059413909912,
      "rewards/margins": 0.18554404377937317,
      "rewards/rejected": -1.268049955368042,
      "step": 11993
    },
    {
      "epoch": 1.5694937433548704,
      "grad_norm": 9.442916870117188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16579669713974,
      "logits/rejected": -0.04641851782798767,
      "logps/chosen": -497.35400390625,
      "logps/rejected": -635.651123046875,
      "loss": 0.412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.246482014656067,
      "rewards/margins": 1.6934071779251099,
      "rewards/rejected": -2.939889430999756,
      "step": 11994
    },
    {
      "epoch": 1.5696246012922221,
      "grad_norm": 9.771004676818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2579589784145355,
      "logits/rejected": -0.22678738832473755,
      "logps/chosen": -407.0303955078125,
      "logps/rejected": -432.82666015625,
      "loss": 0.4845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5400325059890747,
      "rewards/margins": 1.1370413303375244,
      "rewards/rejected": -2.6770737171173096,
      "step": 11995
    },
    {
      "epoch": 1.569755459229574,
      "grad_norm": 6.67376708984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1883745789527893,
      "logits/rejected": -0.3363165259361267,
      "logps/chosen": -402.22833251953125,
      "logps/rejected": -411.4565734863281,
      "loss": 0.2783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8124189972877502,
      "rewards/margins": 2.15643048286438,
      "rewards/rejected": -2.9688496589660645,
      "step": 11996
    },
    {
      "epoch": 1.5698863171669255,
      "grad_norm": 4.216017723083496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6492467522621155,
      "logits/rejected": -0.6554873585700989,
      "logps/chosen": -395.91876220703125,
      "logps/rejected": -409.0736999511719,
      "loss": 0.1807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4552578330039978,
      "rewards/margins": 2.0367958545684814,
      "rewards/rejected": -2.492053747177124,
      "step": 11997
    },
    {
      "epoch": 1.5700171751042773,
      "grad_norm": 8.128881454467773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19474561512470245,
      "logits/rejected": -0.35546785593032837,
      "logps/chosen": -361.06036376953125,
      "logps/rejected": -406.00732421875,
      "loss": 0.4614,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1234941482543945,
      "rewards/margins": 1.30177640914917,
      "rewards/rejected": -3.4252705574035645,
      "step": 11998
    },
    {
      "epoch": 1.570148033041629,
      "grad_norm": 13.930309295654297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2334119826555252,
      "logits/rejected": -0.21184544265270233,
      "logps/chosen": -369.53533935546875,
      "logps/rejected": -295.42010498046875,
      "loss": 0.79,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4566301107406616,
      "rewards/margins": 0.4261476397514343,
      "rewards/rejected": -1.8827778100967407,
      "step": 11999
    },
    {
      "epoch": 1.5702788909789809,
      "grad_norm": 8.978204727172852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4014211893081665,
      "logits/rejected": -0.5059720873832703,
      "logps/chosen": -405.37451171875,
      "logps/rejected": -425.94598388671875,
      "loss": 0.4175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9772434234619141,
      "rewards/margins": 1.5697349309921265,
      "rewards/rejected": -2.54697847366333,
      "step": 12000
    },
    {
      "epoch": 1.5704097489163327,
      "grad_norm": 6.799941062927246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.001449383795261383,
      "logits/rejected": -0.2563437223434448,
      "logps/chosen": -345.84478759765625,
      "logps/rejected": -455.5766906738281,
      "loss": 0.4032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6292085647583008,
      "rewards/margins": 1.4492876529693604,
      "rewards/rejected": -2.0784964561462402,
      "step": 12001
    },
    {
      "epoch": 1.5705406068536845,
      "grad_norm": 7.849273681640625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04106138274073601,
      "logits/rejected": -0.18169796466827393,
      "logps/chosen": -516.1619873046875,
      "logps/rejected": -461.9865417480469,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8083245158195496,
      "rewards/margins": 1.7964204549789429,
      "rewards/rejected": -2.6047449111938477,
      "step": 12002
    },
    {
      "epoch": 1.5706714647910363,
      "grad_norm": 7.053703308105469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4419955313205719,
      "logits/rejected": -0.3777647614479065,
      "logps/chosen": -460.66552734375,
      "logps/rejected": -441.77447509765625,
      "loss": 0.2778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5191416144371033,
      "rewards/margins": 1.7214910984039307,
      "rewards/rejected": -2.2406327724456787,
      "step": 12003
    },
    {
      "epoch": 1.570802322728388,
      "grad_norm": 8.356582641601562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18118971586227417,
      "logits/rejected": -0.4307737350463867,
      "logps/chosen": -556.5471801757812,
      "logps/rejected": -433.9677734375,
      "loss": 0.3458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8016649484634399,
      "rewards/margins": 1.559553623199463,
      "rewards/rejected": -2.3612184524536133,
      "step": 12004
    },
    {
      "epoch": 1.5709331806657398,
      "grad_norm": 24.655235290527344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34913450479507446,
      "logits/rejected": -0.41853299736976624,
      "logps/chosen": -515.319091796875,
      "logps/rejected": -530.5159301757812,
      "loss": 0.9618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4332799911499023,
      "rewards/margins": 0.6152764558792114,
      "rewards/rejected": -3.048556327819824,
      "step": 12005
    },
    {
      "epoch": 1.5710640386030916,
      "grad_norm": 9.838077545166016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24428260326385498,
      "logits/rejected": -0.27142637968063354,
      "logps/chosen": -330.5050354003906,
      "logps/rejected": -404.18695068359375,
      "loss": 0.7131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3822908401489258,
      "rewards/margins": 0.8955045342445374,
      "rewards/rejected": -2.2777953147888184,
      "step": 12006
    },
    {
      "epoch": 1.5711948965404434,
      "grad_norm": 9.048176765441895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08247403800487518,
      "logits/rejected": -0.17773598432540894,
      "logps/chosen": -526.7066650390625,
      "logps/rejected": -421.3539733886719,
      "loss": 0.524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49092426896095276,
      "rewards/margins": 0.9653967022895813,
      "rewards/rejected": -1.4563210010528564,
      "step": 12007
    },
    {
      "epoch": 1.571325754477795,
      "grad_norm": 10.542803764343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3052379786968231,
      "logits/rejected": -0.28045356273651123,
      "logps/chosen": -427.61175537109375,
      "logps/rejected": -429.3044738769531,
      "loss": 0.5067,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.358608603477478,
      "rewards/margins": 0.7030122876167297,
      "rewards/rejected": -2.0616209506988525,
      "step": 12008
    },
    {
      "epoch": 1.5714566124151468,
      "grad_norm": 7.722842693328857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19879963994026184,
      "logits/rejected": -0.48043563961982727,
      "logps/chosen": -366.8544616699219,
      "logps/rejected": -462.2851257324219,
      "loss": 0.3002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9381381273269653,
      "rewards/margins": 1.8283363580703735,
      "rewards/rejected": -2.766474485397339,
      "step": 12009
    },
    {
      "epoch": 1.5715874703524986,
      "grad_norm": 9.772574424743652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26706811785697937,
      "logits/rejected": -0.26265135407447815,
      "logps/chosen": -550.4691772460938,
      "logps/rejected": -430.0952453613281,
      "loss": 0.5494,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.929198980331421,
      "rewards/margins": 0.8369080424308777,
      "rewards/rejected": -2.7661070823669434,
      "step": 12010
    },
    {
      "epoch": 1.5717183282898504,
      "grad_norm": 12.2349214553833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3054787814617157,
      "logits/rejected": -0.19249725341796875,
      "logps/chosen": -517.2970581054688,
      "logps/rejected": -506.1508483886719,
      "loss": 0.5324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9274089932441711,
      "rewards/margins": 0.7434667944908142,
      "rewards/rejected": -1.6708757877349854,
      "step": 12011
    },
    {
      "epoch": 1.571849186227202,
      "grad_norm": 7.89784574508667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3280789852142334,
      "logits/rejected": -0.2868449091911316,
      "logps/chosen": -320.57147216796875,
      "logps/rejected": -405.43695068359375,
      "loss": 0.4731,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7202417254447937,
      "rewards/margins": 0.80772465467453,
      "rewards/rejected": -1.5279663801193237,
      "step": 12012
    },
    {
      "epoch": 1.5719800441645537,
      "grad_norm": 11.930768013000488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40259259939193726,
      "logits/rejected": -0.6741529107093811,
      "logps/chosen": -476.3970031738281,
      "logps/rejected": -462.8016662597656,
      "loss": 0.7762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7208868265151978,
      "rewards/margins": 0.7683563232421875,
      "rewards/rejected": -2.4892430305480957,
      "step": 12013
    },
    {
      "epoch": 1.5721109021019055,
      "grad_norm": 10.788963317871094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7048477530479431,
      "logits/rejected": -0.4401417672634125,
      "logps/chosen": -444.5176086425781,
      "logps/rejected": -490.13665771484375,
      "loss": 0.4284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2121143341064453,
      "rewards/margins": 1.5607291460037231,
      "rewards/rejected": -3.772843837738037,
      "step": 12014
    },
    {
      "epoch": 1.5722417600392573,
      "grad_norm": 8.507318496704102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22057190537452698,
      "logits/rejected": -0.3114749789237976,
      "logps/chosen": -375.18157958984375,
      "logps/rejected": -418.81378173828125,
      "loss": 0.5221,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8939739465713501,
      "rewards/margins": 1.3576065301895142,
      "rewards/rejected": -2.2515804767608643,
      "step": 12015
    },
    {
      "epoch": 1.572372617976609,
      "grad_norm": 8.763327598571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034718871116638184,
      "logits/rejected": -0.329459011554718,
      "logps/chosen": -518.0952758789062,
      "logps/rejected": -532.7525634765625,
      "loss": 0.3089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7636642456054688,
      "rewards/margins": 1.4646790027618408,
      "rewards/rejected": -2.2283430099487305,
      "step": 12016
    },
    {
      "epoch": 1.572503475913961,
      "grad_norm": 31.46521759033203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15966753661632538,
      "logits/rejected": -0.23548118770122528,
      "logps/chosen": -433.6062316894531,
      "logps/rejected": -345.6315612792969,
      "loss": 0.8843,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -3.0492143630981445,
      "rewards/margins": 0.09814547002315521,
      "rewards/rejected": -3.147359848022461,
      "step": 12017
    },
    {
      "epoch": 1.5726343338513127,
      "grad_norm": 8.125469207763672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22348704934120178,
      "logits/rejected": -0.005140781402587891,
      "logps/chosen": -487.09527587890625,
      "logps/rejected": -552.7975463867188,
      "loss": 0.4781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4014346599578857,
      "rewards/margins": 0.804583728313446,
      "rewards/rejected": -2.2060184478759766,
      "step": 12018
    },
    {
      "epoch": 1.5727651917886645,
      "grad_norm": 8.682580947875977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21201667189598083,
      "logits/rejected": -0.26897749304771423,
      "logps/chosen": -538.1878051757812,
      "logps/rejected": -629.6400146484375,
      "loss": 0.3029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12351875007152557,
      "rewards/margins": 2.3743345737457275,
      "rewards/rejected": -2.4978532791137695,
      "step": 12019
    },
    {
      "epoch": 1.5728960497260163,
      "grad_norm": 8.194519996643066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.424807071685791,
      "logits/rejected": -0.5832246541976929,
      "logps/chosen": -424.5104064941406,
      "logps/rejected": -437.36932373046875,
      "loss": 0.4604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2238441705703735,
      "rewards/margins": 1.4188456535339355,
      "rewards/rejected": -2.6426897048950195,
      "step": 12020
    },
    {
      "epoch": 1.573026907663368,
      "grad_norm": 5.966169834136963,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27125084400177,
      "logits/rejected": -0.3319409489631653,
      "logps/chosen": -558.4159545898438,
      "logps/rejected": -398.499755859375,
      "loss": 0.2223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.586318016052246,
      "rewards/margins": 1.9249517917633057,
      "rewards/rejected": -3.511270046234131,
      "step": 12021
    },
    {
      "epoch": 1.5731577656007198,
      "grad_norm": 10.161273956298828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48601627349853516,
      "logits/rejected": -0.5891450643539429,
      "logps/chosen": -573.9844360351562,
      "logps/rejected": -500.757568359375,
      "loss": 0.4852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7677979469299316,
      "rewards/margins": 1.4562828540802002,
      "rewards/rejected": -2.2240805625915527,
      "step": 12022
    },
    {
      "epoch": 1.5732886235380716,
      "grad_norm": 9.402118682861328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31794536113739014,
      "logits/rejected": -0.29194507002830505,
      "logps/chosen": -449.0277099609375,
      "logps/rejected": -576.02978515625,
      "loss": 0.3475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43868696689605713,
      "rewards/margins": 1.6530263423919678,
      "rewards/rejected": -2.0917134284973145,
      "step": 12023
    },
    {
      "epoch": 1.5734194814754232,
      "grad_norm": 7.648252964019775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3376876711845398,
      "logits/rejected": -0.35364484786987305,
      "logps/chosen": -482.620361328125,
      "logps/rejected": -516.3704833984375,
      "loss": 0.3048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3176654875278473,
      "rewards/margins": 1.4725035429000854,
      "rewards/rejected": -1.7901691198349,
      "step": 12024
    },
    {
      "epoch": 1.573550339412775,
      "grad_norm": 10.957836151123047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3654705584049225,
      "logits/rejected": -0.420831561088562,
      "logps/chosen": -381.25604248046875,
      "logps/rejected": -362.560791015625,
      "loss": 0.5459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2224408388137817,
      "rewards/margins": 0.8374999761581421,
      "rewards/rejected": -2.059940814971924,
      "step": 12025
    },
    {
      "epoch": 1.5736811973501268,
      "grad_norm": 7.823179244995117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14497286081314087,
      "logits/rejected": 0.010727230459451675,
      "logps/chosen": -447.34173583984375,
      "logps/rejected": -420.40380859375,
      "loss": 0.4465,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5720479488372803,
      "rewards/margins": 0.9286336898803711,
      "rewards/rejected": -2.5006816387176514,
      "step": 12026
    },
    {
      "epoch": 1.5738120552874786,
      "grad_norm": 8.58954906463623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06375876814126968,
      "logits/rejected": -0.2953409254550934,
      "logps/chosen": -479.0985412597656,
      "logps/rejected": -403.6251220703125,
      "loss": 0.4306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.053209662437439,
      "rewards/margins": 1.3763501644134521,
      "rewards/rejected": -2.4295599460601807,
      "step": 12027
    },
    {
      "epoch": 1.5739429132248302,
      "grad_norm": 8.490399360656738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26453733444213867,
      "logits/rejected": -0.09657620638608932,
      "logps/chosen": -300.6647033691406,
      "logps/rejected": -423.4956359863281,
      "loss": 0.4632,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.27851402759552,
      "rewards/margins": 0.995404839515686,
      "rewards/rejected": -2.273919105529785,
      "step": 12028
    },
    {
      "epoch": 1.574073771162182,
      "grad_norm": 8.731316566467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13287001848220825,
      "logits/rejected": -0.0171042550355196,
      "logps/chosen": -501.8589172363281,
      "logps/rejected": -405.9536437988281,
      "loss": 0.4256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.096543788909912,
      "rewards/margins": 0.8953428268432617,
      "rewards/rejected": -1.9918866157531738,
      "step": 12029
    },
    {
      "epoch": 1.5742046290995337,
      "grad_norm": 8.702136993408203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14883218705654144,
      "logits/rejected": -0.14438508450984955,
      "logps/chosen": -466.2266845703125,
      "logps/rejected": -461.7901916503906,
      "loss": 0.4805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.296457290649414,
      "rewards/margins": 1.807586908340454,
      "rewards/rejected": -3.1040444374084473,
      "step": 12030
    },
    {
      "epoch": 1.5743354870368855,
      "grad_norm": 4.578505039215088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3445183038711548,
      "logits/rejected": -0.34408771991729736,
      "logps/chosen": -592.4805297851562,
      "logps/rejected": -498.67510986328125,
      "loss": 0.1545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6559009552001953,
      "rewards/margins": 2.315809726715088,
      "rewards/rejected": -2.971710681915283,
      "step": 12031
    },
    {
      "epoch": 1.5744663449742373,
      "grad_norm": 6.341877460479736,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10276477038860321,
      "logits/rejected": -0.1419641375541687,
      "logps/chosen": -320.8935546875,
      "logps/rejected": -527.5048217773438,
      "loss": 0.2733,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.892798900604248,
      "rewards/margins": 1.6405270099639893,
      "rewards/rejected": -3.533325672149658,
      "step": 12032
    },
    {
      "epoch": 1.5745972029115891,
      "grad_norm": 11.338451385498047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2852547764778137,
      "logits/rejected": -0.43196359276771545,
      "logps/chosen": -460.4559020996094,
      "logps/rejected": -504.7677001953125,
      "loss": 0.8786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5783060789108276,
      "rewards/margins": 0.42512279748916626,
      "rewards/rejected": -2.0034289360046387,
      "step": 12033
    },
    {
      "epoch": 1.574728060848941,
      "grad_norm": 9.47722339630127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3219340145587921,
      "logits/rejected": -0.20751933753490448,
      "logps/chosen": -371.08514404296875,
      "logps/rejected": -461.98834228515625,
      "loss": 0.717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1652767658233643,
      "rewards/margins": 0.5126998424530029,
      "rewards/rejected": -2.6779770851135254,
      "step": 12034
    },
    {
      "epoch": 1.5748589187862927,
      "grad_norm": 8.528279304504395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3934314250946045,
      "logits/rejected": -0.4761139154434204,
      "logps/chosen": -220.80038452148438,
      "logps/rejected": -350.39117431640625,
      "loss": 0.4755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5301663875579834,
      "rewards/margins": 1.5553109645843506,
      "rewards/rejected": -3.085477352142334,
      "step": 12035
    },
    {
      "epoch": 1.5749897767236445,
      "grad_norm": 10.330892562866211,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11774474382400513,
      "logits/rejected": -0.08804920315742493,
      "logps/chosen": -503.9277648925781,
      "logps/rejected": -531.6605224609375,
      "loss": 0.3461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2810642719268799,
      "rewards/margins": 1.3361259698867798,
      "rewards/rejected": -2.617190361022949,
      "step": 12036
    },
    {
      "epoch": 1.5751206346609963,
      "grad_norm": 8.204400062561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0987541526556015,
      "logits/rejected": -0.23048511147499084,
      "logps/chosen": -486.733642578125,
      "logps/rejected": -582.2451171875,
      "loss": 0.3842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35911867022514343,
      "rewards/margins": 1.2880690097808838,
      "rewards/rejected": -1.6471877098083496,
      "step": 12037
    },
    {
      "epoch": 1.575251492598348,
      "grad_norm": 5.902798652648926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08325594663619995,
      "logits/rejected": -0.19744369387626648,
      "logps/chosen": -534.78466796875,
      "logps/rejected": -569.8522338867188,
      "loss": 0.2888,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8008649945259094,
      "rewards/margins": 2.088491678237915,
      "rewards/rejected": -2.8893566131591797,
      "step": 12038
    },
    {
      "epoch": 1.5753823505356996,
      "grad_norm": 8.727018356323242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08026663959026337,
      "logits/rejected": -0.1952708214521408,
      "logps/chosen": -606.2730712890625,
      "logps/rejected": -381.593994140625,
      "loss": 0.4427,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0187656879425049,
      "rewards/margins": 0.746210515499115,
      "rewards/rejected": -1.7649762630462646,
      "step": 12039
    },
    {
      "epoch": 1.5755132084730514,
      "grad_norm": 11.035335540771484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057057060301303864,
      "logits/rejected": -0.07222311198711395,
      "logps/chosen": -434.99786376953125,
      "logps/rejected": -511.7623596191406,
      "loss": 0.6857,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9865785837173462,
      "rewards/margins": 0.24120715260505676,
      "rewards/rejected": -2.22778582572937,
      "step": 12040
    },
    {
      "epoch": 1.5756440664104032,
      "grad_norm": 9.514230728149414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2619698643684387,
      "logits/rejected": -0.34520551562309265,
      "logps/chosen": -432.8980712890625,
      "logps/rejected": -362.007568359375,
      "loss": 0.5424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4143129587173462,
      "rewards/margins": 0.8119373321533203,
      "rewards/rejected": -2.226250410079956,
      "step": 12041
    },
    {
      "epoch": 1.575774924347755,
      "grad_norm": 7.112872123718262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44251537322998047,
      "logits/rejected": -0.43509188294410706,
      "logps/chosen": -345.2752380371094,
      "logps/rejected": -759.8294677734375,
      "loss": 0.3809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5010957717895508,
      "rewards/margins": 2.0369298458099365,
      "rewards/rejected": -3.5380256175994873,
      "step": 12042
    },
    {
      "epoch": 1.5759057822851066,
      "grad_norm": 10.671422958374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30466148257255554,
      "logits/rejected": -0.36134257912635803,
      "logps/chosen": -419.6682434082031,
      "logps/rejected": -483.91900634765625,
      "loss": 0.7337,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.762488842010498,
      "rewards/margins": 0.6555740833282471,
      "rewards/rejected": -2.418062925338745,
      "step": 12043
    },
    {
      "epoch": 1.5760366402224584,
      "grad_norm": 9.596701622009277,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12150299549102783,
      "logits/rejected": 0.18480853736400604,
      "logps/chosen": -555.6658935546875,
      "logps/rejected": -452.1837463378906,
      "loss": 0.5154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4026954174041748,
      "rewards/margins": 1.2935545444488525,
      "rewards/rejected": -2.6962502002716064,
      "step": 12044
    },
    {
      "epoch": 1.5761674981598102,
      "grad_norm": 6.37217378616333,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03805716335773468,
      "logits/rejected": -0.42502689361572266,
      "logps/chosen": -533.5158081054688,
      "logps/rejected": -506.6703796386719,
      "loss": 0.3014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.534759521484375,
      "rewards/margins": 1.391585111618042,
      "rewards/rejected": -2.926344394683838,
      "step": 12045
    },
    {
      "epoch": 1.576298356097162,
      "grad_norm": 10.01566219329834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4391592741012573,
      "logits/rejected": -0.19142085313796997,
      "logps/chosen": -347.60882568359375,
      "logps/rejected": -409.0799255371094,
      "loss": 0.6835,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5621967315673828,
      "rewards/margins": 0.13775157928466797,
      "rewards/rejected": -1.6999483108520508,
      "step": 12046
    },
    {
      "epoch": 1.5764292140345137,
      "grad_norm": 8.423029899597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5470128655433655,
      "logits/rejected": -0.5154370069503784,
      "logps/chosen": -398.0048828125,
      "logps/rejected": -400.2231140136719,
      "loss": 0.4646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1211166381835938,
      "rewards/margins": 1.8012592792510986,
      "rewards/rejected": -2.9223759174346924,
      "step": 12047
    },
    {
      "epoch": 1.5765600719718655,
      "grad_norm": 10.555432319641113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015256084501743317,
      "logits/rejected": -0.11719755828380585,
      "logps/chosen": -453.5733337402344,
      "logps/rejected": -635.7946166992188,
      "loss": 0.4822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.610072135925293,
      "rewards/margins": 1.8008196353912354,
      "rewards/rejected": -2.4108920097351074,
      "step": 12048
    },
    {
      "epoch": 1.5766909299092173,
      "grad_norm": 4.7675089836120605,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03946564719080925,
      "logits/rejected": -0.2874296307563782,
      "logps/chosen": -416.7695007324219,
      "logps/rejected": -558.89111328125,
      "loss": 0.1558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4291245937347412,
      "rewards/margins": 2.8972835540771484,
      "rewards/rejected": -4.3264079093933105,
      "step": 12049
    },
    {
      "epoch": 1.5768217878465691,
      "grad_norm": 8.49344253540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31000033020973206,
      "logits/rejected": -0.29502278566360474,
      "logps/chosen": -386.98126220703125,
      "logps/rejected": -454.6126708984375,
      "loss": 0.5917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9329723119735718,
      "rewards/margins": 0.7422075867652893,
      "rewards/rejected": -2.6751797199249268,
      "step": 12050
    },
    {
      "epoch": 1.576952645783921,
      "grad_norm": 8.054737091064453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3274816870689392,
      "logits/rejected": -0.13733723759651184,
      "logps/chosen": -356.972412109375,
      "logps/rejected": -324.0223388671875,
      "loss": 0.4517,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4060475826263428,
      "rewards/margins": 0.9738789796829224,
      "rewards/rejected": -2.3799264430999756,
      "step": 12051
    },
    {
      "epoch": 1.5770835037212727,
      "grad_norm": 5.834048271179199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3521001935005188,
      "logits/rejected": -0.5380140542984009,
      "logps/chosen": -378.007568359375,
      "logps/rejected": -590.4691162109375,
      "loss": 0.4235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2359808683395386,
      "rewards/margins": 1.2603803873062134,
      "rewards/rejected": -2.496361017227173,
      "step": 12052
    },
    {
      "epoch": 1.5772143616586245,
      "grad_norm": 3.622101068496704,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31627124547958374,
      "logits/rejected": -0.2639061510562897,
      "logps/chosen": -352.4175720214844,
      "logps/rejected": -439.1539001464844,
      "loss": 0.1384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1456806659698486,
      "rewards/margins": 2.0683531761169434,
      "rewards/rejected": -3.214033603668213,
      "step": 12053
    },
    {
      "epoch": 1.577345219595976,
      "grad_norm": 9.042752265930176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1393580585718155,
      "logits/rejected": -0.3813433051109314,
      "logps/chosen": -550.0606689453125,
      "logps/rejected": -452.1699523925781,
      "loss": 0.544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0270928144454956,
      "rewards/margins": 1.0332858562469482,
      "rewards/rejected": -2.0603785514831543,
      "step": 12054
    },
    {
      "epoch": 1.5774760775333279,
      "grad_norm": 6.387688159942627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13418835401535034,
      "logits/rejected": -0.2787593901157379,
      "logps/chosen": -490.6960754394531,
      "logps/rejected": -428.69573974609375,
      "loss": 0.2047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10395995527505875,
      "rewards/margins": 2.5191309452056885,
      "rewards/rejected": -2.6230907440185547,
      "step": 12055
    },
    {
      "epoch": 1.5776069354706796,
      "grad_norm": 6.419216156005859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12777230143547058,
      "logits/rejected": -0.132691890001297,
      "logps/chosen": -407.7785339355469,
      "logps/rejected": -435.7009582519531,
      "loss": 0.3,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1647759675979614,
      "rewards/margins": 1.204985499382019,
      "rewards/rejected": -2.3697614669799805,
      "step": 12056
    },
    {
      "epoch": 1.5777377934080314,
      "grad_norm": 15.500109672546387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1789158433675766,
      "logits/rejected": -0.03512084111571312,
      "logps/chosen": -416.9155578613281,
      "logps/rejected": -425.40924072265625,
      "loss": 0.7476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4602839946746826,
      "rewards/margins": 0.38914012908935547,
      "rewards/rejected": -1.8494240045547485,
      "step": 12057
    },
    {
      "epoch": 1.577868651345383,
      "grad_norm": 11.715239524841309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28465431928634644,
      "logits/rejected": -0.24229130148887634,
      "logps/chosen": -412.53790283203125,
      "logps/rejected": -334.41656494140625,
      "loss": 0.5201,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.08076810836792,
      "rewards/margins": 0.845553994178772,
      "rewards/rejected": -2.9263217449188232,
      "step": 12058
    },
    {
      "epoch": 1.5779995092827348,
      "grad_norm": 10.043647766113281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.452282190322876,
      "logits/rejected": -0.5530754923820496,
      "logps/chosen": -460.48492431640625,
      "logps/rejected": -429.8458557128906,
      "loss": 0.5174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5812690854072571,
      "rewards/margins": 0.6553776264190674,
      "rewards/rejected": -1.2366467714309692,
      "step": 12059
    },
    {
      "epoch": 1.5781303672200866,
      "grad_norm": 7.676651477813721,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09316622465848923,
      "logits/rejected": -0.21689437329769135,
      "logps/chosen": -553.5951538085938,
      "logps/rejected": -467.892578125,
      "loss": 0.2579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9706020355224609,
      "rewards/margins": 2.270421266555786,
      "rewards/rejected": -3.241023302078247,
      "step": 12060
    },
    {
      "epoch": 1.5782612251574384,
      "grad_norm": 14.187941551208496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33485347032546997,
      "logits/rejected": -0.54305499792099,
      "logps/chosen": -561.6866455078125,
      "logps/rejected": -501.873291015625,
      "loss": 0.6328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6489484310150146,
      "rewards/margins": 1.4271340370178223,
      "rewards/rejected": -3.076082706451416,
      "step": 12061
    },
    {
      "epoch": 1.5783920830947902,
      "grad_norm": 10.668529510498047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6397320032119751,
      "logits/rejected": -0.38030529022216797,
      "logps/chosen": -321.63543701171875,
      "logps/rejected": -499.5821228027344,
      "loss": 0.6911,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.413508653640747,
      "rewards/margins": 0.31574997305870056,
      "rewards/rejected": -1.72925865650177,
      "step": 12062
    },
    {
      "epoch": 1.578522941032142,
      "grad_norm": 5.943424701690674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44669288396835327,
      "logits/rejected": -0.43356379866600037,
      "logps/chosen": -334.51251220703125,
      "logps/rejected": -452.0221862792969,
      "loss": 0.2282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3916504383087158,
      "rewards/margins": 1.9719693660736084,
      "rewards/rejected": -3.3636200428009033,
      "step": 12063
    },
    {
      "epoch": 1.5786537989694938,
      "grad_norm": 14.064719200134277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31367939710617065,
      "logits/rejected": -0.5552499294281006,
      "logps/chosen": -687.0637817382812,
      "logps/rejected": -478.00634765625,
      "loss": 0.6369,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3765392303466797,
      "rewards/margins": 0.9511627554893494,
      "rewards/rejected": -2.327702045440674,
      "step": 12064
    },
    {
      "epoch": 1.5787846569068456,
      "grad_norm": 7.464825630187988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4463842511177063,
      "logits/rejected": -0.3885612487792969,
      "logps/chosen": -351.6374816894531,
      "logps/rejected": -419.9853515625,
      "loss": 0.3475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7121790051460266,
      "rewards/margins": 1.515730381011963,
      "rewards/rejected": -2.227909564971924,
      "step": 12065
    },
    {
      "epoch": 1.5789155148441973,
      "grad_norm": 8.462860107421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3123994469642639,
      "logits/rejected": -0.26930510997772217,
      "logps/chosen": -532.5279541015625,
      "logps/rejected": -401.0735168457031,
      "loss": 0.5301,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5266145467758179,
      "rewards/margins": 0.8422867059707642,
      "rewards/rejected": -2.368901252746582,
      "step": 12066
    },
    {
      "epoch": 1.5790463727815491,
      "grad_norm": 6.449829578399658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24191541969776154,
      "logits/rejected": -0.31182587146759033,
      "logps/chosen": -433.8033447265625,
      "logps/rejected": -429.58111572265625,
      "loss": 0.4578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5043526887893677,
      "rewards/margins": 0.8516042232513428,
      "rewards/rejected": -2.35595703125,
      "step": 12067
    },
    {
      "epoch": 1.579177230718901,
      "grad_norm": 12.35863208770752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4798862636089325,
      "logits/rejected": -0.44200900197029114,
      "logps/chosen": -348.9580078125,
      "logps/rejected": -332.48388671875,
      "loss": 0.712,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9993101358413696,
      "rewards/margins": 0.3416207432746887,
      "rewards/rejected": -2.340930700302124,
      "step": 12068
    },
    {
      "epoch": 1.5793080886562527,
      "grad_norm": 9.240741729736328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20053765177726746,
      "logits/rejected": -0.3548181653022766,
      "logps/chosen": -417.2625427246094,
      "logps/rejected": -345.831787109375,
      "loss": 0.6121,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.673017144203186,
      "rewards/margins": 0.32929670810699463,
      "rewards/rejected": -2.0023138523101807,
      "step": 12069
    },
    {
      "epoch": 1.5794389465936043,
      "grad_norm": 9.1333646774292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043991558253765106,
      "logits/rejected": -0.502476692199707,
      "logps/chosen": -474.2200927734375,
      "logps/rejected": -459.0752258300781,
      "loss": 0.5572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5608812570571899,
      "rewards/margins": 1.2273728847503662,
      "rewards/rejected": -1.7882541418075562,
      "step": 12070
    },
    {
      "epoch": 1.579569804530956,
      "grad_norm": 4.533941268920898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28823310136795044,
      "logits/rejected": -0.3514088988304138,
      "logps/chosen": -517.6926879882812,
      "logps/rejected": -367.5411376953125,
      "loss": 0.2084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4395868480205536,
      "rewards/margins": 2.1365408897399902,
      "rewards/rejected": -2.576127767562866,
      "step": 12071
    },
    {
      "epoch": 1.5797006624683079,
      "grad_norm": 11.946003913879395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47239407896995544,
      "logits/rejected": -0.19511520862579346,
      "logps/chosen": -512.061279296875,
      "logps/rejected": -463.89263916015625,
      "loss": 0.6303,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0364593267440796,
      "rewards/margins": 0.7008552551269531,
      "rewards/rejected": -1.7373145818710327,
      "step": 12072
    },
    {
      "epoch": 1.5798315204056594,
      "grad_norm": 7.715478420257568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3389813303947449,
      "logits/rejected": -0.5817213654518127,
      "logps/chosen": -412.68572998046875,
      "logps/rejected": -533.1187744140625,
      "loss": 0.2988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0183528661727905,
      "rewards/margins": 2.7837793827056885,
      "rewards/rejected": -3.8021321296691895,
      "step": 12073
    },
    {
      "epoch": 1.5799623783430112,
      "grad_norm": 7.618716239929199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.443621963262558,
      "logits/rejected": -0.2980242669582367,
      "logps/chosen": -424.5168762207031,
      "logps/rejected": -354.8795166015625,
      "loss": 0.4107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5887870788574219,
      "rewards/margins": 0.7313209772109985,
      "rewards/rejected": -2.32010817527771,
      "step": 12074
    },
    {
      "epoch": 1.580093236280363,
      "grad_norm": 10.839603424072266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47653478384017944,
      "logits/rejected": -0.47521233558654785,
      "logps/chosen": -392.8361511230469,
      "logps/rejected": -372.43780517578125,
      "loss": 0.455,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.747753918170929,
      "rewards/margins": 1.1740245819091797,
      "rewards/rejected": -1.9217785596847534,
      "step": 12075
    },
    {
      "epoch": 1.5802240942177148,
      "grad_norm": 10.58819580078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39920490980148315,
      "logits/rejected": -0.4575955867767334,
      "logps/chosen": -339.47735595703125,
      "logps/rejected": -312.833740234375,
      "loss": 0.614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9938711524009705,
      "rewards/margins": 0.8780981302261353,
      "rewards/rejected": -1.871969223022461,
      "step": 12076
    },
    {
      "epoch": 1.5803549521550666,
      "grad_norm": 9.5145845413208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5552469491958618,
      "logits/rejected": -0.3141353726387024,
      "logps/chosen": -345.17254638671875,
      "logps/rejected": -476.72265625,
      "loss": 0.4966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.184025526046753,
      "rewards/margins": 1.2160837650299072,
      "rewards/rejected": -2.4001095294952393,
      "step": 12077
    },
    {
      "epoch": 1.5804858100924184,
      "grad_norm": 3.4741029739379883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10324449837207794,
      "logits/rejected": -0.005138948559761047,
      "logps/chosen": -762.7689208984375,
      "logps/rejected": -627.1981201171875,
      "loss": 0.1042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1033713817596436,
      "rewards/margins": 2.905493974685669,
      "rewards/rejected": -1.802122712135315,
      "step": 12078
    },
    {
      "epoch": 1.5806166680297702,
      "grad_norm": 7.205311298370361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5418115854263306,
      "logits/rejected": -0.40796422958374023,
      "logps/chosen": -426.999755859375,
      "logps/rejected": -387.281982421875,
      "loss": 0.4779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6828857660293579,
      "rewards/margins": 0.8305017948150635,
      "rewards/rejected": -1.5133875608444214,
      "step": 12079
    },
    {
      "epoch": 1.580747525967122,
      "grad_norm": 4.172253131866455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06160128489136696,
      "logits/rejected": -0.3212083876132965,
      "logps/chosen": -570.635986328125,
      "logps/rejected": -374.68719482421875,
      "loss": 0.1294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.013540655374526978,
      "rewards/margins": 2.633638381958008,
      "rewards/rejected": -2.647179126739502,
      "step": 12080
    },
    {
      "epoch": 1.5808783839044738,
      "grad_norm": 9.29824161529541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14489468932151794,
      "logits/rejected": -0.09194698929786682,
      "logps/chosen": -443.85040283203125,
      "logps/rejected": -458.5897216796875,
      "loss": 0.3796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.611802339553833,
      "rewards/margins": 1.658384084701538,
      "rewards/rejected": -2.270186424255371,
      "step": 12081
    },
    {
      "epoch": 1.5810092418418256,
      "grad_norm": 6.658335208892822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35681241750717163,
      "logits/rejected": -0.46091747283935547,
      "logps/chosen": -401.54473876953125,
      "logps/rejected": -531.8773803710938,
      "loss": 0.2976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2259284257888794,
      "rewards/margins": 1.5965243577957153,
      "rewards/rejected": -2.822453022003174,
      "step": 12082
    },
    {
      "epoch": 1.5811400997791774,
      "grad_norm": 6.360199928283691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06938835233449936,
      "logits/rejected": -0.3742958903312683,
      "logps/chosen": -543.3690795898438,
      "logps/rejected": -388.88916015625,
      "loss": 0.2692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20117121934890747,
      "rewards/margins": 2.239938259124756,
      "rewards/rejected": -2.038767099380493,
      "step": 12083
    },
    {
      "epoch": 1.5812709577165291,
      "grad_norm": 11.33871841430664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42013680934906006,
      "logits/rejected": -0.5041930675506592,
      "logps/chosen": -335.5157775878906,
      "logps/rejected": -301.9921875,
      "loss": 0.8193,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4648973941802979,
      "rewards/margins": 0.5048933029174805,
      "rewards/rejected": -1.9697908163070679,
      "step": 12084
    },
    {
      "epoch": 1.5814018156538807,
      "grad_norm": 7.586859703063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4696507453918457,
      "logits/rejected": -0.6342519521713257,
      "logps/chosen": -430.9544677734375,
      "logps/rejected": -398.883544921875,
      "loss": 0.4492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7055009603500366,
      "rewards/margins": 1.1659698486328125,
      "rewards/rejected": -1.8714709281921387,
      "step": 12085
    },
    {
      "epoch": 1.5815326735912325,
      "grad_norm": 11.839738845825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034546978771686554,
      "logits/rejected": -0.16670164465904236,
      "logps/chosen": -346.72698974609375,
      "logps/rejected": -474.630615234375,
      "loss": 0.6247,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0334203243255615,
      "rewards/margins": 0.7328518629074097,
      "rewards/rejected": -1.7662723064422607,
      "step": 12086
    },
    {
      "epoch": 1.5816635315285843,
      "grad_norm": 13.227726936340332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.532118558883667,
      "logits/rejected": -0.5802744626998901,
      "logps/chosen": -407.48992919921875,
      "logps/rejected": -475.36163330078125,
      "loss": 0.8589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0516554117202759,
      "rewards/margins": 0.40770024061203003,
      "rewards/rejected": -1.4593557119369507,
      "step": 12087
    },
    {
      "epoch": 1.581794389465936,
      "grad_norm": 5.0585246086120605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09251660108566284,
      "logits/rejected": -0.17134416103363037,
      "logps/chosen": -391.80242919921875,
      "logps/rejected": -383.592529296875,
      "loss": 0.2277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4822610914707184,
      "rewards/margins": 1.85142183303833,
      "rewards/rejected": -2.3336830139160156,
      "step": 12088
    },
    {
      "epoch": 1.5819252474032877,
      "grad_norm": 15.036191940307617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4180607497692108,
      "logits/rejected": -0.5183483362197876,
      "logps/chosen": -440.48162841796875,
      "logps/rejected": -482.60296630859375,
      "loss": 0.6108,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6326475143432617,
      "rewards/margins": 1.248001217842102,
      "rewards/rejected": -2.880648374557495,
      "step": 12089
    },
    {
      "epoch": 1.5820561053406395,
      "grad_norm": 9.269970893859863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10360221564769745,
      "logits/rejected": -0.29174965620040894,
      "logps/chosen": -453.69097900390625,
      "logps/rejected": -524.3714599609375,
      "loss": 0.4687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12475837022066116,
      "rewards/margins": 1.3415844440460205,
      "rewards/rejected": -1.4663430452346802,
      "step": 12090
    },
    {
      "epoch": 1.5821869632779912,
      "grad_norm": 11.355331420898438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2062554806470871,
      "logits/rejected": -0.16377627849578857,
      "logps/chosen": -578.833984375,
      "logps/rejected": -562.3599243164062,
      "loss": 0.5377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09127084910869598,
      "rewards/margins": 1.000849962234497,
      "rewards/rejected": -1.09212064743042,
      "step": 12091
    },
    {
      "epoch": 1.582317821215343,
      "grad_norm": 8.209332466125488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44318342208862305,
      "logits/rejected": -0.6977376341819763,
      "logps/chosen": -505.3175048828125,
      "logps/rejected": -487.86322021484375,
      "loss": 0.3824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3519926071166992,
      "rewards/margins": 1.0413286685943604,
      "rewards/rejected": -1.3933212757110596,
      "step": 12092
    },
    {
      "epoch": 1.5824486791526948,
      "grad_norm": 9.49608325958252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3189495801925659,
      "logits/rejected": -0.2431100457906723,
      "logps/chosen": -520.3357543945312,
      "logps/rejected": -526.8684692382812,
      "loss": 0.3725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4284528493881226,
      "rewards/margins": 1.7199290990829468,
      "rewards/rejected": -3.1483817100524902,
      "step": 12093
    },
    {
      "epoch": 1.5825795370900466,
      "grad_norm": 12.4155912399292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43327805399894714,
      "logits/rejected": -0.42146140336990356,
      "logps/chosen": -617.8641357421875,
      "logps/rejected": -684.7658081054688,
      "loss": 0.7622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3778858184814453,
      "rewards/margins": 0.6137411594390869,
      "rewards/rejected": -2.9916272163391113,
      "step": 12094
    },
    {
      "epoch": 1.5827103950273984,
      "grad_norm": 4.692709922790527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.276233971118927,
      "logits/rejected": -0.06821537017822266,
      "logps/chosen": -326.7373962402344,
      "logps/rejected": -562.6217651367188,
      "loss": 0.2286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8460994362831116,
      "rewards/margins": 2.0245871543884277,
      "rewards/rejected": -2.8706867694854736,
      "step": 12095
    },
    {
      "epoch": 1.5828412529647502,
      "grad_norm": 4.4169511795043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42113196849823,
      "logits/rejected": -0.2905313968658447,
      "logps/chosen": -483.8715515136719,
      "logps/rejected": -436.2152099609375,
      "loss": 0.2636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0106298923492432,
      "rewards/margins": 1.5865422487258911,
      "rewards/rejected": -2.5971720218658447,
      "step": 12096
    },
    {
      "epoch": 1.582972110902102,
      "grad_norm": 6.209376811981201,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04080462455749512,
      "logits/rejected": -0.18061837553977966,
      "logps/chosen": -471.5697021484375,
      "logps/rejected": -490.07940673828125,
      "loss": 0.3002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9897432327270508,
      "rewards/margins": 2.209866523742676,
      "rewards/rejected": -3.1996099948883057,
      "step": 12097
    },
    {
      "epoch": 1.5831029688394538,
      "grad_norm": 12.6949462890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21396192908287048,
      "logits/rejected": -0.23504260182380676,
      "logps/chosen": -406.28717041015625,
      "logps/rejected": -425.95263671875,
      "loss": 0.8203,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6471260786056519,
      "rewards/margins": 0.00747014582157135,
      "rewards/rejected": -1.6545960903167725,
      "step": 12098
    },
    {
      "epoch": 1.5832338267768056,
      "grad_norm": 10.428829193115234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24066339433193207,
      "logits/rejected": -0.5462898015975952,
      "logps/chosen": -627.2008666992188,
      "logps/rejected": -426.6672058105469,
      "loss": 0.4476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0017764568328857,
      "rewards/margins": 1.2529585361480713,
      "rewards/rejected": -2.254735231399536,
      "step": 12099
    },
    {
      "epoch": 1.5833646847141571,
      "grad_norm": 7.070333003997803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39612245559692383,
      "logits/rejected": -0.48019149899482727,
      "logps/chosen": -467.8288269042969,
      "logps/rejected": -461.2722473144531,
      "loss": 0.5404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1861146688461304,
      "rewards/margins": 0.8881384134292603,
      "rewards/rejected": -2.0742530822753906,
      "step": 12100
    },
    {
      "epoch": 1.583495542651509,
      "grad_norm": 5.9111199378967285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0965818241238594,
      "logits/rejected": -0.23406460881233215,
      "logps/chosen": -425.6918029785156,
      "logps/rejected": -389.5396423339844,
      "loss": 0.4245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4549255669116974,
      "rewards/margins": 1.4375455379486084,
      "rewards/rejected": -1.8924710750579834,
      "step": 12101
    },
    {
      "epoch": 1.5836264005888607,
      "grad_norm": 5.707487106323242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13427934050559998,
      "logits/rejected": -0.08859914541244507,
      "logps/chosen": -384.2674560546875,
      "logps/rejected": -516.980712890625,
      "loss": 0.2425,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37047672271728516,
      "rewards/margins": 1.6858114004135132,
      "rewards/rejected": -2.056288242340088,
      "step": 12102
    },
    {
      "epoch": 1.5837572585262125,
      "grad_norm": 11.938097953796387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17208513617515564,
      "logits/rejected": -0.2297002077102661,
      "logps/chosen": -532.2093505859375,
      "logps/rejected": -500.69488525390625,
      "loss": 0.5409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7881126403808594,
      "rewards/margins": 1.4642629623413086,
      "rewards/rejected": -2.252375602722168,
      "step": 12103
    },
    {
      "epoch": 1.583888116463564,
      "grad_norm": 11.260064125061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07935956865549088,
      "logits/rejected": -0.11923874914646149,
      "logps/chosen": -668.7174682617188,
      "logps/rejected": -559.8642578125,
      "loss": 0.5603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5120469927787781,
      "rewards/margins": 1.4268088340759277,
      "rewards/rejected": -1.938855767250061,
      "step": 12104
    },
    {
      "epoch": 1.5840189744009159,
      "grad_norm": 13.391011238098145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18493153154850006,
      "logits/rejected": -0.3473774790763855,
      "logps/chosen": -374.1202697753906,
      "logps/rejected": -367.755615234375,
      "loss": 0.7554,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.629726529121399,
      "rewards/margins": 0.04097484052181244,
      "rewards/rejected": -1.670701265335083,
      "step": 12105
    },
    {
      "epoch": 1.5841498323382677,
      "grad_norm": 9.72796630859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5889639258384705,
      "logits/rejected": -0.3282160758972168,
      "logps/chosen": -531.15283203125,
      "logps/rejected": -658.85693359375,
      "loss": 0.4863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6409691572189331,
      "rewards/margins": 2.1612415313720703,
      "rewards/rejected": -2.802210807800293,
      "step": 12106
    },
    {
      "epoch": 1.5842806902756195,
      "grad_norm": 15.208534240722656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042394235730171204,
      "logits/rejected": -0.3423527479171753,
      "logps/chosen": -627.3614501953125,
      "logps/rejected": -552.1088256835938,
      "loss": 0.6969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6337148547172546,
      "rewards/margins": 1.1179406642913818,
      "rewards/rejected": -1.7516553401947021,
      "step": 12107
    },
    {
      "epoch": 1.5844115482129713,
      "grad_norm": 8.43718147277832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21508100628852844,
      "logits/rejected": -0.4000450074672699,
      "logps/chosen": -463.74444580078125,
      "logps/rejected": -388.14300537109375,
      "loss": 0.4352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5315680503845215,
      "rewards/margins": 1.2598536014556885,
      "rewards/rejected": -2.791421890258789,
      "step": 12108
    },
    {
      "epoch": 1.584542406150323,
      "grad_norm": 8.993789672851562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2866203784942627,
      "logits/rejected": -0.36016717553138733,
      "logps/chosen": -344.3277282714844,
      "logps/rejected": -396.4501647949219,
      "loss": 0.5834,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.051957495510578156,
      "rewards/margins": 1.274704933166504,
      "rewards/rejected": -1.326662540435791,
      "step": 12109
    },
    {
      "epoch": 1.5846732640876748,
      "grad_norm": 8.409095764160156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2850411534309387,
      "logits/rejected": -0.5563274025917053,
      "logps/chosen": -577.5648193359375,
      "logps/rejected": -465.8306884765625,
      "loss": 0.2798,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1418049335479736,
      "rewards/margins": 2.2813339233398438,
      "rewards/rejected": -3.4231390953063965,
      "step": 12110
    },
    {
      "epoch": 1.5848041220250266,
      "grad_norm": 8.316778182983398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3515583574771881,
      "logits/rejected": -0.4972628057003021,
      "logps/chosen": -508.1964111328125,
      "logps/rejected": -471.1150207519531,
      "loss": 0.4113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2481670379638672,
      "rewards/margins": 1.8829065561294556,
      "rewards/rejected": -3.1310737133026123,
      "step": 12111
    },
    {
      "epoch": 1.5849349799623784,
      "grad_norm": 9.456588745117188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45984286069869995,
      "logits/rejected": -0.2139361947774887,
      "logps/chosen": -648.8587646484375,
      "logps/rejected": -420.17559814453125,
      "loss": 0.402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.335294485092163,
      "rewards/margins": 1.3312525749206543,
      "rewards/rejected": -2.6665470600128174,
      "step": 12112
    },
    {
      "epoch": 1.5850658378997302,
      "grad_norm": 15.028836250305176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05980853736400604,
      "logits/rejected": -0.21797500550746918,
      "logps/chosen": -397.1131896972656,
      "logps/rejected": -352.5327453613281,
      "loss": 0.8037,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4781546592712402,
      "rewards/margins": 0.7498067021369934,
      "rewards/rejected": -2.227961540222168,
      "step": 12113
    },
    {
      "epoch": 1.585196695837082,
      "grad_norm": 7.426340579986572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21424707770347595,
      "logits/rejected": -0.4999784231185913,
      "logps/chosen": -372.65216064453125,
      "logps/rejected": -361.8949279785156,
      "loss": 0.4387,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8364626169204712,
      "rewards/margins": 1.1555336713790894,
      "rewards/rejected": -1.99199640750885,
      "step": 12114
    },
    {
      "epoch": 1.5853275537744338,
      "grad_norm": 6.691139221191406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44018447399139404,
      "logits/rejected": -0.2364148497581482,
      "logps/chosen": -388.31195068359375,
      "logps/rejected": -395.76226806640625,
      "loss": 0.4525,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7433727979660034,
      "rewards/margins": 1.325803279876709,
      "rewards/rejected": -2.069176197052002,
      "step": 12115
    },
    {
      "epoch": 1.5854584117117854,
      "grad_norm": 9.64513111114502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3134448528289795,
      "logits/rejected": -0.4060983955860138,
      "logps/chosen": -581.8046264648438,
      "logps/rejected": -417.1565246582031,
      "loss": 0.5669,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1608104705810547,
      "rewards/margins": 1.141505479812622,
      "rewards/rejected": -1.3023159503936768,
      "step": 12116
    },
    {
      "epoch": 1.5855892696491372,
      "grad_norm": 8.485137939453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14875569939613342,
      "logits/rejected": -0.20899975299835205,
      "logps/chosen": -438.6676025390625,
      "logps/rejected": -273.6279602050781,
      "loss": 0.6514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5581270456314087,
      "rewards/margins": 0.6134656667709351,
      "rewards/rejected": -2.1715927124023438,
      "step": 12117
    },
    {
      "epoch": 1.585720127586489,
      "grad_norm": 12.532702445983887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3660116195678711,
      "logits/rejected": -0.25363588333129883,
      "logps/chosen": -525.7185668945312,
      "logps/rejected": -463.951904296875,
      "loss": 0.4988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.155203104019165,
      "rewards/margins": 1.2424947023391724,
      "rewards/rejected": -2.397697687149048,
      "step": 12118
    },
    {
      "epoch": 1.5858509855238405,
      "grad_norm": 7.136266231536865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4555657207965851,
      "logits/rejected": -0.2107679843902588,
      "logps/chosen": -423.681396484375,
      "logps/rejected": -358.28814697265625,
      "loss": 0.3889,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9239420294761658,
      "rewards/margins": 1.617990493774414,
      "rewards/rejected": -2.5419325828552246,
      "step": 12119
    },
    {
      "epoch": 1.5859818434611923,
      "grad_norm": 10.291533470153809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.280392587184906,
      "logits/rejected": -0.4576079845428467,
      "logps/chosen": -538.4996337890625,
      "logps/rejected": -503.658447265625,
      "loss": 0.532,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2137882709503174,
      "rewards/margins": 0.7900558114051819,
      "rewards/rejected": -2.0038440227508545,
      "step": 12120
    },
    {
      "epoch": 1.586112701398544,
      "grad_norm": 6.858623027801514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27259737253189087,
      "logits/rejected": -0.3709040880203247,
      "logps/chosen": -452.268310546875,
      "logps/rejected": -354.3421936035156,
      "loss": 0.4594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7368608117103577,
      "rewards/margins": 0.9897698163986206,
      "rewards/rejected": -1.7266309261322021,
      "step": 12121
    },
    {
      "epoch": 1.586243559335896,
      "grad_norm": 8.956796646118164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058329563587903976,
      "logits/rejected": 0.0018156282603740692,
      "logps/chosen": -421.0630798339844,
      "logps/rejected": -433.5938720703125,
      "loss": 0.5055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11767902970314026,
      "rewards/margins": 1.1993647813796997,
      "rewards/rejected": -1.3170439004898071,
      "step": 12122
    },
    {
      "epoch": 1.5863744172732477,
      "grad_norm": 11.447504043579102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25116217136383057,
      "logits/rejected": -0.2884105145931244,
      "logps/chosen": -479.78973388671875,
      "logps/rejected": -496.2969665527344,
      "loss": 0.5422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5550612211227417,
      "rewards/margins": 1.1940950155258179,
      "rewards/rejected": -2.7491559982299805,
      "step": 12123
    },
    {
      "epoch": 1.5865052752105995,
      "grad_norm": 8.071516036987305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1468820571899414,
      "logits/rejected": -0.10232807695865631,
      "logps/chosen": -627.2285766601562,
      "logps/rejected": -512.9293823242188,
      "loss": 0.3247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19882506132125854,
      "rewards/margins": 1.485277771949768,
      "rewards/rejected": -1.2864528894424438,
      "step": 12124
    },
    {
      "epoch": 1.5866361331479513,
      "grad_norm": 4.9097676277160645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3759794533252716,
      "logits/rejected": -0.4047600030899048,
      "logps/chosen": -445.8604736328125,
      "logps/rejected": -419.8509826660156,
      "loss": 0.2391,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6558006405830383,
      "rewards/margins": 1.7791966199874878,
      "rewards/rejected": -2.434997081756592,
      "step": 12125
    },
    {
      "epoch": 1.586766991085303,
      "grad_norm": 6.91312837600708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01684480719268322,
      "logits/rejected": -0.20955701172351837,
      "logps/chosen": -292.2415466308594,
      "logps/rejected": -413.18548583984375,
      "loss": 0.5415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0153738260269165,
      "rewards/margins": 1.138906717300415,
      "rewards/rejected": -2.154280662536621,
      "step": 12126
    },
    {
      "epoch": 1.5868978490226548,
      "grad_norm": 9.499920845031738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.239231139421463,
      "logits/rejected": -0.3357413113117218,
      "logps/chosen": -414.93792724609375,
      "logps/rejected": -398.7696228027344,
      "loss": 0.4829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6563029885292053,
      "rewards/margins": 1.310776948928833,
      "rewards/rejected": -1.967079997062683,
      "step": 12127
    },
    {
      "epoch": 1.5870287069600066,
      "grad_norm": 5.170044898986816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4305618703365326,
      "logits/rejected": -0.5138826370239258,
      "logps/chosen": -315.81634521484375,
      "logps/rejected": -440.9522399902344,
      "loss": 0.3164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2219820022583008,
      "rewards/margins": 1.4957354068756104,
      "rewards/rejected": -2.717717409133911,
      "step": 12128
    },
    {
      "epoch": 1.5871595648973584,
      "grad_norm": 4.4553399085998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32483744621276855,
      "logits/rejected": -0.1679920256137848,
      "logps/chosen": -354.0684814453125,
      "logps/rejected": -423.88922119140625,
      "loss": 0.2124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07181987166404724,
      "rewards/margins": 1.9129011631011963,
      "rewards/rejected": -1.984721064567566,
      "step": 12129
    },
    {
      "epoch": 1.5872904228347102,
      "grad_norm": 8.983135223388672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11169681698083878,
      "logits/rejected": 0.01327441819012165,
      "logps/chosen": -295.18231201171875,
      "logps/rejected": -361.8647766113281,
      "loss": 0.5502,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2918243408203125,
      "rewards/margins": 0.7469409704208374,
      "rewards/rejected": -2.0387651920318604,
      "step": 12130
    },
    {
      "epoch": 1.5874212807720618,
      "grad_norm": 8.530427932739258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016817031428217888,
      "logits/rejected": -0.15019413828849792,
      "logps/chosen": -633.3526611328125,
      "logps/rejected": -436.71038818359375,
      "loss": 0.4799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1822963953018188,
      "rewards/margins": 1.2843271493911743,
      "rewards/rejected": -2.466623544692993,
      "step": 12131
    },
    {
      "epoch": 1.5875521387094136,
      "grad_norm": 11.659214973449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1077764630317688,
      "logits/rejected": -0.12619082629680634,
      "logps/chosen": -568.540283203125,
      "logps/rejected": -408.45849609375,
      "loss": 0.3686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1517002582550049,
      "rewards/margins": 1.1712963581085205,
      "rewards/rejected": -2.3229963779449463,
      "step": 12132
    },
    {
      "epoch": 1.5876829966467654,
      "grad_norm": 10.29178237915039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08332701027393341,
      "logits/rejected": -0.18257038295269012,
      "logps/chosen": -387.5334167480469,
      "logps/rejected": -426.4089660644531,
      "loss": 0.6696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9771227240562439,
      "rewards/margins": 0.3940418064594269,
      "rewards/rejected": -1.3711645603179932,
      "step": 12133
    },
    {
      "epoch": 1.5878138545841172,
      "grad_norm": 5.933955669403076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05402621254324913,
      "logits/rejected": -0.34790652990341187,
      "logps/chosen": -548.9385986328125,
      "logps/rejected": -392.9427490234375,
      "loss": 0.2985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4386320114135742,
      "rewards/margins": 1.4025224447250366,
      "rewards/rejected": -1.8411544561386108,
      "step": 12134
    },
    {
      "epoch": 1.5879447125214687,
      "grad_norm": 7.621493339538574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28732579946517944,
      "logits/rejected": -0.3308267593383789,
      "logps/chosen": -430.0062255859375,
      "logps/rejected": -455.6131286621094,
      "loss": 0.4231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8352896571159363,
      "rewards/margins": 0.9344356656074524,
      "rewards/rejected": -1.7697252035140991,
      "step": 12135
    },
    {
      "epoch": 1.5880755704588205,
      "grad_norm": 6.069456100463867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10638898611068726,
      "logits/rejected": -0.291826456785202,
      "logps/chosen": -694.9010009765625,
      "logps/rejected": -585.4982299804688,
      "loss": 0.2675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6881816983222961,
      "rewards/margins": 1.7107254266738892,
      "rewards/rejected": -2.39890718460083,
      "step": 12136
    },
    {
      "epoch": 1.5882064283961723,
      "grad_norm": 8.278465270996094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4477570950984955,
      "logits/rejected": -0.3619956076145172,
      "logps/chosen": -402.68896484375,
      "logps/rejected": -439.3206787109375,
      "loss": 0.4828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6093398332595825,
      "rewards/margins": 1.0816737413406372,
      "rewards/rejected": -2.6910133361816406,
      "step": 12137
    },
    {
      "epoch": 1.5883372863335241,
      "grad_norm": 9.440238952636719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02217470109462738,
      "logits/rejected": -0.17488481104373932,
      "logps/chosen": -506.6266784667969,
      "logps/rejected": -473.7424011230469,
      "loss": 0.632,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.032648831605911255,
      "rewards/margins": 0.6331764459609985,
      "rewards/rejected": -0.6005275845527649,
      "step": 12138
    },
    {
      "epoch": 1.588468144270876,
      "grad_norm": 10.775150299072266,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0164545439183712,
      "logits/rejected": -0.1561269909143448,
      "logps/chosen": -622.67919921875,
      "logps/rejected": -488.3189697265625,
      "loss": 0.6084,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19864901900291443,
      "rewards/margins": 1.1327396631240845,
      "rewards/rejected": -1.3313885927200317,
      "step": 12139
    },
    {
      "epoch": 1.5885990022082277,
      "grad_norm": 10.48799991607666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4359600841999054,
      "logits/rejected": -0.63069748878479,
      "logps/chosen": -436.6252136230469,
      "logps/rejected": -333.1154479980469,
      "loss": 0.7937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9530725479125977,
      "rewards/margins": 0.11483177542686462,
      "rewards/rejected": -2.067904233932495,
      "step": 12140
    },
    {
      "epoch": 1.5887298601455795,
      "grad_norm": 10.671992301940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07487861812114716,
      "logits/rejected": -0.08576521277427673,
      "logps/chosen": -491.9449462890625,
      "logps/rejected": -335.40869140625,
      "loss": 0.5284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5248563885688782,
      "rewards/margins": 1.6633939743041992,
      "rewards/rejected": -2.1882503032684326,
      "step": 12141
    },
    {
      "epoch": 1.5888607180829313,
      "grad_norm": 14.428740501403809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12554597854614258,
      "logits/rejected": -0.31808027625083923,
      "logps/chosen": -652.779052734375,
      "logps/rejected": -512.4923706054688,
      "loss": 0.6401,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9307223558425903,
      "rewards/margins": 1.0915409326553345,
      "rewards/rejected": -2.022263288497925,
      "step": 12142
    },
    {
      "epoch": 1.588991576020283,
      "grad_norm": 8.612414360046387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5030034184455872,
      "logits/rejected": -0.2146461009979248,
      "logps/chosen": -321.3497314453125,
      "logps/rejected": -453.0179748535156,
      "loss": 0.5425,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6038431525230408,
      "rewards/margins": 0.9311121702194214,
      "rewards/rejected": -1.5349552631378174,
      "step": 12143
    },
    {
      "epoch": 1.5891224339576349,
      "grad_norm": 11.226086616516113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018894046545028687,
      "logits/rejected": -0.22504347562789917,
      "logps/chosen": -716.3966674804688,
      "logps/rejected": -598.1954345703125,
      "loss": 0.5211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23041878640651703,
      "rewards/margins": 0.9285814762115479,
      "rewards/rejected": -1.159000277519226,
      "step": 12144
    },
    {
      "epoch": 1.5892532918949867,
      "grad_norm": 9.093667030334473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06297224760055542,
      "logits/rejected": -0.07046901434659958,
      "logps/chosen": -554.8867797851562,
      "logps/rejected": -461.8187255859375,
      "loss": 0.616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.379067063331604,
      "rewards/margins": 0.5168358087539673,
      "rewards/rejected": -1.8959028720855713,
      "step": 12145
    },
    {
      "epoch": 1.5893841498323382,
      "grad_norm": 10.760210990905762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3153701722621918,
      "logits/rejected": -0.3337361514568329,
      "logps/chosen": -323.27178955078125,
      "logps/rejected": -430.0892333984375,
      "loss": 0.6913,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1772772073745728,
      "rewards/margins": 1.1773858070373535,
      "rewards/rejected": -2.3546628952026367,
      "step": 12146
    },
    {
      "epoch": 1.58951500776969,
      "grad_norm": 5.769914627075195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3419640362262726,
      "logits/rejected": -0.12237191945314407,
      "logps/chosen": -417.52777099609375,
      "logps/rejected": -374.1166076660156,
      "loss": 0.2738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7187190651893616,
      "rewards/margins": 1.8359003067016602,
      "rewards/rejected": -2.554619312286377,
      "step": 12147
    },
    {
      "epoch": 1.5896458657070418,
      "grad_norm": 4.446997165679932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14872601628303528,
      "logits/rejected": 0.04044707864522934,
      "logps/chosen": -407.76373291015625,
      "logps/rejected": -442.36956787109375,
      "loss": 0.3021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.554943323135376,
      "rewards/margins": 1.6722882986068726,
      "rewards/rejected": -2.227231740951538,
      "step": 12148
    },
    {
      "epoch": 1.5897767236443936,
      "grad_norm": 11.90849494934082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22174370288848877,
      "logits/rejected": -0.38125234842300415,
      "logps/chosen": -565.2100830078125,
      "logps/rejected": -365.1999206542969,
      "loss": 0.7445,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5605272650718689,
      "rewards/margins": 0.5237158536911011,
      "rewards/rejected": -1.0842430591583252,
      "step": 12149
    },
    {
      "epoch": 1.5899075815817452,
      "grad_norm": 12.0825777053833,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2657606601715088,
      "logits/rejected": -0.14774054288864136,
      "logps/chosen": -439.2027282714844,
      "logps/rejected": -485.72613525390625,
      "loss": 0.6374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5038639307022095,
      "rewards/margins": 0.8423259854316711,
      "rewards/rejected": -1.3461898565292358,
      "step": 12150
    },
    {
      "epoch": 1.590038439519097,
      "grad_norm": 6.225100517272949,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2985922694206238,
      "logits/rejected": -0.06590361893177032,
      "logps/chosen": -599.9124145507812,
      "logps/rejected": -401.2880554199219,
      "loss": 0.2199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.275556355714798,
      "rewards/margins": 1.9942864179611206,
      "rewards/rejected": -2.2698428630828857,
      "step": 12151
    },
    {
      "epoch": 1.5901692974564487,
      "grad_norm": 15.010910034179688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34653350710868835,
      "logits/rejected": -0.18233910202980042,
      "logps/chosen": -503.95404052734375,
      "logps/rejected": -492.53863525390625,
      "loss": 0.9417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0310912132263184,
      "rewards/margins": 0.36120733618736267,
      "rewards/rejected": -1.3922985792160034,
      "step": 12152
    },
    {
      "epoch": 1.5903001553938005,
      "grad_norm": 13.332584381103516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36797478795051575,
      "logits/rejected": -0.40534132719039917,
      "logps/chosen": -371.56024169921875,
      "logps/rejected": -391.4741516113281,
      "loss": 0.7966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9691884517669678,
      "rewards/margins": 0.580375075340271,
      "rewards/rejected": -2.5495636463165283,
      "step": 12153
    },
    {
      "epoch": 1.5904310133311523,
      "grad_norm": 11.149656295776367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12498936057090759,
      "logits/rejected": 0.023657239973545074,
      "logps/chosen": -433.62353515625,
      "logps/rejected": -504.4493408203125,
      "loss": 0.6342,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0819196701049805,
      "rewards/margins": 0.8153898119926453,
      "rewards/rejected": -1.8973095417022705,
      "step": 12154
    },
    {
      "epoch": 1.5905618712685041,
      "grad_norm": 5.812382221221924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19564580917358398,
      "logits/rejected": -0.20407770574092865,
      "logps/chosen": -508.40411376953125,
      "logps/rejected": -427.14678955078125,
      "loss": 0.2763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6489053964614868,
      "rewards/margins": 1.3614217042922974,
      "rewards/rejected": -2.010327100753784,
      "step": 12155
    },
    {
      "epoch": 1.590692729205856,
      "grad_norm": 9.931756973266602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1763257384300232,
      "logits/rejected": -0.03518734499812126,
      "logps/chosen": -505.4212341308594,
      "logps/rejected": -630.776611328125,
      "loss": 0.4519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5161478519439697,
      "rewards/margins": 2.1460421085357666,
      "rewards/rejected": -3.6621897220611572,
      "step": 12156
    },
    {
      "epoch": 1.5908235871432077,
      "grad_norm": 6.50807523727417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30385622382164,
      "logits/rejected": -0.3994998633861542,
      "logps/chosen": -364.1164855957031,
      "logps/rejected": -359.38140869140625,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9253511428833008,
      "rewards/margins": 1.397884726524353,
      "rewards/rejected": -2.3232359886169434,
      "step": 12157
    },
    {
      "epoch": 1.5909544450805595,
      "grad_norm": 10.275283813476562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39947235584259033,
      "logits/rejected": -0.36397111415863037,
      "logps/chosen": -445.824462890625,
      "logps/rejected": -506.60870361328125,
      "loss": 0.5637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0354726314544678,
      "rewards/margins": 0.8076765537261963,
      "rewards/rejected": -1.843149185180664,
      "step": 12158
    },
    {
      "epoch": 1.5910853030179113,
      "grad_norm": 11.187444686889648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16872239112854004,
      "logits/rejected": -0.2920386791229248,
      "logps/chosen": -554.5282592773438,
      "logps/rejected": -500.3863525390625,
      "loss": 0.5498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6101358532905579,
      "rewards/margins": 0.9648460149765015,
      "rewards/rejected": -1.574981927871704,
      "step": 12159
    },
    {
      "epoch": 1.591216160955263,
      "grad_norm": 7.401239395141602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31227850914001465,
      "logits/rejected": -0.1952318251132965,
      "logps/chosen": -376.2117004394531,
      "logps/rejected": -441.360595703125,
      "loss": 0.2955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.26273444294929504,
      "rewards/margins": 1.8826990127563477,
      "rewards/rejected": -2.1454334259033203,
      "step": 12160
    },
    {
      "epoch": 1.5913470188926147,
      "grad_norm": 6.154369831085205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24255792796611786,
      "logits/rejected": -0.36123883724212646,
      "logps/chosen": -404.1825256347656,
      "logps/rejected": -370.4241943359375,
      "loss": 0.329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9587920904159546,
      "rewards/margins": 1.4144237041473389,
      "rewards/rejected": -2.373215913772583,
      "step": 12161
    },
    {
      "epoch": 1.5914778768299664,
      "grad_norm": 7.695044994354248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22555552423000336,
      "logits/rejected": -0.5384819507598877,
      "logps/chosen": -445.14532470703125,
      "logps/rejected": -462.1645812988281,
      "loss": 0.3774,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5100345611572266,
      "rewards/margins": 1.6221503019332886,
      "rewards/rejected": -2.1321847438812256,
      "step": 12162
    },
    {
      "epoch": 1.5916087347673182,
      "grad_norm": 4.302084922790527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.359749436378479,
      "logits/rejected": -0.49196112155914307,
      "logps/chosen": -363.8901672363281,
      "logps/rejected": -545.8930053710938,
      "loss": 0.1372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1701738834381104,
      "rewards/margins": 2.863752603530884,
      "rewards/rejected": -4.033926486968994,
      "step": 12163
    },
    {
      "epoch": 1.59173959270467,
      "grad_norm": 9.755865097045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.326264888048172,
      "logits/rejected": -0.29947608709335327,
      "logps/chosen": -774.8209838867188,
      "logps/rejected": -507.6581726074219,
      "loss": 0.4856,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4332807958126068,
      "rewards/margins": 1.1178503036499023,
      "rewards/rejected": -1.551131010055542,
      "step": 12164
    },
    {
      "epoch": 1.5918704506420216,
      "grad_norm": 7.030782699584961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06572512537240982,
      "logits/rejected": -0.26236996054649353,
      "logps/chosen": -466.4172058105469,
      "logps/rejected": -368.6914367675781,
      "loss": 0.3385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37688982486724854,
      "rewards/margins": 2.604503870010376,
      "rewards/rejected": -2.981393814086914,
      "step": 12165
    },
    {
      "epoch": 1.5920013085793734,
      "grad_norm": 8.105957984924316,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26817768812179565,
      "logits/rejected": -0.2435634732246399,
      "logps/chosen": -526.7285766601562,
      "logps/rejected": -414.5616149902344,
      "loss": 0.3909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07595445215702057,
      "rewards/margins": 1.7260425090789795,
      "rewards/rejected": -1.8019969463348389,
      "step": 12166
    },
    {
      "epoch": 1.5921321665167252,
      "grad_norm": 9.910419464111328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11315082758665085,
      "logits/rejected": -0.23389507830142975,
      "logps/chosen": -484.6379089355469,
      "logps/rejected": -443.34228515625,
      "loss": 0.3675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7026050686836243,
      "rewards/margins": 1.3481736183166504,
      "rewards/rejected": -2.05077862739563,
      "step": 12167
    },
    {
      "epoch": 1.592263024454077,
      "grad_norm": 9.563017845153809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1226683259010315,
      "logits/rejected": -0.23998457193374634,
      "logps/chosen": -368.9144287109375,
      "logps/rejected": -350.59356689453125,
      "loss": 0.5576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0660667419433594,
      "rewards/margins": 0.4985630214214325,
      "rewards/rejected": -1.5646297931671143,
      "step": 12168
    },
    {
      "epoch": 1.5923938823914288,
      "grad_norm": 6.625025749206543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2006906270980835,
      "logits/rejected": -0.11663995683193207,
      "logps/chosen": -397.9346923828125,
      "logps/rejected": -433.2805480957031,
      "loss": 0.3582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7613928318023682,
      "rewards/margins": 1.7578977346420288,
      "rewards/rejected": -2.5192906856536865,
      "step": 12169
    },
    {
      "epoch": 1.5925247403287806,
      "grad_norm": 5.539305210113525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07814765721559525,
      "logits/rejected": -0.44709891080856323,
      "logps/chosen": -320.3141784667969,
      "logps/rejected": -354.65838623046875,
      "loss": 0.3162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1812402307987213,
      "rewards/margins": 1.459531307220459,
      "rewards/rejected": -1.640771508216858,
      "step": 12170
    },
    {
      "epoch": 1.5926555982661323,
      "grad_norm": 10.788846969604492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3524625897407532,
      "logits/rejected": -0.36570805311203003,
      "logps/chosen": -584.9676513671875,
      "logps/rejected": -559.7716674804688,
      "loss": 0.8621,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6420681476593018,
      "rewards/margins": 0.5669946074485779,
      "rewards/rejected": -1.2090625762939453,
      "step": 12171
    },
    {
      "epoch": 1.5927864562034841,
      "grad_norm": 9.140628814697266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19781513512134552,
      "logits/rejected": -0.3996718227863312,
      "logps/chosen": -406.03729248046875,
      "logps/rejected": -257.51690673828125,
      "loss": 0.3541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.667606770992279,
      "rewards/margins": 1.1796029806137085,
      "rewards/rejected": -1.8472098112106323,
      "step": 12172
    },
    {
      "epoch": 1.592917314140836,
      "grad_norm": 13.194169998168945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3029753267765045,
      "logits/rejected": -0.396982342004776,
      "logps/chosen": -337.8985900878906,
      "logps/rejected": -329.05914306640625,
      "loss": 0.5353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9465527534484863,
      "rewards/margins": 1.2788634300231934,
      "rewards/rejected": -2.2254161834716797,
      "step": 12173
    },
    {
      "epoch": 1.5930481720781877,
      "grad_norm": 9.030109405517578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09152478724718094,
      "logits/rejected": -0.21584469079971313,
      "logps/chosen": -474.61151123046875,
      "logps/rejected": -465.66510009765625,
      "loss": 0.5293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9008433818817139,
      "rewards/margins": 1.2653568983078003,
      "rewards/rejected": -2.1662003993988037,
      "step": 12174
    },
    {
      "epoch": 1.5931790300155395,
      "grad_norm": 5.368324279785156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12163778394460678,
      "logits/rejected": -0.2551285922527313,
      "logps/chosen": -468.3518981933594,
      "logps/rejected": -358.2231750488281,
      "loss": 0.3519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7246274352073669,
      "rewards/margins": 1.7024962902069092,
      "rewards/rejected": -0.9778689742088318,
      "step": 12175
    },
    {
      "epoch": 1.5933098879528913,
      "grad_norm": 8.400275230407715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021361909806728363,
      "logits/rejected": -0.1870049089193344,
      "logps/chosen": -517.5006103515625,
      "logps/rejected": -558.6016235351562,
      "loss": 0.3415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8516107201576233,
      "rewards/margins": 1.0822538137435913,
      "rewards/rejected": -1.9338645935058594,
      "step": 12176
    },
    {
      "epoch": 1.5934407458902429,
      "grad_norm": 9.247380256652832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13715600967407227,
      "logits/rejected": -0.18149913847446442,
      "logps/chosen": -446.70635986328125,
      "logps/rejected": -324.34173583984375,
      "loss": 0.4195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.903828501701355,
      "rewards/margins": 1.3985424041748047,
      "rewards/rejected": -2.30237078666687,
      "step": 12177
    },
    {
      "epoch": 1.5935716038275947,
      "grad_norm": 12.48391342163086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.039812177419662476,
      "logits/rejected": 0.3037303388118744,
      "logps/chosen": -470.65814208984375,
      "logps/rejected": -504.3554382324219,
      "loss": 0.6852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3279528617858887,
      "rewards/margins": 0.5772300958633423,
      "rewards/rejected": -1.9051828384399414,
      "step": 12178
    },
    {
      "epoch": 1.5937024617649465,
      "grad_norm": 8.103036880493164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17062559723854065,
      "logits/rejected": -0.30069050192832947,
      "logps/chosen": -492.1036682128906,
      "logps/rejected": -440.64923095703125,
      "loss": 0.3306,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7495352029800415,
      "rewards/margins": 1.6028246879577637,
      "rewards/rejected": -2.3523597717285156,
      "step": 12179
    },
    {
      "epoch": 1.5938333197022982,
      "grad_norm": 13.455791473388672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4820278584957123,
      "logits/rejected": -0.3946451246738434,
      "logps/chosen": -509.4749755859375,
      "logps/rejected": -430.9691162109375,
      "loss": 0.7837,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.269268274307251,
      "rewards/margins": 0.09333686530590057,
      "rewards/rejected": -1.362605094909668,
      "step": 12180
    },
    {
      "epoch": 1.5939641776396498,
      "grad_norm": 7.215168476104736,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0495324581861496,
      "logits/rejected": -0.2543790340423584,
      "logps/chosen": -449.2579345703125,
      "logps/rejected": -407.5911865234375,
      "loss": 0.3522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.821003794670105,
      "rewards/margins": 1.2798652648925781,
      "rewards/rejected": -2.1008689403533936,
      "step": 12181
    },
    {
      "epoch": 1.5940950355770016,
      "grad_norm": 6.619994163513184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12194783985614777,
      "logits/rejected": -0.23076698184013367,
      "logps/chosen": -480.028564453125,
      "logps/rejected": -465.1098937988281,
      "loss": 0.4307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3547158241271973,
      "rewards/margins": 1.3789339065551758,
      "rewards/rejected": -2.733649492263794,
      "step": 12182
    },
    {
      "epoch": 1.5942258935143534,
      "grad_norm": 9.722525596618652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1999555081129074,
      "logits/rejected": -0.5051900148391724,
      "logps/chosen": -488.98504638671875,
      "logps/rejected": -432.5574035644531,
      "loss": 0.4761,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5824037790298462,
      "rewards/margins": 0.6969717741012573,
      "rewards/rejected": -1.279375433921814,
      "step": 12183
    },
    {
      "epoch": 1.5943567514517052,
      "grad_norm": 11.668463706970215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23598304390907288,
      "logits/rejected": -0.2896144390106201,
      "logps/chosen": -329.9464416503906,
      "logps/rejected": -383.2503356933594,
      "loss": 0.5954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6315687894821167,
      "rewards/margins": 0.7014312744140625,
      "rewards/rejected": -1.3329999446868896,
      "step": 12184
    },
    {
      "epoch": 1.594487609389057,
      "grad_norm": 13.41312026977539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13753628730773926,
      "logits/rejected": -0.23752549290657043,
      "logps/chosen": -405.738525390625,
      "logps/rejected": -435.57421875,
      "loss": 0.9775,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6525470614433289,
      "rewards/margins": -0.28837278485298157,
      "rewards/rejected": -0.3641743063926697,
      "step": 12185
    },
    {
      "epoch": 1.5946184673264088,
      "grad_norm": 8.580329895019531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014238717034459114,
      "logits/rejected": -0.17025341093540192,
      "logps/chosen": -444.5984802246094,
      "logps/rejected": -496.6978759765625,
      "loss": 0.5346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0461249351501465,
      "rewards/margins": 1.100951910018921,
      "rewards/rejected": -2.1470768451690674,
      "step": 12186
    },
    {
      "epoch": 1.5947493252637606,
      "grad_norm": 9.054458618164062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31503206491470337,
      "logits/rejected": -0.7082418203353882,
      "logps/chosen": -390.57708740234375,
      "logps/rejected": -279.15069580078125,
      "loss": 0.4939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8563867807388306,
      "rewards/margins": 1.1521333456039429,
      "rewards/rejected": -2.0085201263427734,
      "step": 12187
    },
    {
      "epoch": 1.5948801832011124,
      "grad_norm": 8.637908935546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2588925063610077,
      "logits/rejected": -0.3222273886203766,
      "logps/chosen": -375.81658935546875,
      "logps/rejected": -483.1878356933594,
      "loss": 0.4534,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2156891822814941,
      "rewards/margins": 1.6168259382247925,
      "rewards/rejected": -2.832514762878418,
      "step": 12188
    },
    {
      "epoch": 1.5950110411384641,
      "grad_norm": 13.351985931396484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28735703229904175,
      "logits/rejected": -0.19028937816619873,
      "logps/chosen": -378.82379150390625,
      "logps/rejected": -504.4668884277344,
      "loss": 0.7307,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.589326024055481,
      "rewards/margins": 0.9124031066894531,
      "rewards/rejected": -2.5017292499542236,
      "step": 12189
    },
    {
      "epoch": 1.595141899075816,
      "grad_norm": 10.573807716369629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38088205456733704,
      "logits/rejected": -0.3178119659423828,
      "logps/chosen": -377.4930419921875,
      "logps/rejected": -418.967041015625,
      "loss": 0.5617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1833442449569702,
      "rewards/margins": 0.5188276767730713,
      "rewards/rejected": -1.702172040939331,
      "step": 12190
    },
    {
      "epoch": 1.5952727570131677,
      "grad_norm": 9.390090942382812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05087948590517044,
      "logits/rejected": -0.1688624620437622,
      "logps/chosen": -473.68499755859375,
      "logps/rejected": -480.9628601074219,
      "loss": 0.5049,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49447816610336304,
      "rewards/margins": 0.8232667446136475,
      "rewards/rejected": -1.3177449703216553,
      "step": 12191
    },
    {
      "epoch": 1.5954036149505193,
      "grad_norm": 9.714264869689941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08579080551862717,
      "logits/rejected": -0.09796403348445892,
      "logps/chosen": -356.2947998046875,
      "logps/rejected": -379.4524841308594,
      "loss": 0.5926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28149890899658203,
      "rewards/margins": 0.9473657608032227,
      "rewards/rejected": -1.2288646697998047,
      "step": 12192
    },
    {
      "epoch": 1.595534472887871,
      "grad_norm": 12.165287017822266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2979280650615692,
      "logits/rejected": -0.15799951553344727,
      "logps/chosen": -672.4510498046875,
      "logps/rejected": -525.84423828125,
      "loss": 0.6002,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1066365242004395,
      "rewards/margins": 0.6433665752410889,
      "rewards/rejected": -1.7500030994415283,
      "step": 12193
    },
    {
      "epoch": 1.5956653308252229,
      "grad_norm": 9.328309059143066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06387743353843689,
      "logits/rejected": 0.08984914422035217,
      "logps/chosen": -382.3890686035156,
      "logps/rejected": -490.9567565917969,
      "loss": 0.4361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9585132598876953,
      "rewards/margins": 0.9022880792617798,
      "rewards/rejected": -1.860801339149475,
      "step": 12194
    },
    {
      "epoch": 1.5957961887625747,
      "grad_norm": 9.612791061401367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2707138657569885,
      "logits/rejected": -0.13023681938648224,
      "logps/chosen": -489.73577880859375,
      "logps/rejected": -510.86090087890625,
      "loss": 0.6692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.130990982055664,
      "rewards/margins": 0.6703617572784424,
      "rewards/rejected": -1.801352620124817,
      "step": 12195
    },
    {
      "epoch": 1.5959270466999262,
      "grad_norm": 7.2000250816345215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006172679364681244,
      "logits/rejected": -0.035320788621902466,
      "logps/chosen": -421.54229736328125,
      "logps/rejected": -368.201416015625,
      "loss": 0.431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16577130556106567,
      "rewards/margins": 1.1359989643096924,
      "rewards/rejected": -1.3017702102661133,
      "step": 12196
    },
    {
      "epoch": 1.596057904637278,
      "grad_norm": 7.5635857582092285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30918917059898376,
      "logits/rejected": -0.34867772459983826,
      "logps/chosen": -329.34423828125,
      "logps/rejected": -306.01141357421875,
      "loss": 0.4407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6163488626480103,
      "rewards/margins": 0.9385925531387329,
      "rewards/rejected": -1.5549414157867432,
      "step": 12197
    },
    {
      "epoch": 1.5961887625746298,
      "grad_norm": 12.669564247131348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24924007058143616,
      "logits/rejected": -0.3379463255405426,
      "logps/chosen": -812.360107421875,
      "logps/rejected": -470.2431945800781,
      "loss": 0.6482,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5873854756355286,
      "rewards/margins": 1.4312653541564941,
      "rewards/rejected": -2.018650531768799,
      "step": 12198
    },
    {
      "epoch": 1.5963196205119816,
      "grad_norm": 7.96816349029541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3598143756389618,
      "logits/rejected": -0.36508122086524963,
      "logps/chosen": -490.5789794921875,
      "logps/rejected": -379.2511901855469,
      "loss": 0.5004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4374506175518036,
      "rewards/margins": 0.7378951907157898,
      "rewards/rejected": -1.175345778465271,
      "step": 12199
    },
    {
      "epoch": 1.5964504784493334,
      "grad_norm": 11.843931198120117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28881219029426575,
      "logits/rejected": -0.08837494254112244,
      "logps/chosen": -492.7612609863281,
      "logps/rejected": -497.9524841308594,
      "loss": 0.6683,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9652835726737976,
      "rewards/margins": 0.6528918743133545,
      "rewards/rejected": -1.6181755065917969,
      "step": 12200
    },
    {
      "epoch": 1.5965813363866852,
      "grad_norm": 7.676364421844482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011636067181825638,
      "logits/rejected": -0.07207771390676498,
      "logps/chosen": -411.3196105957031,
      "logps/rejected": -454.46209716796875,
      "loss": 0.3794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4277347326278687,
      "rewards/margins": 1.2514032125473022,
      "rewards/rejected": -2.679137945175171,
      "step": 12201
    },
    {
      "epoch": 1.596712194324037,
      "grad_norm": 7.905526638031006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02249245159327984,
      "logits/rejected": -0.3110406696796417,
      "logps/chosen": -473.8563537597656,
      "logps/rejected": -460.42437744140625,
      "loss": 0.4288,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12828120589256287,
      "rewards/margins": 1.282479166984558,
      "rewards/rejected": -1.4107604026794434,
      "step": 12202
    },
    {
      "epoch": 1.5968430522613888,
      "grad_norm": 10.772685050964355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08792257308959961,
      "logits/rejected": -0.19508343935012817,
      "logps/chosen": -634.4967651367188,
      "logps/rejected": -667.9381713867188,
      "loss": 0.6021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2378779947757721,
      "rewards/margins": 0.8746918439865112,
      "rewards/rejected": -1.1125696897506714,
      "step": 12203
    },
    {
      "epoch": 1.5969739101987406,
      "grad_norm": 6.8640289306640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2323124259710312,
      "logits/rejected": -0.19246791303157806,
      "logps/chosen": -472.18585205078125,
      "logps/rejected": -394.3040771484375,
      "loss": 0.3211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8654990196228027,
      "rewards/margins": 1.3356847763061523,
      "rewards/rejected": -2.201183795928955,
      "step": 12204
    },
    {
      "epoch": 1.5971047681360924,
      "grad_norm": 8.77323055267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037950120866298676,
      "logits/rejected": -0.2594207525253296,
      "logps/chosen": -496.86328125,
      "logps/rejected": -449.36859130859375,
      "loss": 0.4283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.43934059143066406,
      "rewards/margins": 1.0071239471435547,
      "rewards/rejected": -0.5677833557128906,
      "step": 12205
    },
    {
      "epoch": 1.5972356260734442,
      "grad_norm": 9.186689376831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19923748075962067,
      "logits/rejected": -0.014360131695866585,
      "logps/chosen": -369.8032531738281,
      "logps/rejected": -395.00103759765625,
      "loss": 0.5279,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5109919309616089,
      "rewards/margins": 0.9141929149627686,
      "rewards/rejected": -1.425184726715088,
      "step": 12206
    },
    {
      "epoch": 1.5973664840107957,
      "grad_norm": 8.970170021057129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46838366985321045,
      "logits/rejected": -0.6540536284446716,
      "logps/chosen": -572.6026000976562,
      "logps/rejected": -337.5539855957031,
      "loss": 0.4765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.096589207649231,
      "rewards/margins": 1.1221165657043457,
      "rewards/rejected": -2.218705415725708,
      "step": 12207
    },
    {
      "epoch": 1.5974973419481475,
      "grad_norm": 4.783992767333984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24061250686645508,
      "logits/rejected": -0.25846487283706665,
      "logps/chosen": -582.3264770507812,
      "logps/rejected": -390.99334716796875,
      "loss": 0.2061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3546760678291321,
      "rewards/margins": 2.00091290473938,
      "rewards/rejected": -2.355588912963867,
      "step": 12208
    },
    {
      "epoch": 1.5976281998854993,
      "grad_norm": 13.648696899414062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22272898256778717,
      "logits/rejected": -0.02152692899107933,
      "logps/chosen": -483.17694091796875,
      "logps/rejected": -486.1755676269531,
      "loss": 0.6938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2603025436401367,
      "rewards/margins": 0.4945409595966339,
      "rewards/rejected": -1.7548434734344482,
      "step": 12209
    },
    {
      "epoch": 1.597759057822851,
      "grad_norm": 9.44477653503418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06844951212406158,
      "logits/rejected": -0.08270613104104996,
      "logps/chosen": -555.8975830078125,
      "logps/rejected": -343.2728271484375,
      "loss": 0.4533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02567940205335617,
      "rewards/margins": 0.9280841946601868,
      "rewards/rejected": -0.9537636041641235,
      "step": 12210
    },
    {
      "epoch": 1.5978899157602027,
      "grad_norm": 9.595040321350098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28513461351394653,
      "logits/rejected": -0.4049806594848633,
      "logps/chosen": -383.6047668457031,
      "logps/rejected": -373.6172180175781,
      "loss": 0.6815,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8041470050811768,
      "rewards/margins": 0.8941752910614014,
      "rewards/rejected": -1.6983222961425781,
      "step": 12211
    },
    {
      "epoch": 1.5980207736975545,
      "grad_norm": 6.432190895080566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011401496827602386,
      "logits/rejected": -0.32443588972091675,
      "logps/chosen": -520.5589599609375,
      "logps/rejected": -634.6332397460938,
      "loss": 0.2803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6096222400665283,
      "rewards/margins": 1.5636560916900635,
      "rewards/rejected": -2.173278331756592,
      "step": 12212
    },
    {
      "epoch": 1.5981516316349063,
      "grad_norm": 10.593390464782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16661733388900757,
      "logits/rejected": -0.08007029443979263,
      "logps/chosen": -497.50933837890625,
      "logps/rejected": -637.3460083007812,
      "loss": 0.4285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8528990149497986,
      "rewards/margins": 1.0787311792373657,
      "rewards/rejected": -1.9316301345825195,
      "step": 12213
    },
    {
      "epoch": 1.598282489572258,
      "grad_norm": 11.766231536865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16557593643665314,
      "logits/rejected": -0.32416146993637085,
      "logps/chosen": -417.465576171875,
      "logps/rejected": -371.9227294921875,
      "loss": 0.7413,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9400870203971863,
      "rewards/margins": 0.7307804822921753,
      "rewards/rejected": -1.6708673238754272,
      "step": 12214
    },
    {
      "epoch": 1.5984133475096098,
      "grad_norm": 9.461634635925293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03459553420543671,
      "logits/rejected": -0.39458534121513367,
      "logps/chosen": -841.2373657226562,
      "logps/rejected": -783.90673828125,
      "loss": 0.3902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9321972131729126,
      "rewards/margins": 0.9653070569038391,
      "rewards/rejected": -1.897504210472107,
      "step": 12215
    },
    {
      "epoch": 1.5985442054469616,
      "grad_norm": 7.4788713455200195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38608402013778687,
      "logits/rejected": -0.5402696132659912,
      "logps/chosen": -370.9571228027344,
      "logps/rejected": -385.2929992675781,
      "loss": 0.4933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1684824228286743,
      "rewards/margins": 0.9558887481689453,
      "rewards/rejected": -2.12437105178833,
      "step": 12216
    },
    {
      "epoch": 1.5986750633843134,
      "grad_norm": 6.774923801422119,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1645336002111435,
      "logits/rejected": -0.191181480884552,
      "logps/chosen": -287.0036315917969,
      "logps/rejected": -293.0347595214844,
      "loss": 0.3773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12776392698287964,
      "rewards/margins": 1.0290076732635498,
      "rewards/rejected": -1.1567716598510742,
      "step": 12217
    },
    {
      "epoch": 1.5988059213216652,
      "grad_norm": 10.62181568145752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24376018345355988,
      "logits/rejected": -0.16200530529022217,
      "logps/chosen": -457.0630187988281,
      "logps/rejected": -347.84674072265625,
      "loss": 0.5749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4738510549068451,
      "rewards/margins": 0.7657420635223389,
      "rewards/rejected": -1.2395931482315063,
      "step": 12218
    },
    {
      "epoch": 1.598936779259017,
      "grad_norm": 8.687385559082031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.034569092094898224,
      "logits/rejected": -0.19148960709571838,
      "logps/chosen": -526.7488403320312,
      "logps/rejected": -526.4783935546875,
      "loss": 0.5787,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9596782922744751,
      "rewards/margins": 0.6937949657440186,
      "rewards/rejected": -1.6534732580184937,
      "step": 12219
    },
    {
      "epoch": 1.5990676371963688,
      "grad_norm": 7.734033584594727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25185275077819824,
      "logits/rejected": -0.44702577590942383,
      "logps/chosen": -338.2401123046875,
      "logps/rejected": -422.94219970703125,
      "loss": 0.4252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2850195169448853,
      "rewards/margins": 1.2064766883850098,
      "rewards/rejected": -2.4914960861206055,
      "step": 12220
    },
    {
      "epoch": 1.5991984951337206,
      "grad_norm": 6.005056381225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23625938594341278,
      "logits/rejected": -0.2426583170890808,
      "logps/chosen": -345.9871826171875,
      "logps/rejected": -442.1165466308594,
      "loss": 0.2925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470748782157898,
      "rewards/margins": 1.3931257724761963,
      "rewards/rejected": -1.5402005910873413,
      "step": 12221
    },
    {
      "epoch": 1.5993293530710724,
      "grad_norm": 5.894096374511719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08367311954498291,
      "logits/rejected": 0.12043046951293945,
      "logps/chosen": -677.8843994140625,
      "logps/rejected": -686.7808227539062,
      "loss": 0.3011,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09327259659767151,
      "rewards/margins": 2.6110434532165527,
      "rewards/rejected": -2.517770767211914,
      "step": 12222
    },
    {
      "epoch": 1.599460211008424,
      "grad_norm": 6.388150691986084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17168614268302917,
      "logits/rejected": -0.34453535079956055,
      "logps/chosen": -573.4622802734375,
      "logps/rejected": -523.2603149414062,
      "loss": 0.2665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16746863722801208,
      "rewards/margins": 1.5407966375350952,
      "rewards/rejected": -1.7082653045654297,
      "step": 12223
    },
    {
      "epoch": 1.5995910689457757,
      "grad_norm": 8.529719352722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35650765895843506,
      "logits/rejected": -0.3314102590084076,
      "logps/chosen": -495.08392333984375,
      "logps/rejected": -471.02349853515625,
      "loss": 0.2814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.721077561378479,
      "rewards/margins": 1.6157503128051758,
      "rewards/rejected": -2.3368277549743652,
      "step": 12224
    },
    {
      "epoch": 1.5997219268831275,
      "grad_norm": 4.639079570770264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3276134133338928,
      "logits/rejected": -0.3774687647819519,
      "logps/chosen": -461.7423095703125,
      "logps/rejected": -359.75396728515625,
      "loss": 0.2359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13731423020362854,
      "rewards/margins": 1.6160838603973389,
      "rewards/rejected": -1.753398060798645,
      "step": 12225
    },
    {
      "epoch": 1.599852784820479,
      "grad_norm": 11.618276596069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1804095357656479,
      "logits/rejected": -0.24185815453529358,
      "logps/chosen": -495.3494873046875,
      "logps/rejected": -423.1729736328125,
      "loss": 0.6229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1250025033950806,
      "rewards/margins": 0.8277246952056885,
      "rewards/rejected": -1.9527270793914795,
      "step": 12226
    },
    {
      "epoch": 1.599983642757831,
      "grad_norm": 6.60426664352417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2826879024505615,
      "logits/rejected": -0.08555717766284943,
      "logps/chosen": -413.21417236328125,
      "logps/rejected": -434.9566650390625,
      "loss": 0.2991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3056211769580841,
      "rewards/margins": 1.3706388473510742,
      "rewards/rejected": -1.6762598752975464,
      "step": 12227
    },
    {
      "epoch": 1.6001145006951827,
      "grad_norm": 6.485929012298584,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06472894549369812,
      "logits/rejected": 0.03605719283223152,
      "logps/chosen": -542.6214599609375,
      "logps/rejected": -537.6998291015625,
      "loss": 0.3057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.029438018798828125,
      "rewards/margins": 1.1671019792556763,
      "rewards/rejected": -1.196540117263794,
      "step": 12228
    },
    {
      "epoch": 1.6002453586325345,
      "grad_norm": 4.92225980758667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005738344043493271,
      "logits/rejected": 0.03739360719919205,
      "logps/chosen": -658.5970458984375,
      "logps/rejected": -552.6488037109375,
      "loss": 0.1702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6964653730392456,
      "rewards/margins": 2.0870442390441895,
      "rewards/rejected": -2.7835094928741455,
      "step": 12229
    },
    {
      "epoch": 1.6003762165698863,
      "grad_norm": 10.39513874053955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.033952195197343826,
      "logits/rejected": 0.015152066946029663,
      "logps/chosen": -365.19732666015625,
      "logps/rejected": -327.0453796386719,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9175476431846619,
      "rewards/margins": 0.19272935390472412,
      "rewards/rejected": -1.1102769374847412,
      "step": 12230
    },
    {
      "epoch": 1.600507074507238,
      "grad_norm": 2.8814096450805664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23483355343341827,
      "logits/rejected": -0.28765013813972473,
      "logps/chosen": -350.37353515625,
      "logps/rejected": -327.8395080566406,
      "loss": 0.2861,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.045027494430542,
      "rewards/margins": 1.9050769805908203,
      "rewards/rejected": -2.9501044750213623,
      "step": 12231
    },
    {
      "epoch": 1.6006379324445899,
      "grad_norm": 6.664541244506836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0793149471282959,
      "logits/rejected": -0.2012495994567871,
      "logps/chosen": -406.86090087890625,
      "logps/rejected": -409.10345458984375,
      "loss": 0.3619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4921329617500305,
      "rewards/margins": 1.422614336013794,
      "rewards/rejected": -1.9147471189498901,
      "step": 12232
    },
    {
      "epoch": 1.6007687903819416,
      "grad_norm": 7.953220844268799,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05839803069829941,
      "logits/rejected": 0.23334433138370514,
      "logps/chosen": -605.8992919921875,
      "logps/rejected": -509.057861328125,
      "loss": 0.4891,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.156370148062706,
      "rewards/margins": 0.8501207828521729,
      "rewards/rejected": -1.00649094581604,
      "step": 12233
    },
    {
      "epoch": 1.6008996483192934,
      "grad_norm": 9.810832977294922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2024557739496231,
      "logits/rejected": -0.29033970832824707,
      "logps/chosen": -361.4825744628906,
      "logps/rejected": -425.97454833984375,
      "loss": 0.3328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2585867643356323,
      "rewards/margins": 1.37693190574646,
      "rewards/rejected": -2.635518789291382,
      "step": 12234
    },
    {
      "epoch": 1.6010305062566452,
      "grad_norm": 5.701183319091797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09787146747112274,
      "logits/rejected": -0.2871534526348114,
      "logps/chosen": -452.68341064453125,
      "logps/rejected": -386.4613342285156,
      "loss": 0.2142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3094680905342102,
      "rewards/margins": 2.33598256111145,
      "rewards/rejected": -2.026514768600464,
      "step": 12235
    },
    {
      "epoch": 1.601161364193997,
      "grad_norm": 7.6792893409729,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2661469280719757,
      "logits/rejected": -0.31046822667121887,
      "logps/chosen": -324.478759765625,
      "logps/rejected": -281.20013427734375,
      "loss": 0.4231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9275394678115845,
      "rewards/margins": 0.8620059490203857,
      "rewards/rejected": -1.7895454168319702,
      "step": 12236
    },
    {
      "epoch": 1.6012922221313488,
      "grad_norm": 8.15040111541748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1569962203502655,
      "logits/rejected": -0.36916130781173706,
      "logps/chosen": -593.689453125,
      "logps/rejected": -389.6627197265625,
      "loss": 0.4027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12394942343235016,
      "rewards/margins": 2.3369762897491455,
      "rewards/rejected": -2.460925817489624,
      "step": 12237
    },
    {
      "epoch": 1.6014230800687004,
      "grad_norm": 6.674679279327393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04507817327976227,
      "logits/rejected": 0.03911686688661575,
      "logps/chosen": -444.98284912109375,
      "logps/rejected": -462.5661926269531,
      "loss": 0.3253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.055805597454309464,
      "rewards/margins": 1.7243984937667847,
      "rewards/rejected": -1.668592929840088,
      "step": 12238
    },
    {
      "epoch": 1.6015539380060522,
      "grad_norm": 8.057636260986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22330862283706665,
      "logits/rejected": -0.0879773199558258,
      "logps/chosen": -410.2188720703125,
      "logps/rejected": -532.7238159179688,
      "loss": 0.4371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33290672302246094,
      "rewards/margins": 1.0774866342544556,
      "rewards/rejected": -1.4103933572769165,
      "step": 12239
    },
    {
      "epoch": 1.601684795943404,
      "grad_norm": 9.806869506835938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47739842534065247,
      "logits/rejected": -0.47148171067237854,
      "logps/chosen": -427.30975341796875,
      "logps/rejected": -565.0569458007812,
      "loss": 0.4202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45089825987815857,
      "rewards/margins": 1.3420759439468384,
      "rewards/rejected": -1.7929742336273193,
      "step": 12240
    },
    {
      "epoch": 1.6018156538807558,
      "grad_norm": 10.894590377807617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22870007157325745,
      "logits/rejected": -0.3971341550350189,
      "logps/chosen": -349.40582275390625,
      "logps/rejected": -496.9942321777344,
      "loss": 0.6007,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5036113262176514,
      "rewards/margins": 0.8249177932739258,
      "rewards/rejected": -2.328529119491577,
      "step": 12241
    },
    {
      "epoch": 1.6019465118181073,
      "grad_norm": 13.879537582397461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2095809280872345,
      "logits/rejected": -0.3202317953109741,
      "logps/chosen": -409.8881530761719,
      "logps/rejected": -389.50836181640625,
      "loss": 0.7039,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3405892848968506,
      "rewards/margins": 1.0321961641311646,
      "rewards/rejected": -2.3727853298187256,
      "step": 12242
    },
    {
      "epoch": 1.6020773697554591,
      "grad_norm": 6.332188606262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38999423384666443,
      "logits/rejected": -0.4280318021774292,
      "logps/chosen": -454.62591552734375,
      "logps/rejected": -406.16998291015625,
      "loss": 0.3776,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8225827813148499,
      "rewards/margins": 1.581281065940857,
      "rewards/rejected": -2.4038641452789307,
      "step": 12243
    },
    {
      "epoch": 1.602208227692811,
      "grad_norm": 8.259156227111816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15808385610580444,
      "logits/rejected": -0.14974068105220795,
      "logps/chosen": -431.66357421875,
      "logps/rejected": -486.61260986328125,
      "loss": 0.4992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9722996950149536,
      "rewards/margins": 1.084897518157959,
      "rewards/rejected": -2.057197093963623,
      "step": 12244
    },
    {
      "epoch": 1.6023390856301627,
      "grad_norm": 9.763996124267578,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0076399678364396095,
      "logits/rejected": -0.17645472288131714,
      "logps/chosen": -448.34808349609375,
      "logps/rejected": -326.924560546875,
      "loss": 0.5034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9793900847434998,
      "rewards/margins": 0.9049316048622131,
      "rewards/rejected": -1.884321928024292,
      "step": 12245
    },
    {
      "epoch": 1.6024699435675145,
      "grad_norm": 7.335413455963135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08519167453050613,
      "logits/rejected": -0.16899845004081726,
      "logps/chosen": -419.04473876953125,
      "logps/rejected": -369.3721923828125,
      "loss": 0.2835,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17896977066993713,
      "rewards/margins": 1.96499764919281,
      "rewards/rejected": -2.143967628479004,
      "step": 12246
    },
    {
      "epoch": 1.6026008015048663,
      "grad_norm": 5.721279144287109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5864048600196838,
      "logits/rejected": -0.4513961672782898,
      "logps/chosen": -337.6383972167969,
      "logps/rejected": -391.4002990722656,
      "loss": 0.3324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.590161144733429,
      "rewards/margins": 1.4166337251663208,
      "rewards/rejected": -2.0067949295043945,
      "step": 12247
    },
    {
      "epoch": 1.602731659442218,
      "grad_norm": 15.18624210357666,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06269507110118866,
      "logits/rejected": -0.31451666355133057,
      "logps/chosen": -439.12371826171875,
      "logps/rejected": -573.1182250976562,
      "loss": 0.4941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3473103046417236,
      "rewards/margins": 0.7889301776885986,
      "rewards/rejected": -2.1362404823303223,
      "step": 12248
    },
    {
      "epoch": 1.6028625173795699,
      "grad_norm": 5.037960052490234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04148877039551735,
      "logits/rejected": -0.30369484424591064,
      "logps/chosen": -481.04254150390625,
      "logps/rejected": -337.7257995605469,
      "loss": 0.2181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7671139240264893,
      "rewards/margins": 2.2633039951324463,
      "rewards/rejected": -3.0304179191589355,
      "step": 12249
    },
    {
      "epoch": 1.6029933753169217,
      "grad_norm": 8.58342170715332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23672489821910858,
      "logits/rejected": -0.3327648937702179,
      "logps/chosen": -309.1908874511719,
      "logps/rejected": -409.9425354003906,
      "loss": 0.5272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7326713800430298,
      "rewards/margins": 0.5426645278930664,
      "rewards/rejected": -1.2753359079360962,
      "step": 12250
    },
    {
      "epoch": 1.6031242332542734,
      "grad_norm": 8.978693962097168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18220576643943787,
      "logits/rejected": -0.19865980744361877,
      "logps/chosen": -374.89947509765625,
      "logps/rejected": -420.2400207519531,
      "loss": 0.445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2955389022827148,
      "rewards/margins": 0.7464694380760193,
      "rewards/rejected": -2.042008399963379,
      "step": 12251
    },
    {
      "epoch": 1.6032550911916252,
      "grad_norm": 11.81664752960205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09003175050020218,
      "logits/rejected": -0.11615907400846481,
      "logps/chosen": -419.4094543457031,
      "logps/rejected": -410.4575500488281,
      "loss": 0.6734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7694880366325378,
      "rewards/margins": 0.800267219543457,
      "rewards/rejected": -1.56975519657135,
      "step": 12252
    },
    {
      "epoch": 1.6033859491289768,
      "grad_norm": 8.052643775939941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3196711540222168,
      "logits/rejected": -0.39942264556884766,
      "logps/chosen": -645.537353515625,
      "logps/rejected": -670.3146362304688,
      "loss": 0.3374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.509154736995697,
      "rewards/margins": 1.6255594491958618,
      "rewards/rejected": -2.134714126586914,
      "step": 12253
    },
    {
      "epoch": 1.6035168070663286,
      "grad_norm": 13.214764595031738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3255962133407593,
      "logits/rejected": -0.4418456256389618,
      "logps/chosen": -468.61517333984375,
      "logps/rejected": -349.64813232421875,
      "loss": 0.7693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7755924463272095,
      "rewards/margins": 0.3433138132095337,
      "rewards/rejected": -2.118906259536743,
      "step": 12254
    },
    {
      "epoch": 1.6036476650036804,
      "grad_norm": 8.816178321838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27848124504089355,
      "logits/rejected": -0.3767797350883484,
      "logps/chosen": -255.88182067871094,
      "logps/rejected": -338.9354248046875,
      "loss": 0.6283,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0242431163787842,
      "rewards/margins": 0.5304324626922607,
      "rewards/rejected": -1.5546754598617554,
      "step": 12255
    },
    {
      "epoch": 1.6037785229410322,
      "grad_norm": 10.230679512023926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13129571080207825,
      "logits/rejected": -0.1936156153678894,
      "logps/chosen": -467.9709777832031,
      "logps/rejected": -555.840087890625,
      "loss": 0.446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3075003623962402,
      "rewards/margins": 0.8729078769683838,
      "rewards/rejected": -2.180408239364624,
      "step": 12256
    },
    {
      "epoch": 1.6039093808783838,
      "grad_norm": 6.952223777770996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23989659547805786,
      "logits/rejected": -0.24989598989486694,
      "logps/chosen": -767.5844116210938,
      "logps/rejected": -617.0195922851562,
      "loss": 0.3372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3496244549751282,
      "rewards/margins": 1.6374573707580566,
      "rewards/rejected": -1.2878328561782837,
      "step": 12257
    },
    {
      "epoch": 1.6040402388157355,
      "grad_norm": 5.552672863006592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053146302700042725,
      "logits/rejected": -0.15930810570716858,
      "logps/chosen": -453.17254638671875,
      "logps/rejected": -503.97882080078125,
      "loss": 0.3096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.032868504524231,
      "rewards/margins": 1.7906818389892578,
      "rewards/rejected": -2.8235507011413574,
      "step": 12258
    },
    {
      "epoch": 1.6041710967530873,
      "grad_norm": 10.938255310058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07185691595077515,
      "logits/rejected": -0.0548606812953949,
      "logps/chosen": -741.5894165039062,
      "logps/rejected": -506.386474609375,
      "loss": 0.6153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04155406355857849,
      "rewards/margins": 0.5913429260253906,
      "rewards/rejected": -0.6328970193862915,
      "step": 12259
    },
    {
      "epoch": 1.6043019546904391,
      "grad_norm": 8.075312614440918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03467632085084915,
      "logits/rejected": -0.33004945516586304,
      "logps/chosen": -389.768310546875,
      "logps/rejected": -381.1480712890625,
      "loss": 0.3553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.867755651473999,
      "rewards/margins": 1.2390704154968262,
      "rewards/rejected": -3.106826066970825,
      "step": 12260
    },
    {
      "epoch": 1.604432812627791,
      "grad_norm": 9.750117301940918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0415191724896431,
      "logits/rejected": -0.25792452692985535,
      "logps/chosen": -588.2288818359375,
      "logps/rejected": -402.1199645996094,
      "loss": 0.4059,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24782583117485046,
      "rewards/margins": 1.5376590490341187,
      "rewards/rejected": -1.785485029220581,
      "step": 12261
    },
    {
      "epoch": 1.6045636705651427,
      "grad_norm": 8.731810569763184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2949509620666504,
      "logits/rejected": -0.37944093346595764,
      "logps/chosen": -417.7113952636719,
      "logps/rejected": -462.7957458496094,
      "loss": 0.3686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7409824728965759,
      "rewards/margins": 1.1720871925354004,
      "rewards/rejected": -1.913069725036621,
      "step": 12262
    },
    {
      "epoch": 1.6046945285024945,
      "grad_norm": 8.045304298400879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3311069905757904,
      "logits/rejected": -0.4298369884490967,
      "logps/chosen": -451.2891540527344,
      "logps/rejected": -482.53173828125,
      "loss": 0.3799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7724255323410034,
      "rewards/margins": 1.1171791553497314,
      "rewards/rejected": -1.8896046876907349,
      "step": 12263
    },
    {
      "epoch": 1.6048253864398463,
      "grad_norm": 10.004438400268555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2467089593410492,
      "logits/rejected": -0.1970985233783722,
      "logps/chosen": -460.0947570800781,
      "logps/rejected": -558.8953857421875,
      "loss": 0.3713,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4972461462020874,
      "rewards/margins": 1.5295934677124023,
      "rewards/rejected": -3.0268397331237793,
      "step": 12264
    },
    {
      "epoch": 1.604956244377198,
      "grad_norm": 9.46066951751709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26365649700164795,
      "logits/rejected": -0.40826743841171265,
      "logps/chosen": -558.3825073242188,
      "logps/rejected": -360.7793884277344,
      "loss": 0.4612,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3448925018310547,
      "rewards/margins": 0.7735666632652283,
      "rewards/rejected": -2.1184592247009277,
      "step": 12265
    },
    {
      "epoch": 1.6050871023145499,
      "grad_norm": 7.1548638343811035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3447885513305664,
      "logits/rejected": -0.4176250100135803,
      "logps/chosen": -578.7196044921875,
      "logps/rejected": -538.1272583007812,
      "loss": 0.3812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7072297930717468,
      "rewards/margins": 1.38960862159729,
      "rewards/rejected": -2.0968384742736816,
      "step": 12266
    },
    {
      "epoch": 1.6052179602519017,
      "grad_norm": 8.604644775390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1886691153049469,
      "logits/rejected": -0.2808062434196472,
      "logps/chosen": -299.2169189453125,
      "logps/rejected": -247.6050567626953,
      "loss": 0.4871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0590437650680542,
      "rewards/margins": 0.9501628279685974,
      "rewards/rejected": -2.0092062950134277,
      "step": 12267
    },
    {
      "epoch": 1.6053488181892535,
      "grad_norm": 9.725412368774414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29469382762908936,
      "logits/rejected": -0.41744470596313477,
      "logps/chosen": -423.66192626953125,
      "logps/rejected": -373.0144348144531,
      "loss": 0.6815,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.315263032913208,
      "rewards/margins": 1.2656198740005493,
      "rewards/rejected": -1.5808827877044678,
      "step": 12268
    },
    {
      "epoch": 1.605479676126605,
      "grad_norm": 9.974519729614258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37631919980049133,
      "logits/rejected": -0.06580423563718796,
      "logps/chosen": -426.4013671875,
      "logps/rejected": -679.136474609375,
      "loss": 0.4377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9364838004112244,
      "rewards/margins": 1.0925393104553223,
      "rewards/rejected": -2.0290231704711914,
      "step": 12269
    },
    {
      "epoch": 1.6056105340639568,
      "grad_norm": 11.352230072021484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28606438636779785,
      "logits/rejected": -0.41030794382095337,
      "logps/chosen": -507.1344909667969,
      "logps/rejected": -562.51123046875,
      "loss": 0.594,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21265700459480286,
      "rewards/margins": 0.8207341432571411,
      "rewards/rejected": -1.0333912372589111,
      "step": 12270
    },
    {
      "epoch": 1.6057413920013086,
      "grad_norm": 9.967572212219238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2898558974266052,
      "logits/rejected": -0.34499356150627136,
      "logps/chosen": -637.0499267578125,
      "logps/rejected": -497.9663391113281,
      "loss": 0.4237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7648061513900757,
      "rewards/margins": 1.041804552078247,
      "rewards/rejected": -2.8066108226776123,
      "step": 12271
    },
    {
      "epoch": 1.6058722499386602,
      "grad_norm": 12.27411937713623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3521580994129181,
      "logits/rejected": -0.24117226898670197,
      "logps/chosen": -433.3251953125,
      "logps/rejected": -355.0225524902344,
      "loss": 0.7515,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.000542402267456,
      "rewards/margins": 0.4300442934036255,
      "rewards/rejected": -1.430586814880371,
      "step": 12272
    },
    {
      "epoch": 1.606003107876012,
      "grad_norm": 14.30724048614502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06798210740089417,
      "logits/rejected": -0.1812405288219452,
      "logps/chosen": -482.4394836425781,
      "logps/rejected": -394.0166931152344,
      "loss": 0.5603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3873296976089478,
      "rewards/margins": 1.1824092864990234,
      "rewards/rejected": -2.5697391033172607,
      "step": 12273
    },
    {
      "epoch": 1.6061339658133638,
      "grad_norm": 8.380297660827637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3014765679836273,
      "logits/rejected": 0.05004199594259262,
      "logps/chosen": -618.9462280273438,
      "logps/rejected": -457.37628173828125,
      "loss": 0.4989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6209948658943176,
      "rewards/margins": 1.0359818935394287,
      "rewards/rejected": -1.6569766998291016,
      "step": 12274
    },
    {
      "epoch": 1.6062648237507156,
      "grad_norm": 8.20344352722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2823312282562256,
      "logits/rejected": -0.33665740489959717,
      "logps/chosen": -600.4269409179688,
      "logps/rejected": -558.1609497070312,
      "loss": 0.3208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.915778636932373,
      "rewards/margins": 2.026233434677124,
      "rewards/rejected": -2.942011833190918,
      "step": 12275
    },
    {
      "epoch": 1.6063956816880673,
      "grad_norm": 8.122161865234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2785051465034485,
      "logits/rejected": -0.23493173718452454,
      "logps/chosen": -441.7784729003906,
      "logps/rejected": -642.5960693359375,
      "loss": 0.3705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0981794595718384,
      "rewards/margins": 1.6877543926239014,
      "rewards/rejected": -2.78593373298645,
      "step": 12276
    },
    {
      "epoch": 1.6065265396254191,
      "grad_norm": 1.2965967655181885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2692348062992096,
      "logits/rejected": -0.394531786441803,
      "logps/chosen": -494.60345458984375,
      "logps/rejected": -349.7237243652344,
      "loss": 0.1573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5671086311340332,
      "rewards/margins": 3.054499626159668,
      "rewards/rejected": -3.621608018875122,
      "step": 12277
    },
    {
      "epoch": 1.606657397562771,
      "grad_norm": 6.0178680419921875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6231485605239868,
      "logits/rejected": -0.6980713605880737,
      "logps/chosen": -235.77711486816406,
      "logps/rejected": -294.8254089355469,
      "loss": 0.2243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5642415881156921,
      "rewards/margins": 2.0280508995056152,
      "rewards/rejected": -2.592292308807373,
      "step": 12278
    },
    {
      "epoch": 1.6067882555001227,
      "grad_norm": 15.623872756958008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6961196064949036,
      "logits/rejected": -0.45931580662727356,
      "logps/chosen": -405.385009765625,
      "logps/rejected": -430.4324645996094,
      "loss": 0.6021,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1614986658096313,
      "rewards/margins": 1.0639677047729492,
      "rewards/rejected": -2.225466251373291,
      "step": 12279
    },
    {
      "epoch": 1.6069191134374745,
      "grad_norm": 8.197242736816406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06137526035308838,
      "logits/rejected": 0.005955498665571213,
      "logps/chosen": -438.5537414550781,
      "logps/rejected": -477.2070617675781,
      "loss": 0.4792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7444652915000916,
      "rewards/margins": 2.077479362487793,
      "rewards/rejected": -2.8219447135925293,
      "step": 12280
    },
    {
      "epoch": 1.6070499713748263,
      "grad_norm": 8.050639152526855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12399991601705551,
      "logits/rejected": -0.007238693535327911,
      "logps/chosen": -679.163330078125,
      "logps/rejected": -627.5055541992188,
      "loss": 0.2593,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13155214488506317,
      "rewards/margins": 2.0758275985717773,
      "rewards/rejected": -2.2073795795440674,
      "step": 12281
    },
    {
      "epoch": 1.607180829312178,
      "grad_norm": 9.878175735473633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31553688645362854,
      "logits/rejected": -0.3840171694755554,
      "logps/chosen": -588.7861328125,
      "logps/rejected": -641.448974609375,
      "loss": 0.5269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8802019357681274,
      "rewards/margins": 1.4844305515289307,
      "rewards/rejected": -2.3646326065063477,
      "step": 12282
    },
    {
      "epoch": 1.6073116872495299,
      "grad_norm": 11.62994384765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5615525841712952,
      "logits/rejected": -0.4891318380832672,
      "logps/chosen": -437.6723937988281,
      "logps/rejected": -289.5445556640625,
      "loss": 0.5993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0854685306549072,
      "rewards/margins": 0.686549186706543,
      "rewards/rejected": -1.7720178365707397,
      "step": 12283
    },
    {
      "epoch": 1.6074425451868815,
      "grad_norm": 12.030252456665039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18128111958503723,
      "logits/rejected": -0.2901886999607086,
      "logps/chosen": -438.6957092285156,
      "logps/rejected": -413.7118225097656,
      "loss": 0.7023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.021315813064575,
      "rewards/margins": 0.3213321566581726,
      "rewards/rejected": -2.3426480293273926,
      "step": 12284
    },
    {
      "epoch": 1.6075734031242332,
      "grad_norm": 5.63617467880249,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05034545063972473,
      "logits/rejected": 0.003897339105606079,
      "logps/chosen": -539.4031982421875,
      "logps/rejected": -499.2810974121094,
      "loss": 0.2642,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0177032947540283,
      "rewards/margins": 1.8772995471954346,
      "rewards/rejected": -2.895002603530884,
      "step": 12285
    },
    {
      "epoch": 1.607704261061585,
      "grad_norm": 9.459587097167969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5094995498657227,
      "logits/rejected": -0.3981702923774719,
      "logps/chosen": -272.2289123535156,
      "logps/rejected": -323.82318115234375,
      "loss": 0.5543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.534033179283142,
      "rewards/margins": 1.1124615669250488,
      "rewards/rejected": -2.6464946269989014,
      "step": 12286
    },
    {
      "epoch": 1.6078351189989368,
      "grad_norm": 8.536338806152344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3444342017173767,
      "logits/rejected": -0.37489086389541626,
      "logps/chosen": -534.6405639648438,
      "logps/rejected": -410.36468505859375,
      "loss": 0.4823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7232046127319336,
      "rewards/margins": 0.8321613669395447,
      "rewards/rejected": -1.555366039276123,
      "step": 12287
    },
    {
      "epoch": 1.6079659769362884,
      "grad_norm": 7.816764831542969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4031904339790344,
      "logits/rejected": -0.3828113079071045,
      "logps/chosen": -520.3316040039062,
      "logps/rejected": -402.45526123046875,
      "loss": 0.4096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3084555864334106,
      "rewards/margins": 2.0172343254089355,
      "rewards/rejected": -3.3256900310516357,
      "step": 12288
    },
    {
      "epoch": 1.6080968348736402,
      "grad_norm": 6.743383407592773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17575803399085999,
      "logits/rejected": -0.456715852022171,
      "logps/chosen": -569.00634765625,
      "logps/rejected": -547.195068359375,
      "loss": 0.3751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46889522671699524,
      "rewards/margins": 1.6794275045394897,
      "rewards/rejected": -2.148322820663452,
      "step": 12289
    },
    {
      "epoch": 1.608227692810992,
      "grad_norm": 8.344552040100098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30852147936820984,
      "logits/rejected": -0.24813106656074524,
      "logps/chosen": -471.07830810546875,
      "logps/rejected": -500.442138671875,
      "loss": 0.3731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7127485871315002,
      "rewards/margins": 1.5586204528808594,
      "rewards/rejected": -2.271368980407715,
      "step": 12290
    },
    {
      "epoch": 1.6083585507483438,
      "grad_norm": 12.834884643554688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16234084963798523,
      "logits/rejected": -0.12856464087963104,
      "logps/chosen": -686.844482421875,
      "logps/rejected": -434.0204772949219,
      "loss": 0.3556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5699477195739746,
      "rewards/margins": 1.7511738538742065,
      "rewards/rejected": -3.3211214542388916,
      "step": 12291
    },
    {
      "epoch": 1.6084894086856956,
      "grad_norm": 4.8617730140686035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.056204814463853836,
      "logits/rejected": -0.07139521092176437,
      "logps/chosen": -745.453125,
      "logps/rejected": -436.05084228515625,
      "loss": 0.1852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3144954442977905,
      "rewards/margins": 2.0831542015075684,
      "rewards/rejected": -2.3976497650146484,
      "step": 12292
    },
    {
      "epoch": 1.6086202666230474,
      "grad_norm": 6.8627705574035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3217492997646332,
      "logits/rejected": -0.12995368242263794,
      "logps/chosen": -286.6806945800781,
      "logps/rejected": -335.3758544921875,
      "loss": 0.345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4543379545211792,
      "rewards/margins": 1.310302495956421,
      "rewards/rejected": -2.7646405696868896,
      "step": 12293
    },
    {
      "epoch": 1.6087511245603991,
      "grad_norm": 9.409119606018066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47254490852355957,
      "logits/rejected": -0.218826562166214,
      "logps/chosen": -439.6755065917969,
      "logps/rejected": -407.11053466796875,
      "loss": 0.5449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0994536876678467,
      "rewards/margins": 0.9523161053657532,
      "rewards/rejected": -2.051769971847534,
      "step": 12294
    },
    {
      "epoch": 1.608881982497751,
      "grad_norm": 13.115787506103516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38928914070129395,
      "logits/rejected": -0.3152449131011963,
      "logps/chosen": -458.09613037109375,
      "logps/rejected": -370.61328125,
      "loss": 0.4281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3785533905029297,
      "rewards/margins": 1.3119757175445557,
      "rewards/rejected": -2.6905291080474854,
      "step": 12295
    },
    {
      "epoch": 1.6090128404351027,
      "grad_norm": 10.998098373413086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4104805588722229,
      "logits/rejected": -0.3017313480377197,
      "logps/chosen": -453.73736572265625,
      "logps/rejected": -424.8702087402344,
      "loss": 0.5282,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2706072330474854,
      "rewards/margins": 0.759320855140686,
      "rewards/rejected": -2.029928207397461,
      "step": 12296
    },
    {
      "epoch": 1.6091436983724545,
      "grad_norm": 10.729571342468262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15388888120651245,
      "logits/rejected": -0.22038434445858002,
      "logps/chosen": -476.14337158203125,
      "logps/rejected": -489.7294616699219,
      "loss": 0.5414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7399196624755859,
      "rewards/margins": 0.5325038433074951,
      "rewards/rejected": -1.272423505783081,
      "step": 12297
    },
    {
      "epoch": 1.6092745563098063,
      "grad_norm": 10.698914527893066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2683723270893097,
      "logits/rejected": -0.2776675820350647,
      "logps/chosen": -561.8268432617188,
      "logps/rejected": -504.62701416015625,
      "loss": 0.5273,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4824063777923584,
      "rewards/margins": 1.0848058462142944,
      "rewards/rejected": -2.5672121047973633,
      "step": 12298
    },
    {
      "epoch": 1.6094054142471579,
      "grad_norm": 7.548573017120361,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40581995248794556,
      "logits/rejected": -0.47675821185112,
      "logps/chosen": -389.45391845703125,
      "logps/rejected": -517.2564697265625,
      "loss": 0.4146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.419157862663269,
      "rewards/margins": 0.8610485196113586,
      "rewards/rejected": -2.2802062034606934,
      "step": 12299
    },
    {
      "epoch": 1.6095362721845097,
      "grad_norm": 9.718905448913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24881896376609802,
      "logits/rejected": -0.2176211178302765,
      "logps/chosen": -388.3528137207031,
      "logps/rejected": -462.9277038574219,
      "loss": 0.6485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1638472080230713,
      "rewards/margins": 1.1184622049331665,
      "rewards/rejected": -2.2823095321655273,
      "step": 12300
    },
    {
      "epoch": 1.6096671301218615,
      "grad_norm": 8.652833938598633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19299280643463135,
      "logits/rejected": -0.13508060574531555,
      "logps/chosen": -557.9404907226562,
      "logps/rejected": -577.5081176757812,
      "loss": 0.4324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20392760634422302,
      "rewards/margins": 1.3537614345550537,
      "rewards/rejected": -1.5576889514923096,
      "step": 12301
    },
    {
      "epoch": 1.6097979880592133,
      "grad_norm": 5.8870744705200195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5895369648933411,
      "logits/rejected": -0.6926684975624084,
      "logps/chosen": -295.9052734375,
      "logps/rejected": -325.14337158203125,
      "loss": 0.3196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6624701023101807,
      "rewards/margins": 1.2921501398086548,
      "rewards/rejected": -2.954620361328125,
      "step": 12302
    },
    {
      "epoch": 1.6099288459965648,
      "grad_norm": 9.809061050415039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1276206523180008,
      "logits/rejected": -0.15374642610549927,
      "logps/chosen": -506.1998291015625,
      "logps/rejected": -440.4709167480469,
      "loss": 0.506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.961524248123169,
      "rewards/margins": 0.6255915760993958,
      "rewards/rejected": -1.58711576461792,
      "step": 12303
    },
    {
      "epoch": 1.6100597039339166,
      "grad_norm": 9.637199401855469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43298155069351196,
      "logits/rejected": -0.3029869794845581,
      "logps/chosen": -454.67266845703125,
      "logps/rejected": -472.29290771484375,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.082832932472229,
      "rewards/margins": 0.42308753728866577,
      "rewards/rejected": -1.50592041015625,
      "step": 12304
    },
    {
      "epoch": 1.6101905618712684,
      "grad_norm": 7.5960693359375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.449897825717926,
      "logits/rejected": -0.012872792780399323,
      "logps/chosen": -400.0588684082031,
      "logps/rejected": -447.3926086425781,
      "loss": 0.442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4257827699184418,
      "rewards/margins": 0.8047857284545898,
      "rewards/rejected": -1.2305684089660645,
      "step": 12305
    },
    {
      "epoch": 1.6103214198086202,
      "grad_norm": 7.599272727966309,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14724916219711304,
      "logits/rejected": -0.13145068287849426,
      "logps/chosen": -481.286865234375,
      "logps/rejected": -401.1571044921875,
      "loss": 0.435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4489521384239197,
      "rewards/margins": 1.7072179317474365,
      "rewards/rejected": -2.156170129776001,
      "step": 12306
    },
    {
      "epoch": 1.610452277745972,
      "grad_norm": 6.241687774658203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09711655974388123,
      "logits/rejected": -0.37455761432647705,
      "logps/chosen": -462.241943359375,
      "logps/rejected": -370.00201416015625,
      "loss": 0.3222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6607265472412109,
      "rewards/margins": 1.8895409107208252,
      "rewards/rejected": -2.550267457962036,
      "step": 12307
    },
    {
      "epoch": 1.6105831356833238,
      "grad_norm": 10.115259170532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27612584829330444,
      "logits/rejected": -0.3524337410926819,
      "logps/chosen": -508.7354431152344,
      "logps/rejected": -486.71044921875,
      "loss": 0.6015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2423291206359863,
      "rewards/margins": 0.8558393120765686,
      "rewards/rejected": -2.09816837310791,
      "step": 12308
    },
    {
      "epoch": 1.6107139936206756,
      "grad_norm": 8.354390144348145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3622921407222748,
      "logits/rejected": -0.3790891170501709,
      "logps/chosen": -705.4873657226562,
      "logps/rejected": -531.6530151367188,
      "loss": 0.3347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13404390215873718,
      "rewards/margins": 1.4866876602172852,
      "rewards/rejected": -1.3526437282562256,
      "step": 12309
    },
    {
      "epoch": 1.6108448515580274,
      "grad_norm": 1.9910939931869507,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2259978950023651,
      "logits/rejected": -0.15740162134170532,
      "logps/chosen": -494.757080078125,
      "logps/rejected": -435.49566650390625,
      "loss": 0.0847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14786651730537415,
      "rewards/margins": 2.646531343460083,
      "rewards/rejected": -2.7943978309631348,
      "step": 12310
    },
    {
      "epoch": 1.6109757094953792,
      "grad_norm": 10.757047653198242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19391536712646484,
      "logits/rejected": -0.2716633081436157,
      "logps/chosen": -381.2188415527344,
      "logps/rejected": -328.9326477050781,
      "loss": 0.742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4078071117401123,
      "rewards/margins": 1.2265055179595947,
      "rewards/rejected": -2.634312868118286,
      "step": 12311
    },
    {
      "epoch": 1.611106567432731,
      "grad_norm": 10.223992347717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.148248553276062,
      "logits/rejected": -0.24909663200378418,
      "logps/chosen": -601.8615112304688,
      "logps/rejected": -606.2012329101562,
      "loss": 0.543,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5441100001335144,
      "rewards/margins": 0.9883408546447754,
      "rewards/rejected": -1.5324509143829346,
      "step": 12312
    },
    {
      "epoch": 1.6112374253700827,
      "grad_norm": 6.531219482421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12948235869407654,
      "logits/rejected": -0.2557336390018463,
      "logps/chosen": -471.9267883300781,
      "logps/rejected": -643.7366333007812,
      "loss": 0.3317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7097950577735901,
      "rewards/margins": 1.311349630355835,
      "rewards/rejected": -2.0211448669433594,
      "step": 12313
    },
    {
      "epoch": 1.6113682833074343,
      "grad_norm": 10.651796340942383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11994470655918121,
      "logits/rejected": 0.08048371970653534,
      "logps/chosen": -455.9441833496094,
      "logps/rejected": -337.3514404296875,
      "loss": 0.5204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9473516941070557,
      "rewards/margins": 0.7841084599494934,
      "rewards/rejected": -1.7314600944519043,
      "step": 12314
    },
    {
      "epoch": 1.611499141244786,
      "grad_norm": 9.550870895385742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1422838717699051,
      "logits/rejected": -0.2947158217430115,
      "logps/chosen": -483.9757995605469,
      "logps/rejected": -310.22412109375,
      "loss": 0.5549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9747236371040344,
      "rewards/margins": 0.7123110890388489,
      "rewards/rejected": -1.6870348453521729,
      "step": 12315
    },
    {
      "epoch": 1.611629999182138,
      "grad_norm": 6.313661575317383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026368722319602966,
      "logits/rejected": -0.17706477642059326,
      "logps/chosen": -342.1278381347656,
      "logps/rejected": -351.9173889160156,
      "loss": 0.2543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.524949073791504,
      "rewards/margins": 1.8510184288024902,
      "rewards/rejected": -3.3759677410125732,
      "step": 12316
    },
    {
      "epoch": 1.6117608571194897,
      "grad_norm": 11.072653770446777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4022762179374695,
      "logits/rejected": -0.3189772367477417,
      "logps/chosen": -431.759765625,
      "logps/rejected": -416.2156677246094,
      "loss": 0.5689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3182584047317505,
      "rewards/margins": 0.8291452527046204,
      "rewards/rejected": -2.1474037170410156,
      "step": 12317
    },
    {
      "epoch": 1.6118917150568413,
      "grad_norm": 5.324456691741943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06767408549785614,
      "logits/rejected": -0.29455193877220154,
      "logps/chosen": -449.1997375488281,
      "logps/rejected": -450.57916259765625,
      "loss": 0.1889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6837612390518188,
      "rewards/margins": 2.2386200428009033,
      "rewards/rejected": -2.9223809242248535,
      "step": 12318
    },
    {
      "epoch": 1.612022572994193,
      "grad_norm": 13.049877166748047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35510149598121643,
      "logits/rejected": -0.6549670100212097,
      "logps/chosen": -571.1825561523438,
      "logps/rejected": -467.0618896484375,
      "loss": 0.5226,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1475956439971924,
      "rewards/margins": 0.9905071258544922,
      "rewards/rejected": -2.1381027698516846,
      "step": 12319
    },
    {
      "epoch": 1.6121534309315448,
      "grad_norm": 12.51006031036377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6093966960906982,
      "logits/rejected": -0.23852390050888062,
      "logps/chosen": -483.56591796875,
      "logps/rejected": -460.061767578125,
      "loss": 0.5316,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.661505937576294,
      "rewards/margins": 1.588062047958374,
      "rewards/rejected": -3.249568223953247,
      "step": 12320
    },
    {
      "epoch": 1.6122842888688966,
      "grad_norm": 8.497977256774902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42796018719673157,
      "logits/rejected": -0.38692718744277954,
      "logps/chosen": -387.99468994140625,
      "logps/rejected": -378.2360534667969,
      "loss": 0.5457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8562966585159302,
      "rewards/margins": 0.8110510110855103,
      "rewards/rejected": -2.6673476696014404,
      "step": 12321
    },
    {
      "epoch": 1.6124151468062484,
      "grad_norm": 15.32364559173584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22237573564052582,
      "logits/rejected": -0.2596569061279297,
      "logps/chosen": -570.511474609375,
      "logps/rejected": -424.822998046875,
      "loss": 1.1223,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4292572736740112,
      "rewards/margins": 0.012933388352394104,
      "rewards/rejected": -1.4421905279159546,
      "step": 12322
    },
    {
      "epoch": 1.6125460047436002,
      "grad_norm": 7.0695624351501465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027336496859788895,
      "logits/rejected": -0.2703109085559845,
      "logps/chosen": -486.09075927734375,
      "logps/rejected": -482.01483154296875,
      "loss": 0.298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2232520580291748,
      "rewards/margins": 1.986477017402649,
      "rewards/rejected": -3.2097291946411133,
      "step": 12323
    },
    {
      "epoch": 1.612676862680952,
      "grad_norm": 6.713502883911133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2672979235649109,
      "logits/rejected": -0.42674732208251953,
      "logps/chosen": -439.75238037109375,
      "logps/rejected": -455.6800231933594,
      "loss": 0.2651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6545734405517578,
      "rewards/margins": 2.312997579574585,
      "rewards/rejected": -2.967571258544922,
      "step": 12324
    },
    {
      "epoch": 1.6128077206183038,
      "grad_norm": 11.887758255004883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15086466073989868,
      "logits/rejected": -0.44907352328300476,
      "logps/chosen": -409.9033508300781,
      "logps/rejected": -357.25469970703125,
      "loss": 0.4882,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0998488664627075,
      "rewards/margins": 1.6242635250091553,
      "rewards/rejected": -2.7241125106811523,
      "step": 12325
    },
    {
      "epoch": 1.6129385785556556,
      "grad_norm": 9.12987232208252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05735528841614723,
      "logits/rejected": -0.08071447908878326,
      "logps/chosen": -405.8029479980469,
      "logps/rejected": -416.2208251953125,
      "loss": 0.5733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9564889669418335,
      "rewards/margins": 1.275491714477539,
      "rewards/rejected": -2.231980562210083,
      "step": 12326
    },
    {
      "epoch": 1.6130694364930074,
      "grad_norm": 9.723668098449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6195648312568665,
      "logits/rejected": -0.6797930002212524,
      "logps/chosen": -396.15692138671875,
      "logps/rejected": -378.7803039550781,
      "loss": 0.4756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6836069822311401,
      "rewards/margins": 1.044219732284546,
      "rewards/rejected": -1.727826714515686,
      "step": 12327
    },
    {
      "epoch": 1.6132002944303592,
      "grad_norm": 12.2940673828125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40028026700019836,
      "logits/rejected": -0.3522554934024811,
      "logps/chosen": -549.3709716796875,
      "logps/rejected": -683.6365966796875,
      "loss": 0.8291,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2069740295410156,
      "rewards/margins": 0.4235507845878601,
      "rewards/rejected": -1.6305248737335205,
      "step": 12328
    },
    {
      "epoch": 1.613331152367711,
      "grad_norm": 9.514049530029297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39776676893234253,
      "logits/rejected": -0.22167843580245972,
      "logps/chosen": -327.7952575683594,
      "logps/rejected": -394.2187194824219,
      "loss": 0.6347,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.754812479019165,
      "rewards/margins": 0.5739092230796814,
      "rewards/rejected": -1.3287216424942017,
      "step": 12329
    },
    {
      "epoch": 1.6134620103050625,
      "grad_norm": 5.878501892089844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15485024452209473,
      "logits/rejected": -0.4157036244869232,
      "logps/chosen": -492.3005676269531,
      "logps/rejected": -429.8810119628906,
      "loss": 0.3494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5557180643081665,
      "rewards/margins": 1.5821818113327026,
      "rewards/rejected": -2.137899875640869,
      "step": 12330
    },
    {
      "epoch": 1.6135928682424143,
      "grad_norm": 5.1312408447265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.291222482919693,
      "logits/rejected": -0.1537005752325058,
      "logps/chosen": -474.22296142578125,
      "logps/rejected": -551.414794921875,
      "loss": 0.324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22773677110671997,
      "rewards/margins": 1.6251139640808105,
      "rewards/rejected": -1.8528507947921753,
      "step": 12331
    },
    {
      "epoch": 1.6137237261797661,
      "grad_norm": 5.962197303771973,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07105115056037903,
      "logits/rejected": -0.22661155462265015,
      "logps/chosen": -565.9652709960938,
      "logps/rejected": -352.1942138671875,
      "loss": 0.2233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9757121801376343,
      "rewards/margins": 1.8678526878356934,
      "rewards/rejected": -2.843564987182617,
      "step": 12332
    },
    {
      "epoch": 1.613854584117118,
      "grad_norm": 12.185877799987793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3832721412181854,
      "logits/rejected": -0.07695120573043823,
      "logps/chosen": -487.1362609863281,
      "logps/rejected": -582.2847290039062,
      "loss": 0.7738,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8003708124160767,
      "rewards/margins": 0.4077102541923523,
      "rewards/rejected": -1.2080811262130737,
      "step": 12333
    },
    {
      "epoch": 1.6139854420544695,
      "grad_norm": 10.13034439086914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1686433106660843,
      "logits/rejected": -0.2702759802341461,
      "logps/chosen": -427.4799499511719,
      "logps/rejected": -444.2446594238281,
      "loss": 0.6048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45367681980133057,
      "rewards/margins": 1.068192958831787,
      "rewards/rejected": -1.5218698978424072,
      "step": 12334
    },
    {
      "epoch": 1.6141162999918213,
      "grad_norm": 10.28466510772705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18333229422569275,
      "logits/rejected": -0.15384145081043243,
      "logps/chosen": -523.455322265625,
      "logps/rejected": -555.3914794921875,
      "loss": 0.724,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5834890604019165,
      "rewards/margins": 1.0592994689941406,
      "rewards/rejected": -1.6427884101867676,
      "step": 12335
    },
    {
      "epoch": 1.614247157929173,
      "grad_norm": 7.101367950439453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2156582921743393,
      "logits/rejected": -0.3908894956111908,
      "logps/chosen": -408.1355285644531,
      "logps/rejected": -586.3181762695312,
      "loss": 0.2418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38225966691970825,
      "rewards/margins": 2.4193544387817383,
      "rewards/rejected": -2.8016138076782227,
      "step": 12336
    },
    {
      "epoch": 1.6143780158665249,
      "grad_norm": 7.944795608520508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2161892205476761,
      "logits/rejected": -0.669850766658783,
      "logps/chosen": -596.8110961914062,
      "logps/rejected": -465.8760986328125,
      "loss": 0.2923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4895913600921631,
      "rewards/margins": 2.406487464904785,
      "rewards/rejected": -2.8960790634155273,
      "step": 12337
    },
    {
      "epoch": 1.6145088738038766,
      "grad_norm": 10.510074615478516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5553862452507019,
      "logits/rejected": -0.3121013641357422,
      "logps/chosen": -405.84332275390625,
      "logps/rejected": -468.524658203125,
      "loss": 0.4883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.546005368232727,
      "rewards/margins": 0.8699967861175537,
      "rewards/rejected": -2.416002035140991,
      "step": 12338
    },
    {
      "epoch": 1.6146397317412284,
      "grad_norm": 12.000476837158203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23682089149951935,
      "logits/rejected": -0.3108402192592621,
      "logps/chosen": -639.4396362304688,
      "logps/rejected": -486.12127685546875,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.02054762840271,
      "rewards/margins": 0.9267581105232239,
      "rewards/rejected": -1.9473057985305786,
      "step": 12339
    },
    {
      "epoch": 1.6147705896785802,
      "grad_norm": 16.25864028930664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010348922573029995,
      "logits/rejected": -0.06537964940071106,
      "logps/chosen": -576.1034545898438,
      "logps/rejected": -608.4541625976562,
      "loss": 0.8722,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1058311462402344,
      "rewards/margins": 0.3952043652534485,
      "rewards/rejected": -1.501035451889038,
      "step": 12340
    },
    {
      "epoch": 1.614901447615932,
      "grad_norm": 4.908670425415039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2145405113697052,
      "logits/rejected": -0.15945973992347717,
      "logps/chosen": -558.5554809570312,
      "logps/rejected": -581.436279296875,
      "loss": 0.2546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3150964975357056,
      "rewards/margins": 1.6726306676864624,
      "rewards/rejected": -2.987727403640747,
      "step": 12341
    },
    {
      "epoch": 1.6150323055532838,
      "grad_norm": 7.023016452789307,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20302234590053558,
      "logits/rejected": -0.1350439488887787,
      "logps/chosen": -419.02520751953125,
      "logps/rejected": -444.2438659667969,
      "loss": 0.2824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5267238616943359,
      "rewards/margins": 1.7211793661117554,
      "rewards/rejected": -2.247903347015381,
      "step": 12342
    },
    {
      "epoch": 1.6151631634906356,
      "grad_norm": 8.404367446899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1982678472995758,
      "logits/rejected": -0.1308547407388687,
      "logps/chosen": -425.3418273925781,
      "logps/rejected": -441.7594299316406,
      "loss": 0.4076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5856544971466064,
      "rewards/margins": 1.0153392553329468,
      "rewards/rejected": -2.6009936332702637,
      "step": 12343
    },
    {
      "epoch": 1.6152940214279874,
      "grad_norm": 5.804149150848389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03622633218765259,
      "logits/rejected": -0.395435631275177,
      "logps/chosen": -343.171630859375,
      "logps/rejected": -413.7999267578125,
      "loss": 0.2061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6419126987457275,
      "rewards/margins": 1.5750677585601807,
      "rewards/rejected": -3.216980218887329,
      "step": 12344
    },
    {
      "epoch": 1.615424879365339,
      "grad_norm": 7.706183910369873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14191655814647675,
      "logits/rejected": -0.3047933578491211,
      "logps/chosen": -322.39501953125,
      "logps/rejected": -338.242919921875,
      "loss": 0.5188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.450177788734436,
      "rewards/margins": 0.9192575216293335,
      "rewards/rejected": -2.3694353103637695,
      "step": 12345
    },
    {
      "epoch": 1.6155557373026908,
      "grad_norm": 7.438366889953613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13303214311599731,
      "logits/rejected": -0.45767641067504883,
      "logps/chosen": -377.3768005371094,
      "logps/rejected": -450.19134521484375,
      "loss": 0.357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.301177740097046,
      "rewards/margins": 1.658102035522461,
      "rewards/rejected": -2.959280014038086,
      "step": 12346
    },
    {
      "epoch": 1.6156865952400425,
      "grad_norm": 8.103591918945312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2969934046268463,
      "logits/rejected": -0.36982885003089905,
      "logps/chosen": -402.4354553222656,
      "logps/rejected": -302.9122314453125,
      "loss": 0.4572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9188109636306763,
      "rewards/margins": 0.8997266888618469,
      "rewards/rejected": -1.8185378313064575,
      "step": 12347
    },
    {
      "epoch": 1.6158174531773943,
      "grad_norm": 7.7928853034973145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1454716920852661,
      "logits/rejected": -0.35046106576919556,
      "logps/chosen": -462.8045959472656,
      "logps/rejected": -461.4195251464844,
      "loss": 0.3212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5371741056442261,
      "rewards/margins": 1.248615026473999,
      "rewards/rejected": -1.785789132118225,
      "step": 12348
    },
    {
      "epoch": 1.615948311114746,
      "grad_norm": 9.434224128723145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2818164825439453,
      "logits/rejected": -0.3138333261013031,
      "logps/chosen": -431.0251159667969,
      "logps/rejected": -386.85626220703125,
      "loss": 0.5257,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2420212030410767,
      "rewards/margins": 0.7600008249282837,
      "rewards/rejected": -2.0020220279693604,
      "step": 12349
    },
    {
      "epoch": 1.6160791690520977,
      "grad_norm": 6.0514702796936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3213953375816345,
      "logits/rejected": -0.30251818895339966,
      "logps/chosen": -674.472900390625,
      "logps/rejected": -613.4366455078125,
      "loss": 0.2826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6527721285820007,
      "rewards/margins": 1.3864169120788574,
      "rewards/rejected": -2.039188861846924,
      "step": 12350
    },
    {
      "epoch": 1.6162100269894495,
      "grad_norm": 17.069198608398438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3441629409790039,
      "logits/rejected": -0.3963344693183899,
      "logps/chosen": -675.1427001953125,
      "logps/rejected": -519.2969970703125,
      "loss": 0.7928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6980257034301758,
      "rewards/margins": 0.316474974155426,
      "rewards/rejected": -2.014500617980957,
      "step": 12351
    },
    {
      "epoch": 1.6163408849268013,
      "grad_norm": 6.518641948699951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2988629937171936,
      "logits/rejected": -0.3522665202617645,
      "logps/chosen": -478.92230224609375,
      "logps/rejected": -580.7215576171875,
      "loss": 0.3022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24166104197502136,
      "rewards/margins": 1.7219406366348267,
      "rewards/rejected": -1.96360182762146,
      "step": 12352
    },
    {
      "epoch": 1.616471742864153,
      "grad_norm": 8.536051750183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21756736934185028,
      "logits/rejected": -0.25754502415657043,
      "logps/chosen": -504.96923828125,
      "logps/rejected": -373.0162353515625,
      "loss": 0.4687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8652421832084656,
      "rewards/margins": 1.1632556915283203,
      "rewards/rejected": -2.0284979343414307,
      "step": 12353
    },
    {
      "epoch": 1.6166026008015049,
      "grad_norm": 7.672869682312012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18543821573257446,
      "logits/rejected": -0.475481241941452,
      "logps/chosen": -745.909912109375,
      "logps/rejected": -588.9341430664062,
      "loss": 0.3655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2398627996444702,
      "rewards/margins": 0.9740573763847351,
      "rewards/rejected": -2.2139203548431396,
      "step": 12354
    },
    {
      "epoch": 1.6167334587388567,
      "grad_norm": 7.471781253814697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1337842345237732,
      "logits/rejected": -0.2041613608598709,
      "logps/chosen": -392.57049560546875,
      "logps/rejected": -418.941650390625,
      "loss": 0.4036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1377041339874268,
      "rewards/margins": 1.677625060081482,
      "rewards/rejected": -2.815329074859619,
      "step": 12355
    },
    {
      "epoch": 1.6168643166762084,
      "grad_norm": 7.780810832977295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007393326610326767,
      "logits/rejected": -0.13540233671665192,
      "logps/chosen": -750.875244140625,
      "logps/rejected": -602.8861083984375,
      "loss": 0.3694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7824331521987915,
      "rewards/margins": 1.2099361419677734,
      "rewards/rejected": -1.992369294166565,
      "step": 12356
    },
    {
      "epoch": 1.6169951746135602,
      "grad_norm": 7.876255989074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2875464856624603,
      "logits/rejected": -0.24029183387756348,
      "logps/chosen": -691.2101440429688,
      "logps/rejected": -521.2468872070312,
      "loss": 0.4392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1661030054092407,
      "rewards/margins": 0.7157136797904968,
      "rewards/rejected": -1.8818166255950928,
      "step": 12357
    },
    {
      "epoch": 1.617126032550912,
      "grad_norm": 11.488637924194336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5378079414367676,
      "logits/rejected": -0.5788863897323608,
      "logps/chosen": -430.23040771484375,
      "logps/rejected": -478.6191101074219,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3802201747894287,
      "rewards/margins": 1.0481624603271484,
      "rewards/rejected": -2.428382635116577,
      "step": 12358
    },
    {
      "epoch": 1.6172568904882638,
      "grad_norm": 8.788124084472656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10511602461338043,
      "logits/rejected": -0.4070194661617279,
      "logps/chosen": -519.3378295898438,
      "logps/rejected": -428.6695861816406,
      "loss": 0.3147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8618646860122681,
      "rewards/margins": 1.7731480598449707,
      "rewards/rejected": -2.635012626647949,
      "step": 12359
    },
    {
      "epoch": 1.6173877484256154,
      "grad_norm": 10.10334587097168,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05115240067243576,
      "logits/rejected": -0.04809603840112686,
      "logps/chosen": -755.3677368164062,
      "logps/rejected": -590.5592041015625,
      "loss": 0.5232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1029900312423706,
      "rewards/margins": 1.3476183414459229,
      "rewards/rejected": -2.450608253479004,
      "step": 12360
    },
    {
      "epoch": 1.6175186063629672,
      "grad_norm": 7.1137471199035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002186611294746399,
      "logits/rejected": -0.3256719708442688,
      "logps/chosen": -410.0296325683594,
      "logps/rejected": -385.6028747558594,
      "loss": 0.2478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2914273738861084,
      "rewards/margins": 2.03139066696167,
      "rewards/rejected": -2.3228180408477783,
      "step": 12361
    },
    {
      "epoch": 1.617649464300319,
      "grad_norm": 14.764078140258789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6387697458267212,
      "logits/rejected": -0.5408090353012085,
      "logps/chosen": -495.4542236328125,
      "logps/rejected": -458.2268981933594,
      "loss": 1.0315,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.2115726470947266,
      "rewards/margins": 0.3000950813293457,
      "rewards/rejected": -2.5116679668426514,
      "step": 12362
    },
    {
      "epoch": 1.6177803222376708,
      "grad_norm": 15.865955352783203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38512423634529114,
      "logits/rejected": -0.24669848382472992,
      "logps/chosen": -597.9686279296875,
      "logps/rejected": -550.6686401367188,
      "loss": 0.753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9374029636383057,
      "rewards/margins": 0.9096353650093079,
      "rewards/rejected": -1.8470383882522583,
      "step": 12363
    },
    {
      "epoch": 1.6179111801750223,
      "grad_norm": 8.84420108795166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21813905239105225,
      "logits/rejected": -0.43871772289276123,
      "logps/chosen": -611.9832763671875,
      "logps/rejected": -478.1488037109375,
      "loss": 0.547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18296566605567932,
      "rewards/margins": 1.5179274082183838,
      "rewards/rejected": -1.7008931636810303,
      "step": 12364
    },
    {
      "epoch": 1.6180420381123741,
      "grad_norm": 4.5054612159729,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13306863605976105,
      "logits/rejected": -0.15624859929084778,
      "logps/chosen": -380.7064514160156,
      "logps/rejected": -446.9391784667969,
      "loss": 0.1735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17941361665725708,
      "rewards/margins": 2.4525744915008545,
      "rewards/rejected": -2.6319878101348877,
      "step": 12365
    },
    {
      "epoch": 1.618172896049726,
      "grad_norm": 10.61237907409668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23577150702476501,
      "logits/rejected": -0.2019215226173401,
      "logps/chosen": -438.6482849121094,
      "logps/rejected": -466.8385009765625,
      "loss": 0.6746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.908793330192566,
      "rewards/margins": 1.0848580598831177,
      "rewards/rejected": -2.9936513900756836,
      "step": 12366
    },
    {
      "epoch": 1.6183037539870777,
      "grad_norm": 3.9718031883239746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2913461923599243,
      "logits/rejected": -0.35764607787132263,
      "logps/chosen": -577.0577392578125,
      "logps/rejected": -445.7832946777344,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2467426210641861,
      "rewards/margins": 2.3295986652374268,
      "rewards/rejected": -2.576341152191162,
      "step": 12367
    },
    {
      "epoch": 1.6184346119244295,
      "grad_norm": 8.777044296264648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35716891288757324,
      "logits/rejected": -0.257575660943985,
      "logps/chosen": -341.137939453125,
      "logps/rejected": -369.952392578125,
      "loss": 0.4552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8444037437438965,
      "rewards/margins": 0.968037486076355,
      "rewards/rejected": -1.8124412298202515,
      "step": 12368
    },
    {
      "epoch": 1.6185654698617813,
      "grad_norm": 11.102493286132812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031608518213033676,
      "logits/rejected": -0.3104594349861145,
      "logps/chosen": -629.3265380859375,
      "logps/rejected": -501.02508544921875,
      "loss": 0.5361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.350659966468811,
      "rewards/margins": 1.5509618520736694,
      "rewards/rejected": -2.9016218185424805,
      "step": 12369
    },
    {
      "epoch": 1.618696327799133,
      "grad_norm": 9.350400924682617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13692283630371094,
      "logits/rejected": -0.1633571982383728,
      "logps/chosen": -425.3958435058594,
      "logps/rejected": -333.2575378417969,
      "loss": 0.6849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.651161789894104,
      "rewards/margins": 0.5100964903831482,
      "rewards/rejected": -2.1612582206726074,
      "step": 12370
    },
    {
      "epoch": 1.6188271857364849,
      "grad_norm": 7.618694305419922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2783256471157074,
      "logits/rejected": -0.16140322387218475,
      "logps/chosen": -523.5011596679688,
      "logps/rejected": -508.0672912597656,
      "loss": 0.3379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4186595678329468,
      "rewards/margins": 1.5346078872680664,
      "rewards/rejected": -1.9532674551010132,
      "step": 12371
    },
    {
      "epoch": 1.6189580436738367,
      "grad_norm": 14.37968635559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23372358083724976,
      "logits/rejected": -0.2896130681037903,
      "logps/chosen": -425.2702941894531,
      "logps/rejected": -581.44580078125,
      "loss": 0.5825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.066695213317871,
      "rewards/margins": 0.557512104511261,
      "rewards/rejected": -2.6242074966430664,
      "step": 12372
    },
    {
      "epoch": 1.6190889016111885,
      "grad_norm": 8.661042213439941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2884361743927002,
      "logits/rejected": -0.3955264091491699,
      "logps/chosen": -693.223876953125,
      "logps/rejected": -558.2964477539062,
      "loss": 0.3255,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47484973073005676,
      "rewards/margins": 1.758694052696228,
      "rewards/rejected": -2.233543872833252,
      "step": 12373
    },
    {
      "epoch": 1.6192197595485402,
      "grad_norm": 11.14256763458252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3694174289703369,
      "logits/rejected": -0.31306901574134827,
      "logps/chosen": -485.9472351074219,
      "logps/rejected": -505.5380859375,
      "loss": 0.4527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1960877180099487,
      "rewards/margins": 1.1951537132263184,
      "rewards/rejected": -2.3912413120269775,
      "step": 12374
    },
    {
      "epoch": 1.619350617485892,
      "grad_norm": 12.118992805480957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3694763481616974,
      "logits/rejected": -0.424286425113678,
      "logps/chosen": -444.7654113769531,
      "logps/rejected": -342.1949157714844,
      "loss": 0.7873,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6594762802124023,
      "rewards/margins": 0.18029478192329407,
      "rewards/rejected": -0.8397710919380188,
      "step": 12375
    },
    {
      "epoch": 1.6194814754232436,
      "grad_norm": 10.791131019592285,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08589600771665573,
      "logits/rejected": -0.04806774854660034,
      "logps/chosen": -350.2174377441406,
      "logps/rejected": -439.118408203125,
      "loss": 0.6198,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1598507165908813,
      "rewards/margins": 0.914161205291748,
      "rewards/rejected": -2.074012041091919,
      "step": 12376
    },
    {
      "epoch": 1.6196123333605954,
      "grad_norm": 10.064216613769531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09820840507745743,
      "logits/rejected": -0.1299029439687729,
      "logps/chosen": -548.1575927734375,
      "logps/rejected": -522.0826416015625,
      "loss": 0.559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9145517349243164,
      "rewards/margins": 0.868774950504303,
      "rewards/rejected": -1.7833266258239746,
      "step": 12377
    },
    {
      "epoch": 1.6197431912979472,
      "grad_norm": 8.849662780761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19213566184043884,
      "logits/rejected": -0.25244060158729553,
      "logps/chosen": -416.017822265625,
      "logps/rejected": -370.8921203613281,
      "loss": 0.4584,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.070794701576233,
      "rewards/margins": 1.003947138786316,
      "rewards/rejected": -2.074741840362549,
      "step": 12378
    },
    {
      "epoch": 1.6198740492352988,
      "grad_norm": 7.769619464874268,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14443708956241608,
      "logits/rejected": -0.08217666298151016,
      "logps/chosen": -513.0598754882812,
      "logps/rejected": -519.333984375,
      "loss": 0.3153,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31273287534713745,
      "rewards/margins": 1.6109027862548828,
      "rewards/rejected": -1.9236356019973755,
      "step": 12379
    },
    {
      "epoch": 1.6200049071726506,
      "grad_norm": 6.9734601974487305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08845549076795578,
      "logits/rejected": -0.0188909824937582,
      "logps/chosen": -363.5347900390625,
      "logps/rejected": -403.8059997558594,
      "loss": 0.4481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9650153517723083,
      "rewards/margins": 1.055112361907959,
      "rewards/rejected": -2.020127773284912,
      "step": 12380
    },
    {
      "epoch": 1.6201357651100023,
      "grad_norm": 13.015286445617676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43669068813323975,
      "logits/rejected": -0.3176140785217285,
      "logps/chosen": -405.8232421875,
      "logps/rejected": -471.4805908203125,
      "loss": 0.9509,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.089331865310669,
      "rewards/margins": -0.03534048795700073,
      "rewards/rejected": -1.0539913177490234,
      "step": 12381
    },
    {
      "epoch": 1.6202666230473541,
      "grad_norm": 8.138100624084473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04102442413568497,
      "logits/rejected": -0.38854140043258667,
      "logps/chosen": -536.872314453125,
      "logps/rejected": -537.8821411132812,
      "loss": 0.3322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.510964274406433,
      "rewards/margins": 1.4220187664031982,
      "rewards/rejected": -2.932982921600342,
      "step": 12382
    },
    {
      "epoch": 1.620397480984706,
      "grad_norm": 7.610584259033203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14632347226142883,
      "logits/rejected": -0.18496665358543396,
      "logps/chosen": -383.8159484863281,
      "logps/rejected": -325.3971252441406,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1409807205200195,
      "rewards/margins": 1.0137100219726562,
      "rewards/rejected": -2.154690742492676,
      "step": 12383
    },
    {
      "epoch": 1.6205283389220577,
      "grad_norm": 10.069363594055176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07666612416505814,
      "logits/rejected": -0.21458587050437927,
      "logps/chosen": -355.62933349609375,
      "logps/rejected": -383.2857666015625,
      "loss": 0.6818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0409488677978516,
      "rewards/margins": 0.5831174850463867,
      "rewards/rejected": -1.6240663528442383,
      "step": 12384
    },
    {
      "epoch": 1.6206591968594095,
      "grad_norm": 13.785871505737305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2279420793056488,
      "logits/rejected": -0.14183726906776428,
      "logps/chosen": -315.4903259277344,
      "logps/rejected": -365.82989501953125,
      "loss": 0.6608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9788498282432556,
      "rewards/margins": 1.176020622253418,
      "rewards/rejected": -2.1548705101013184,
      "step": 12385
    },
    {
      "epoch": 1.6207900547967613,
      "grad_norm": 16.56498908996582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046266425400972366,
      "logits/rejected": -0.09907807409763336,
      "logps/chosen": -625.7536010742188,
      "logps/rejected": -596.437744140625,
      "loss": 0.968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9113128185272217,
      "rewards/margins": 1.0260270833969116,
      "rewards/rejected": -1.9373395442962646,
      "step": 12386
    },
    {
      "epoch": 1.620920912734113,
      "grad_norm": 9.701205253601074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6019461154937744,
      "logits/rejected": -0.4391743540763855,
      "logps/chosen": -524.8797607421875,
      "logps/rejected": -478.41644287109375,
      "loss": 0.4991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1262091398239136,
      "rewards/margins": 1.2468316555023193,
      "rewards/rejected": -2.3730406761169434,
      "step": 12387
    },
    {
      "epoch": 1.6210517706714649,
      "grad_norm": 12.515629768371582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.041975051164627075,
      "logits/rejected": -0.1303177773952484,
      "logps/chosen": -440.9075622558594,
      "logps/rejected": -385.8689880371094,
      "loss": 0.6406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.310286283493042,
      "rewards/margins": 1.1008665561676025,
      "rewards/rejected": -2.4111528396606445,
      "step": 12388
    },
    {
      "epoch": 1.6211826286088167,
      "grad_norm": 10.61857795715332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35670405626296997,
      "logits/rejected": -0.3805122971534729,
      "logps/chosen": -292.2405700683594,
      "logps/rejected": -399.3880615234375,
      "loss": 0.4617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8169902563095093,
      "rewards/margins": 1.3141200542449951,
      "rewards/rejected": -3.131110429763794,
      "step": 12389
    },
    {
      "epoch": 1.6213134865461685,
      "grad_norm": 7.133853435516357,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0019109249114990234,
      "logits/rejected": -0.29025816917419434,
      "logps/chosen": -275.0465393066406,
      "logps/rejected": -217.82186889648438,
      "loss": 0.509,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5981794595718384,
      "rewards/margins": 0.7858408689498901,
      "rewards/rejected": -1.3840203285217285,
      "step": 12390
    },
    {
      "epoch": 1.62144434448352,
      "grad_norm": 13.826647758483887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38077059388160706,
      "logits/rejected": -0.1583838313817978,
      "logps/chosen": -496.98895263671875,
      "logps/rejected": -488.49652099609375,
      "loss": 0.9766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2863779067993164,
      "rewards/margins": 0.35496509075164795,
      "rewards/rejected": -1.6413428783416748,
      "step": 12391
    },
    {
      "epoch": 1.6215752024208718,
      "grad_norm": 5.38869571685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12499735504388809,
      "logits/rejected": -0.08232459425926208,
      "logps/chosen": -397.2960510253906,
      "logps/rejected": -288.1021728515625,
      "loss": 0.2298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.60249924659729,
      "rewards/margins": 2.3772735595703125,
      "rewards/rejected": -2.9797725677490234,
      "step": 12392
    },
    {
      "epoch": 1.6217060603582236,
      "grad_norm": 6.199689865112305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09624700248241425,
      "logits/rejected": -0.1931854635477066,
      "logps/chosen": -568.6888427734375,
      "logps/rejected": -417.5689697265625,
      "loss": 0.3202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9979344010353088,
      "rewards/margins": 1.5401484966278076,
      "rewards/rejected": -2.538083076477051,
      "step": 12393
    },
    {
      "epoch": 1.6218369182955754,
      "grad_norm": 8.4929780960083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1311919093132019,
      "logits/rejected": -0.3131265342235565,
      "logps/chosen": -473.49346923828125,
      "logps/rejected": -428.78826904296875,
      "loss": 0.3355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6025459170341492,
      "rewards/margins": 1.4374690055847168,
      "rewards/rejected": -2.0400147438049316,
      "step": 12394
    },
    {
      "epoch": 1.621967776232927,
      "grad_norm": 7.575630187988281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15672937035560608,
      "logits/rejected": -0.20046573877334595,
      "logps/chosen": -467.0072021484375,
      "logps/rejected": -457.2146911621094,
      "loss": 0.36,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5791807174682617,
      "rewards/margins": 1.0542373657226562,
      "rewards/rejected": -2.633418083190918,
      "step": 12395
    },
    {
      "epoch": 1.6220986341702788,
      "grad_norm": 6.466512680053711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28414157032966614,
      "logits/rejected": -0.2333371937274933,
      "logps/chosen": -336.48602294921875,
      "logps/rejected": -317.01824951171875,
      "loss": 0.3623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17851600050926208,
      "rewards/margins": 1.1573245525360107,
      "rewards/rejected": -1.3358405828475952,
      "step": 12396
    },
    {
      "epoch": 1.6222294921076306,
      "grad_norm": 7.622833728790283,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018778745085000992,
      "logits/rejected": -0.10315603017807007,
      "logps/chosen": -661.3013916015625,
      "logps/rejected": -576.9798583984375,
      "loss": 0.3528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6859645247459412,
      "rewards/margins": 1.9118409156799316,
      "rewards/rejected": -1.2258762121200562,
      "step": 12397
    },
    {
      "epoch": 1.6223603500449824,
      "grad_norm": 16.22882652282715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3227499723434448,
      "logits/rejected": 0.02538580447435379,
      "logps/chosen": -442.99627685546875,
      "logps/rejected": -452.48199462890625,
      "loss": 0.7379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.278565764427185,
      "rewards/margins": 0.2551115155220032,
      "rewards/rejected": -1.533677339553833,
      "step": 12398
    },
    {
      "epoch": 1.6224912079823341,
      "grad_norm": 5.923842906951904,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02941131591796875,
      "logits/rejected": -0.1893208920955658,
      "logps/chosen": -701.0765991210938,
      "logps/rejected": -545.8052978515625,
      "loss": 0.2894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.007310668006539345,
      "rewards/margins": 1.7321797609329224,
      "rewards/rejected": -1.7248691320419312,
      "step": 12399
    },
    {
      "epoch": 1.622622065919686,
      "grad_norm": 7.46971321105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4081284999847412,
      "logits/rejected": -0.7140413522720337,
      "logps/chosen": -425.8445129394531,
      "logps/rejected": -454.5821533203125,
      "loss": 0.3178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6116651296615601,
      "rewards/margins": 1.4959808588027954,
      "rewards/rejected": -2.1076459884643555,
      "step": 12400
    },
    {
      "epoch": 1.6227529238570377,
      "grad_norm": 8.602578163146973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34725362062454224,
      "logits/rejected": -0.2747424244880676,
      "logps/chosen": -472.1571960449219,
      "logps/rejected": -651.4303588867188,
      "loss": 0.437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24049682915210724,
      "rewards/margins": 1.8387839794158936,
      "rewards/rejected": -1.5982871055603027,
      "step": 12401
    },
    {
      "epoch": 1.6228837817943895,
      "grad_norm": 5.969504356384277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2462579607963562,
      "logits/rejected": -0.3325140178203583,
      "logps/chosen": -498.4667663574219,
      "logps/rejected": -531.73828125,
      "loss": 0.3143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4142313003540039,
      "rewards/margins": 1.4974476099014282,
      "rewards/rejected": -1.9116789102554321,
      "step": 12402
    },
    {
      "epoch": 1.6230146397317413,
      "grad_norm": 7.257445812225342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14866985380649567,
      "logits/rejected": -0.31471896171569824,
      "logps/chosen": -525.1492919921875,
      "logps/rejected": -483.2638854980469,
      "loss": 0.3209,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7912710905075073,
      "rewards/margins": 1.4437525272369385,
      "rewards/rejected": -2.235023260116577,
      "step": 12403
    },
    {
      "epoch": 1.623145497669093,
      "grad_norm": 12.439600944519043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23690907657146454,
      "logits/rejected": -0.2851032018661499,
      "logps/chosen": -704.7869873046875,
      "logps/rejected": -485.4717712402344,
      "loss": 0.7942,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47545090317726135,
      "rewards/margins": 0.7073700428009033,
      "rewards/rejected": -1.1828209161758423,
      "step": 12404
    },
    {
      "epoch": 1.623276355606445,
      "grad_norm": 6.4649977684021,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2724650800228119,
      "logits/rejected": -0.28877344727516174,
      "logps/chosen": -408.8499450683594,
      "logps/rejected": -352.86468505859375,
      "loss": 0.3702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.011446177959442139,
      "rewards/margins": 1.4130254983901978,
      "rewards/rejected": -1.4244716167449951,
      "step": 12405
    },
    {
      "epoch": 1.6234072135437965,
      "grad_norm": 10.07517147064209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06785069406032562,
      "logits/rejected": -0.18370001018047333,
      "logps/chosen": -482.9818115234375,
      "logps/rejected": -397.6233215332031,
      "loss": 0.709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7563889026641846,
      "rewards/margins": 0.17183643579483032,
      "rewards/rejected": -0.9282254576683044,
      "step": 12406
    },
    {
      "epoch": 1.6235380714811483,
      "grad_norm": 7.783456802368164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2519967555999756,
      "logits/rejected": 0.022141121327877045,
      "logps/chosen": -603.1937866210938,
      "logps/rejected": -476.54620361328125,
      "loss": 0.4111,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9745671153068542,
      "rewards/margins": 1.1069681644439697,
      "rewards/rejected": -2.0815351009368896,
      "step": 12407
    },
    {
      "epoch": 1.6236689294185,
      "grad_norm": 8.873516082763672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18471160531044006,
      "logits/rejected": -0.3010377287864685,
      "logps/chosen": -476.39984130859375,
      "logps/rejected": -490.1675109863281,
      "loss": 0.4423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4816225469112396,
      "rewards/margins": 1.075120210647583,
      "rewards/rejected": -1.5567426681518555,
      "step": 12408
    },
    {
      "epoch": 1.6237997873558518,
      "grad_norm": 7.9029059410095215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1334449201822281,
      "logits/rejected": -0.39330434799194336,
      "logps/chosen": -597.3928833007812,
      "logps/rejected": -460.8648986816406,
      "loss": 0.4202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26173707842826843,
      "rewards/margins": 1.1624914407730103,
      "rewards/rejected": -1.4242284297943115,
      "step": 12409
    },
    {
      "epoch": 1.6239306452932034,
      "grad_norm": 8.764581680297852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4109630584716797,
      "logits/rejected": -0.4331411123275757,
      "logps/chosen": -438.3986511230469,
      "logps/rejected": -429.0661926269531,
      "loss": 0.3957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8900042176246643,
      "rewards/margins": 0.8484092950820923,
      "rewards/rejected": -1.7384135723114014,
      "step": 12410
    },
    {
      "epoch": 1.6240615032305552,
      "grad_norm": 10.56834602355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017737308517098427,
      "logits/rejected": 0.13381335139274597,
      "logps/chosen": -531.05224609375,
      "logps/rejected": -396.7494201660156,
      "loss": 0.4709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9362380504608154,
      "rewards/margins": 1.3888590335845947,
      "rewards/rejected": -2.3250973224639893,
      "step": 12411
    },
    {
      "epoch": 1.624192361167907,
      "grad_norm": 8.155420303344727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19714400172233582,
      "logits/rejected": -0.22215159237384796,
      "logps/chosen": -416.5054931640625,
      "logps/rejected": -503.682373046875,
      "loss": 0.3471,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3493286371231079,
      "rewards/margins": 1.170827865600586,
      "rewards/rejected": -1.5201566219329834,
      "step": 12412
    },
    {
      "epoch": 1.6243232191052588,
      "grad_norm": 10.915212631225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.251621812582016,
      "logits/rejected": -0.3344414532184601,
      "logps/chosen": -521.7603149414062,
      "logps/rejected": -491.6360168457031,
      "loss": 0.5244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13360480964183807,
      "rewards/margins": 1.8667575120925903,
      "rewards/rejected": -2.0003621578216553,
      "step": 12413
    },
    {
      "epoch": 1.6244540770426106,
      "grad_norm": 9.86497688293457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27274996042251587,
      "logits/rejected": -0.49718189239501953,
      "logps/chosen": -536.8626708984375,
      "logps/rejected": -462.386474609375,
      "loss": 0.3838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7831299304962158,
      "rewards/margins": 1.4685070514678955,
      "rewards/rejected": -2.2516369819641113,
      "step": 12414
    },
    {
      "epoch": 1.6245849349799624,
      "grad_norm": 7.971275806427002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38494646549224854,
      "logits/rejected": -0.1649375557899475,
      "logps/chosen": -424.0870361328125,
      "logps/rejected": -479.06121826171875,
      "loss": 0.5121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9418569803237915,
      "rewards/margins": 0.5072934627532959,
      "rewards/rejected": -1.4491504430770874,
      "step": 12415
    },
    {
      "epoch": 1.6247157929173142,
      "grad_norm": 10.627928733825684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3299752473831177,
      "logits/rejected": -0.36328205466270447,
      "logps/chosen": -578.4943237304688,
      "logps/rejected": -557.9207153320312,
      "loss": 0.4219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.853265643119812,
      "rewards/margins": 1.2523107528686523,
      "rewards/rejected": -2.105576276779175,
      "step": 12416
    },
    {
      "epoch": 1.624846650854666,
      "grad_norm": 7.041366100311279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03373870626091957,
      "logits/rejected": -0.21380266547203064,
      "logps/chosen": -368.2369079589844,
      "logps/rejected": -387.18658447265625,
      "loss": 0.3671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6764681339263916,
      "rewards/margins": 1.225738763809204,
      "rewards/rejected": -1.9022068977355957,
      "step": 12417
    },
    {
      "epoch": 1.6249775087920177,
      "grad_norm": 12.66111946105957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2658856511116028,
      "logits/rejected": -0.12105710804462433,
      "logps/chosen": -396.883056640625,
      "logps/rejected": -569.360595703125,
      "loss": 0.7268,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2125002145767212,
      "rewards/margins": 0.07498474419116974,
      "rewards/rejected": -1.2874850034713745,
      "step": 12418
    },
    {
      "epoch": 1.6251083667293695,
      "grad_norm": 10.09430980682373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1893535554409027,
      "logits/rejected": -0.3153269290924072,
      "logps/chosen": -424.9225769042969,
      "logps/rejected": -394.8398132324219,
      "loss": 0.6501,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4039369821548462,
      "rewards/margins": 0.9098407030105591,
      "rewards/rejected": -1.3137776851654053,
      "step": 12419
    },
    {
      "epoch": 1.6252392246667213,
      "grad_norm": 11.478118896484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08513942360877991,
      "logits/rejected": 0.1828252673149109,
      "logps/chosen": -445.65704345703125,
      "logps/rejected": -409.3680419921875,
      "loss": 0.6318,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7808338403701782,
      "rewards/margins": 0.5066263675689697,
      "rewards/rejected": -1.2874600887298584,
      "step": 12420
    },
    {
      "epoch": 1.6253700826040731,
      "grad_norm": 6.746714115142822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26677995920181274,
      "logits/rejected": -0.4366055727005005,
      "logps/chosen": -437.62493896484375,
      "logps/rejected": -462.3829040527344,
      "loss": 0.201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6995972394943237,
      "rewards/margins": 2.066053867340088,
      "rewards/rejected": -2.765650987625122,
      "step": 12421
    },
    {
      "epoch": 1.6255009405414247,
      "grad_norm": 11.433280944824219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.046955905854701996,
      "logits/rejected": 0.013139031827449799,
      "logps/chosen": -685.3547973632812,
      "logps/rejected": -574.8726806640625,
      "loss": 0.6802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16118070483207703,
      "rewards/margins": 0.5323032140731812,
      "rewards/rejected": -0.6934839487075806,
      "step": 12422
    },
    {
      "epoch": 1.6256317984787765,
      "grad_norm": 6.10338830947876,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16806800663471222,
      "logits/rejected": 0.0825701504945755,
      "logps/chosen": -473.9996643066406,
      "logps/rejected": -353.2284240722656,
      "loss": 0.306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16871795058250427,
      "rewards/margins": 1.9617078304290771,
      "rewards/rejected": -2.1304256916046143,
      "step": 12423
    },
    {
      "epoch": 1.6257626564161283,
      "grad_norm": 9.532927513122559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08581413328647614,
      "logits/rejected": 0.041591934859752655,
      "logps/chosen": -518.7889404296875,
      "logps/rejected": -513.875732421875,
      "loss": 0.4831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.590715765953064,
      "rewards/margins": 0.8328790068626404,
      "rewards/rejected": -1.4235947132110596,
      "step": 12424
    },
    {
      "epoch": 1.6258935143534798,
      "grad_norm": 7.630619525909424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26997441053390503,
      "logits/rejected": -0.11809355020523071,
      "logps/chosen": -348.56549072265625,
      "logps/rejected": -321.5174560546875,
      "loss": 0.3964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1517183780670166,
      "rewards/margins": 1.419985055923462,
      "rewards/rejected": -2.5717031955718994,
      "step": 12425
    },
    {
      "epoch": 1.6260243722908316,
      "grad_norm": 7.774653434753418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2586536407470703,
      "logits/rejected": -0.5074965953826904,
      "logps/chosen": -498.3907775878906,
      "logps/rejected": -438.5576477050781,
      "loss": 0.3126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.525122344493866,
      "rewards/margins": 1.436018705368042,
      "rewards/rejected": -1.9611409902572632,
      "step": 12426
    },
    {
      "epoch": 1.6261552302281834,
      "grad_norm": 8.29762077331543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22050563991069794,
      "logits/rejected": -0.23809272050857544,
      "logps/chosen": -421.4319152832031,
      "logps/rejected": -523.915771484375,
      "loss": 0.3731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1601192951202393,
      "rewards/margins": 1.593611478805542,
      "rewards/rejected": -2.7537307739257812,
      "step": 12427
    },
    {
      "epoch": 1.6262860881655352,
      "grad_norm": 7.172950267791748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2763422131538391,
      "logits/rejected": -0.23507273197174072,
      "logps/chosen": -502.1483154296875,
      "logps/rejected": -529.231689453125,
      "loss": 0.3454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4070236384868622,
      "rewards/margins": 2.1346585750579834,
      "rewards/rejected": -2.541682004928589,
      "step": 12428
    },
    {
      "epoch": 1.626416946102887,
      "grad_norm": 7.363292694091797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23034194111824036,
      "logits/rejected": -0.5377739667892456,
      "logps/chosen": -537.192138671875,
      "logps/rejected": -369.1673889160156,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39194583892822266,
      "rewards/margins": 1.5837279558181763,
      "rewards/rejected": -1.9756739139556885,
      "step": 12429
    },
    {
      "epoch": 1.6265478040402388,
      "grad_norm": 4.623499393463135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23932406306266785,
      "logits/rejected": -0.32927656173706055,
      "logps/chosen": -480.70440673828125,
      "logps/rejected": -599.733642578125,
      "loss": 0.2314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6019416451454163,
      "rewards/margins": 2.49931001663208,
      "rewards/rejected": -3.1012516021728516,
      "step": 12430
    },
    {
      "epoch": 1.6266786619775906,
      "grad_norm": 6.697850227355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18214412033557892,
      "logits/rejected": -0.23187437653541565,
      "logps/chosen": -443.93609619140625,
      "logps/rejected": -489.91473388671875,
      "loss": 0.3446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46461397409439087,
      "rewards/margins": 1.2589538097381592,
      "rewards/rejected": -1.7235678434371948,
      "step": 12431
    },
    {
      "epoch": 1.6268095199149424,
      "grad_norm": 5.382650375366211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1615702509880066,
      "logits/rejected": -0.16369831562042236,
      "logps/chosen": -453.2022399902344,
      "logps/rejected": -605.3656616210938,
      "loss": 0.278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45725196599960327,
      "rewards/margins": 1.6649843454360962,
      "rewards/rejected": -2.1222362518310547,
      "step": 12432
    },
    {
      "epoch": 1.6269403778522942,
      "grad_norm": 10.07679271697998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26360711455345154,
      "logits/rejected": -0.3735480308532715,
      "logps/chosen": -571.3447265625,
      "logps/rejected": -480.4834899902344,
      "loss": 0.5414,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0610655546188354,
      "rewards/margins": 0.9086993336677551,
      "rewards/rejected": -1.9697648286819458,
      "step": 12433
    },
    {
      "epoch": 1.627071235789646,
      "grad_norm": 8.979549407958984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4702017605304718,
      "logits/rejected": -0.43145841360092163,
      "logps/chosen": -434.52264404296875,
      "logps/rejected": -406.3601379394531,
      "loss": 0.5853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6108591556549072,
      "rewards/margins": 0.490518182516098,
      "rewards/rejected": -2.101377487182617,
      "step": 12434
    },
    {
      "epoch": 1.6272020937269978,
      "grad_norm": 2.6980748176574707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00041949562728405,
      "logits/rejected": -0.1509476602077484,
      "logps/chosen": -367.93670654296875,
      "logps/rejected": -368.9045715332031,
      "loss": 0.174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06385537981987,
      "rewards/margins": 2.374011754989624,
      "rewards/rejected": -2.4378674030303955,
      "step": 12435
    },
    {
      "epoch": 1.6273329516643495,
      "grad_norm": 7.709697246551514,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011838452890515327,
      "logits/rejected": -0.37644287943840027,
      "logps/chosen": -413.4668273925781,
      "logps/rejected": -500.0867614746094,
      "loss": 0.3602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2676143944263458,
      "rewards/margins": 1.4177042245864868,
      "rewards/rejected": -1.6853185892105103,
      "step": 12436
    },
    {
      "epoch": 1.6274638096017011,
      "grad_norm": 5.565796852111816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09777287393808365,
      "logits/rejected": -0.24579808115959167,
      "logps/chosen": -355.63262939453125,
      "logps/rejected": -346.1412048339844,
      "loss": 0.2426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.044501855969429016,
      "rewards/margins": 2.030756950378418,
      "rewards/rejected": -2.07525897026062,
      "step": 12437
    },
    {
      "epoch": 1.627594667539053,
      "grad_norm": 15.483047485351562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2897280156612396,
      "logits/rejected": -0.26725274324417114,
      "logps/chosen": -456.67236328125,
      "logps/rejected": -646.3384399414062,
      "loss": 0.9198,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.850422978401184,
      "rewards/margins": -0.12024781107902527,
      "rewards/rejected": -1.7301751375198364,
      "step": 12438
    },
    {
      "epoch": 1.6277255254764047,
      "grad_norm": 11.286120414733887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16908323764801025,
      "logits/rejected": -0.32820966839790344,
      "logps/chosen": -427.8597412109375,
      "logps/rejected": -481.70965576171875,
      "loss": 0.5805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2251410484313965,
      "rewards/margins": 0.5616819858551025,
      "rewards/rejected": -1.7868231534957886,
      "step": 12439
    },
    {
      "epoch": 1.6278563834137565,
      "grad_norm": 7.9952545166015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35050827264785767,
      "logits/rejected": -0.39957770705223083,
      "logps/chosen": -755.1724853515625,
      "logps/rejected": -677.573974609375,
      "loss": 0.4065,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5270261764526367,
      "rewards/margins": 1.2428869009017944,
      "rewards/rejected": -1.7699131965637207,
      "step": 12440
    },
    {
      "epoch": 1.627987241351108,
      "grad_norm": 9.765756607055664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15226414799690247,
      "logits/rejected": -0.24122335016727448,
      "logps/chosen": -331.94537353515625,
      "logps/rejected": -303.0987548828125,
      "loss": 0.6338,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8237228393554688,
      "rewards/margins": 1.1896891593933105,
      "rewards/rejected": -2.0134119987487793,
      "step": 12441
    },
    {
      "epoch": 1.6281180992884599,
      "grad_norm": 11.562158584594727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16022513806819916,
      "logits/rejected": -0.43042171001434326,
      "logps/chosen": -444.9185791015625,
      "logps/rejected": -462.4808349609375,
      "loss": 0.6661,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5759590268135071,
      "rewards/margins": 0.7542772889137268,
      "rewards/rejected": -1.3302363157272339,
      "step": 12442
    },
    {
      "epoch": 1.6282489572258116,
      "grad_norm": 8.803667068481445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4329568147659302,
      "logits/rejected": -0.06053999066352844,
      "logps/chosen": -410.761962890625,
      "logps/rejected": -600.197998046875,
      "loss": 0.5191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3706339597702026,
      "rewards/margins": 0.9554723501205444,
      "rewards/rejected": -2.326106071472168,
      "step": 12443
    },
    {
      "epoch": 1.6283798151631634,
      "grad_norm": 9.617018699645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39059337973594666,
      "logits/rejected": -0.49260473251342773,
      "logps/chosen": -460.2629089355469,
      "logps/rejected": -351.6541748046875,
      "loss": 0.4637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.499231219291687,
      "rewards/margins": 1.6343063116073608,
      "rewards/rejected": -3.133537530899048,
      "step": 12444
    },
    {
      "epoch": 1.6285106731005152,
      "grad_norm": 5.492281436920166,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035693541169166565,
      "logits/rejected": 0.12054899334907532,
      "logps/chosen": -415.7702331542969,
      "logps/rejected": -458.7421875,
      "loss": 0.2193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5530713796615601,
      "rewards/margins": 1.9044029712677002,
      "rewards/rejected": -2.45747447013855,
      "step": 12445
    },
    {
      "epoch": 1.628641531037867,
      "grad_norm": 4.207820415496826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17139554023742676,
      "logits/rejected": -0.43616989254951477,
      "logps/chosen": -412.11456298828125,
      "logps/rejected": -361.7352600097656,
      "loss": 0.2062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7666530609130859,
      "rewards/margins": 2.3154850006103516,
      "rewards/rejected": -3.0821378231048584,
      "step": 12446
    },
    {
      "epoch": 1.6287723889752188,
      "grad_norm": 6.973130702972412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27092763781547546,
      "logits/rejected": -0.3665119707584381,
      "logps/chosen": -395.606201171875,
      "logps/rejected": -576.6676025390625,
      "loss": 0.3121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0291870832443237,
      "rewards/margins": 2.130276679992676,
      "rewards/rejected": -3.15946364402771,
      "step": 12447
    },
    {
      "epoch": 1.6289032469125706,
      "grad_norm": 4.040392875671387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.344778835773468,
      "logits/rejected": -0.15242129564285278,
      "logps/chosen": -588.7211303710938,
      "logps/rejected": -486.0083312988281,
      "loss": 0.2393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26853635907173157,
      "rewards/margins": 1.7221018075942993,
      "rewards/rejected": -1.990638256072998,
      "step": 12448
    },
    {
      "epoch": 1.6290341048499224,
      "grad_norm": 6.950314521789551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35182297229766846,
      "logits/rejected": -0.5254700779914856,
      "logps/chosen": -309.94317626953125,
      "logps/rejected": -367.0752868652344,
      "loss": 0.4303,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3505146503448486,
      "rewards/margins": 1.6023366451263428,
      "rewards/rejected": -2.9528512954711914,
      "step": 12449
    },
    {
      "epoch": 1.6291649627872742,
      "grad_norm": 6.534112930297852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17377354204654694,
      "logits/rejected": -0.19926190376281738,
      "logps/chosen": -304.440673828125,
      "logps/rejected": -430.23455810546875,
      "loss": 0.3658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2361290454864502,
      "rewards/margins": 1.6735618114471436,
      "rewards/rejected": -2.9096908569335938,
      "step": 12450
    },
    {
      "epoch": 1.629295820724626,
      "grad_norm": 8.5316801071167,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14558638632297516,
      "logits/rejected": -0.07541196048259735,
      "logps/chosen": -477.7415466308594,
      "logps/rejected": -500.8035888671875,
      "loss": 0.4474,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6309089660644531,
      "rewards/margins": 1.0635936260223389,
      "rewards/rejected": -1.694502592086792,
      "step": 12451
    },
    {
      "epoch": 1.6294266786619775,
      "grad_norm": 8.110817909240723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5407257080078125,
      "logits/rejected": -0.46431997418403625,
      "logps/chosen": -261.4229736328125,
      "logps/rejected": -330.7624816894531,
      "loss": 0.6804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6687619090080261,
      "rewards/margins": 0.8856066465377808,
      "rewards/rejected": -1.5543683767318726,
      "step": 12452
    },
    {
      "epoch": 1.6295575365993293,
      "grad_norm": 7.078336715698242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18248200416564941,
      "logits/rejected": -0.5707271099090576,
      "logps/chosen": -478.02044677734375,
      "logps/rejected": -397.87939453125,
      "loss": 0.379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3155975341796875,
      "rewards/margins": 1.792703628540039,
      "rewards/rejected": -2.1083011627197266,
      "step": 12453
    },
    {
      "epoch": 1.6296883945366811,
      "grad_norm": 15.015287399291992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09488622099161148,
      "logits/rejected": -0.1939454972743988,
      "logps/chosen": -459.31396484375,
      "logps/rejected": -360.31280517578125,
      "loss": 0.8278,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9623661041259766,
      "rewards/margins": 0.23995374143123627,
      "rewards/rejected": -2.202320098876953,
      "step": 12454
    },
    {
      "epoch": 1.629819252474033,
      "grad_norm": 9.177986145019531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07716719806194305,
      "logits/rejected": -0.2466452419757843,
      "logps/chosen": -278.1257019042969,
      "logps/rejected": -226.339111328125,
      "loss": 0.589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3551278114318848,
      "rewards/margins": 0.48319581151008606,
      "rewards/rejected": -1.8383235931396484,
      "step": 12455
    },
    {
      "epoch": 1.6299501104113845,
      "grad_norm": 7.6548943519592285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19220614433288574,
      "logits/rejected": -0.4830797016620636,
      "logps/chosen": -438.2454528808594,
      "logps/rejected": -421.5653991699219,
      "loss": 0.3476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5468966364860535,
      "rewards/margins": 1.5332719087600708,
      "rewards/rejected": -2.0801684856414795,
      "step": 12456
    },
    {
      "epoch": 1.6300809683487363,
      "grad_norm": 7.386645317077637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2704201340675354,
      "logits/rejected": -0.2752113342285156,
      "logps/chosen": -617.8397827148438,
      "logps/rejected": -507.41082763671875,
      "loss": 0.456,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26753923296928406,
      "rewards/margins": 0.9842281341552734,
      "rewards/rejected": -1.2517673969268799,
      "step": 12457
    },
    {
      "epoch": 1.630211826286088,
      "grad_norm": 10.762445449829102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30553141236305237,
      "logits/rejected": -0.3760525584220886,
      "logps/chosen": -484.7995910644531,
      "logps/rejected": -511.7100524902344,
      "loss": 0.4709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2981288433074951,
      "rewards/margins": 1.1905457973480225,
      "rewards/rejected": -1.4886746406555176,
      "step": 12458
    },
    {
      "epoch": 1.6303426842234399,
      "grad_norm": 7.34421968460083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2942860722541809,
      "logits/rejected": -0.3620224595069885,
      "logps/chosen": -658.2957763671875,
      "logps/rejected": -549.885009765625,
      "loss": 0.3552,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6226682662963867,
      "rewards/margins": 1.9678313732147217,
      "rewards/rejected": -2.5904996395111084,
      "step": 12459
    },
    {
      "epoch": 1.6304735421607917,
      "grad_norm": 9.020668029785156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16427716612815857,
      "logits/rejected": -0.19554264843463898,
      "logps/chosen": -519.6651000976562,
      "logps/rejected": -550.410888671875,
      "loss": 0.521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6341153979301453,
      "rewards/margins": 1.274484634399414,
      "rewards/rejected": -1.9085999727249146,
      "step": 12460
    },
    {
      "epoch": 1.6306044000981434,
      "grad_norm": 10.756166458129883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3210785984992981,
      "logits/rejected": -0.2697640657424927,
      "logps/chosen": -393.6759338378906,
      "logps/rejected": -392.4488830566406,
      "loss": 0.6652,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.689327597618103,
      "rewards/margins": 0.8442925810813904,
      "rewards/rejected": -1.5336201190948486,
      "step": 12461
    },
    {
      "epoch": 1.6307352580354952,
      "grad_norm": 15.136686325073242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2670748829841614,
      "logits/rejected": -0.19818605482578278,
      "logps/chosen": -501.99615478515625,
      "logps/rejected": -404.00897216796875,
      "loss": 0.7842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2247087955474854,
      "rewards/margins": 0.7047535181045532,
      "rewards/rejected": -1.9294623136520386,
      "step": 12462
    },
    {
      "epoch": 1.630866115972847,
      "grad_norm": 7.570003509521484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03348232060670853,
      "logits/rejected": -0.05951568856835365,
      "logps/chosen": -416.7853088378906,
      "logps/rejected": -388.2143249511719,
      "loss": 0.4551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3405466079711914,
      "rewards/margins": 0.6517708897590637,
      "rewards/rejected": -1.9923175573349,
      "step": 12463
    },
    {
      "epoch": 1.6309969739101988,
      "grad_norm": 8.88654899597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043385472148656845,
      "logits/rejected": -0.06944048404693604,
      "logps/chosen": -383.8716125488281,
      "logps/rejected": -355.46221923828125,
      "loss": 0.4681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0225540399551392,
      "rewards/margins": 1.5759773254394531,
      "rewards/rejected": -2.598531484603882,
      "step": 12464
    },
    {
      "epoch": 1.6311278318475506,
      "grad_norm": 12.362483024597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5323945879936218,
      "logits/rejected": -0.6587773561477661,
      "logps/chosen": -629.11328125,
      "logps/rejected": -347.09637451171875,
      "loss": 0.8589,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2835404872894287,
      "rewards/margins": 0.3208519220352173,
      "rewards/rejected": -1.6043925285339355,
      "step": 12465
    },
    {
      "epoch": 1.6312586897849024,
      "grad_norm": 10.944222450256348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29826995730400085,
      "logits/rejected": -0.25637611746788025,
      "logps/chosen": -290.13397216796875,
      "logps/rejected": -319.7298583984375,
      "loss": 0.9423,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2217683792114258,
      "rewards/margins": -0.18709011375904083,
      "rewards/rejected": -1.0346782207489014,
      "step": 12466
    },
    {
      "epoch": 1.631389547722254,
      "grad_norm": 7.947110652923584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0876370519399643,
      "logits/rejected": -0.11239130795001984,
      "logps/chosen": -366.4352111816406,
      "logps/rejected": -466.5014953613281,
      "loss": 0.3527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9989637136459351,
      "rewards/margins": 1.7663754224777222,
      "rewards/rejected": -2.7653393745422363,
      "step": 12467
    },
    {
      "epoch": 1.6315204056596058,
      "grad_norm": 16.882015228271484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12594617903232574,
      "logits/rejected": -0.1621098816394806,
      "logps/chosen": -303.1273193359375,
      "logps/rejected": -424.8634948730469,
      "loss": 1.1951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8252354860305786,
      "rewards/margins": 0.35011357069015503,
      "rewards/rejected": -2.175348997116089,
      "step": 12468
    },
    {
      "epoch": 1.6316512635969576,
      "grad_norm": 8.682100296020508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41506731510162354,
      "logits/rejected": -0.16190874576568604,
      "logps/chosen": -474.90631103515625,
      "logps/rejected": -476.6028747558594,
      "loss": 0.3433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.819946825504303,
      "rewards/margins": 1.6487728357315063,
      "rewards/rejected": -2.468719720840454,
      "step": 12469
    },
    {
      "epoch": 1.6317821215343093,
      "grad_norm": 9.631481170654297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1251477152109146,
      "logits/rejected": -0.15991541743278503,
      "logps/chosen": -412.34503173828125,
      "logps/rejected": -475.8607482910156,
      "loss": 0.6478,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.061266303062439,
      "rewards/margins": 1.126570701599121,
      "rewards/rejected": -2.1878371238708496,
      "step": 12470
    },
    {
      "epoch": 1.631912979471661,
      "grad_norm": 11.01436996459961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3691816031932831,
      "logits/rejected": -0.18346601724624634,
      "logps/chosen": -425.0616455078125,
      "logps/rejected": -509.65325927734375,
      "loss": 0.6865,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3414201736450195,
      "rewards/margins": 0.7326849699020386,
      "rewards/rejected": -2.0741050243377686,
      "step": 12471
    },
    {
      "epoch": 1.6320438374090127,
      "grad_norm": 9.78581714630127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3959757387638092,
      "logits/rejected": -0.4127572476863861,
      "logps/chosen": -419.3611755371094,
      "logps/rejected": -402.6558837890625,
      "loss": 0.5645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.42915087938308716,
      "rewards/margins": 0.5522232055664062,
      "rewards/rejected": -0.9813742637634277,
      "step": 12472
    },
    {
      "epoch": 1.6321746953463645,
      "grad_norm": 5.046459674835205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1929282546043396,
      "logits/rejected": -0.11706721782684326,
      "logps/chosen": -461.51239013671875,
      "logps/rejected": -396.38519287109375,
      "loss": 0.2325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26165106892585754,
      "rewards/margins": 1.7649646997451782,
      "rewards/rejected": -2.026615858078003,
      "step": 12473
    },
    {
      "epoch": 1.6323055532837163,
      "grad_norm": 7.50883150100708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11482901126146317,
      "logits/rejected": -0.27321657538414,
      "logps/chosen": -388.14349365234375,
      "logps/rejected": -529.9661865234375,
      "loss": 0.2263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0098800659179688,
      "rewards/margins": 2.3978731632232666,
      "rewards/rejected": -3.4077534675598145,
      "step": 12474
    },
    {
      "epoch": 1.632436411221068,
      "grad_norm": 10.500435829162598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1252714991569519,
      "logits/rejected": -0.32041317224502563,
      "logps/chosen": -431.6004638671875,
      "logps/rejected": -358.9919738769531,
      "loss": 0.4926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.374345302581787,
      "rewards/margins": 0.8230355978012085,
      "rewards/rejected": -2.197381019592285,
      "step": 12475
    },
    {
      "epoch": 1.6325672691584199,
      "grad_norm": 10.262499809265137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11904682964086533,
      "logits/rejected": -0.08511052280664444,
      "logps/chosen": -482.76141357421875,
      "logps/rejected": -413.542724609375,
      "loss": 0.5892,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8551042675971985,
      "rewards/margins": 1.041222333908081,
      "rewards/rejected": -1.8963266611099243,
      "step": 12476
    },
    {
      "epoch": 1.6326981270957717,
      "grad_norm": 4.7259979248046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48325860500335693,
      "logits/rejected": -0.4882141053676605,
      "logps/chosen": -230.6764373779297,
      "logps/rejected": -303.6831970214844,
      "loss": 0.2681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.031045913696289,
      "rewards/margins": 1.575530767440796,
      "rewards/rejected": -2.606576442718506,
      "step": 12477
    },
    {
      "epoch": 1.6328289850331235,
      "grad_norm": 7.679007530212402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3337745666503906,
      "logits/rejected": -0.3322899341583252,
      "logps/chosen": -481.09710693359375,
      "logps/rejected": -453.1502685546875,
      "loss": 0.3002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14417609572410583,
      "rewards/margins": 1.8796672821044922,
      "rewards/rejected": -1.7354912757873535,
      "step": 12478
    },
    {
      "epoch": 1.6329598429704753,
      "grad_norm": 12.519024848937988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3755226731300354,
      "logits/rejected": -0.7039150595664978,
      "logps/chosen": -486.8052062988281,
      "logps/rejected": -417.55657958984375,
      "loss": 0.5817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.366511583328247,
      "rewards/margins": 1.0119372606277466,
      "rewards/rejected": -2.378448724746704,
      "step": 12479
    },
    {
      "epoch": 1.633090700907827,
      "grad_norm": 8.832014083862305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6719664335250854,
      "logits/rejected": -0.5014915466308594,
      "logps/chosen": -288.6783752441406,
      "logps/rejected": -449.1974792480469,
      "loss": 0.5,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.733585238456726,
      "rewards/margins": 1.4957237243652344,
      "rewards/rejected": -3.229308843612671,
      "step": 12480
    },
    {
      "epoch": 1.6332215588451788,
      "grad_norm": 5.963607311248779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4693950116634369,
      "logits/rejected": -0.6027005910873413,
      "logps/chosen": -469.39373779296875,
      "logps/rejected": -590.7979125976562,
      "loss": 0.285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4310392141342163,
      "rewards/margins": 2.065030574798584,
      "rewards/rejected": -3.49606990814209,
      "step": 12481
    },
    {
      "epoch": 1.6333524167825306,
      "grad_norm": 9.58398723602295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5964624881744385,
      "logits/rejected": -0.5748050212860107,
      "logps/chosen": -432.63043212890625,
      "logps/rejected": -535.1993408203125,
      "loss": 0.5114,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.141906499862671,
      "rewards/margins": 0.8567618131637573,
      "rewards/rejected": -2.9986681938171387,
      "step": 12482
    },
    {
      "epoch": 1.6334832747198822,
      "grad_norm": 13.665574073791504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08246955275535583,
      "logits/rejected": 0.05560518056154251,
      "logps/chosen": -262.3341064453125,
      "logps/rejected": -324.7662658691406,
      "loss": 0.6175,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0976418256759644,
      "rewards/margins": 0.915635883808136,
      "rewards/rejected": -2.013277530670166,
      "step": 12483
    },
    {
      "epoch": 1.633614132657234,
      "grad_norm": 12.202723503112793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.654649019241333,
      "logits/rejected": -0.5794941186904907,
      "logps/chosen": -381.100830078125,
      "logps/rejected": -503.0677185058594,
      "loss": 0.8555,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4500830173492432,
      "rewards/margins": 0.17685873806476593,
      "rewards/rejected": -1.6269418001174927,
      "step": 12484
    },
    {
      "epoch": 1.6337449905945858,
      "grad_norm": 7.980860710144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2990127205848694,
      "logits/rejected": -0.5028379559516907,
      "logps/chosen": -445.7488098144531,
      "logps/rejected": -510.3851318359375,
      "loss": 0.4069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5358657836914062,
      "rewards/margins": 0.9993146657943726,
      "rewards/rejected": -2.5351805686950684,
      "step": 12485
    },
    {
      "epoch": 1.6338758485319376,
      "grad_norm": 5.822409629821777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46224117279052734,
      "logits/rejected": -0.1710200309753418,
      "logps/chosen": -366.89703369140625,
      "logps/rejected": -382.54400634765625,
      "loss": 0.336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4934263229370117,
      "rewards/margins": 1.408163070678711,
      "rewards/rejected": -2.9015893936157227,
      "step": 12486
    },
    {
      "epoch": 1.6340067064692891,
      "grad_norm": 7.493406772613525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06674416363239288,
      "logits/rejected": -0.28212541341781616,
      "logps/chosen": -683.93701171875,
      "logps/rejected": -484.3408203125,
      "loss": 0.3083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0487570762634277,
      "rewards/margins": 2.0404696464538574,
      "rewards/rejected": -3.089226484298706,
      "step": 12487
    },
    {
      "epoch": 1.634137564406641,
      "grad_norm": 12.916991233825684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12751023471355438,
      "logits/rejected": -0.39478373527526855,
      "logps/chosen": -429.0955505371094,
      "logps/rejected": -454.2839660644531,
      "loss": 0.7832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9533312320709229,
      "rewards/margins": 0.31048184633255005,
      "rewards/rejected": -2.2638132572174072,
      "step": 12488
    },
    {
      "epoch": 1.6342684223439927,
      "grad_norm": 2.6165430545806885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07079049199819565,
      "logits/rejected": -0.09077763557434082,
      "logps/chosen": -375.1894836425781,
      "logps/rejected": -510.3900146484375,
      "loss": 0.0697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5960816144943237,
      "rewards/margins": 3.7391722202301025,
      "rewards/rejected": -4.335254192352295,
      "step": 12489
    },
    {
      "epoch": 1.6343992802813445,
      "grad_norm": 7.742020606994629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24505196511745453,
      "logits/rejected": -0.3163266181945801,
      "logps/chosen": -700.36962890625,
      "logps/rejected": -807.8947143554688,
      "loss": 0.3028,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4285343289375305,
      "rewards/margins": 2.5095014572143555,
      "rewards/rejected": -2.938035488128662,
      "step": 12490
    },
    {
      "epoch": 1.6345301382186963,
      "grad_norm": 9.698989868164062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6820653080940247,
      "logits/rejected": -0.6706278920173645,
      "logps/chosen": -496.9839782714844,
      "logps/rejected": -495.7722473144531,
      "loss": 0.5004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.961620807647705,
      "rewards/margins": 0.9968078136444092,
      "rewards/rejected": -2.9584286212921143,
      "step": 12491
    },
    {
      "epoch": 1.634660996156048,
      "grad_norm": 6.137383460998535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07815468311309814,
      "logits/rejected": -0.02338935248553753,
      "logps/chosen": -361.9311828613281,
      "logps/rejected": -469.0060729980469,
      "loss": 0.36,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1787161827087402,
      "rewards/margins": 1.260944128036499,
      "rewards/rejected": -2.4396603107452393,
      "step": 12492
    },
    {
      "epoch": 1.6347918540933999,
      "grad_norm": 7.608426570892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11400707066059113,
      "logits/rejected": -0.0824592113494873,
      "logps/chosen": -501.5740966796875,
      "logps/rejected": -530.467041015625,
      "loss": 0.2695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.16280996799469,
      "rewards/margins": 1.5720487833023071,
      "rewards/rejected": -2.734858751296997,
      "step": 12493
    },
    {
      "epoch": 1.6349227120307517,
      "grad_norm": 5.641037464141846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16429735720157623,
      "logits/rejected": -0.22399061918258667,
      "logps/chosen": -350.85546875,
      "logps/rejected": -470.05224609375,
      "loss": 0.3197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0505791902542114,
      "rewards/margins": 1.3407081365585327,
      "rewards/rejected": -2.391287326812744,
      "step": 12494
    },
    {
      "epoch": 1.6350535699681035,
      "grad_norm": 11.28878402709961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22895702719688416,
      "logits/rejected": -0.3098958134651184,
      "logps/chosen": -597.2128295898438,
      "logps/rejected": -426.93096923828125,
      "loss": 0.7163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0946686267852783,
      "rewards/margins": 0.9383901953697205,
      "rewards/rejected": -3.0330588817596436,
      "step": 12495
    },
    {
      "epoch": 1.6351844279054553,
      "grad_norm": 4.152055740356445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17214927077293396,
      "logits/rejected": 0.09115445613861084,
      "logps/chosen": -554.27783203125,
      "logps/rejected": -414.9480285644531,
      "loss": 0.1573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3115224838256836,
      "rewards/margins": 2.036257743835449,
      "rewards/rejected": -2.347780227661133,
      "step": 12496
    },
    {
      "epoch": 1.635315285842807,
      "grad_norm": 8.7229642868042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46843624114990234,
      "logits/rejected": -0.45958319306373596,
      "logps/chosen": -475.3913879394531,
      "logps/rejected": -517.8951416015625,
      "loss": 0.4553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1755911111831665,
      "rewards/margins": 1.4980368614196777,
      "rewards/rejected": -2.673628091812134,
      "step": 12497
    },
    {
      "epoch": 1.6354461437801586,
      "grad_norm": 13.512425422668457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22788845002651215,
      "logits/rejected": -0.33860230445861816,
      "logps/chosen": -464.6510314941406,
      "logps/rejected": -506.9246826171875,
      "loss": 0.6252,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8126099109649658,
      "rewards/margins": 1.313850998878479,
      "rewards/rejected": -3.1264607906341553,
      "step": 12498
    },
    {
      "epoch": 1.6355770017175104,
      "grad_norm": 9.430983543395996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0263067577034235,
      "logits/rejected": -0.26704418659210205,
      "logps/chosen": -276.2372741699219,
      "logps/rejected": -360.64697265625,
      "loss": 0.605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3953322172164917,
      "rewards/margins": 0.832260251045227,
      "rewards/rejected": -2.2275924682617188,
      "step": 12499
    },
    {
      "epoch": 1.6357078596548622,
      "grad_norm": 4.740096569061279,
      "learning_rate": 5e-06,
      "logits/chosen": -0.398912250995636,
      "logits/rejected": -0.29913195967674255,
      "logps/chosen": -443.9584655761719,
      "logps/rejected": -586.7294311523438,
      "loss": 0.2027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7350057363510132,
      "rewards/margins": 2.174785852432251,
      "rewards/rejected": -3.9097914695739746,
      "step": 12500
    },
    {
      "epoch": 1.635838717592214,
      "grad_norm": 7.143266201019287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23909401893615723,
      "logits/rejected": -0.4806053936481476,
      "logps/chosen": -481.70428466796875,
      "logps/rejected": -559.322021484375,
      "loss": 0.2474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7710317373275757,
      "rewards/margins": 2.3164782524108887,
      "rewards/rejected": -4.087510108947754,
      "step": 12501
    },
    {
      "epoch": 1.6359695755295656,
      "grad_norm": 11.318375587463379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26418086886405945,
      "logits/rejected": -0.3033819794654846,
      "logps/chosen": -374.2198791503906,
      "logps/rejected": -374.062744140625,
      "loss": 0.7144,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1703799962997437,
      "rewards/margins": 0.4716230034828186,
      "rewards/rejected": -1.642003059387207,
      "step": 12502
    },
    {
      "epoch": 1.6361004334669174,
      "grad_norm": 7.677474498748779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.21226775646209717,
      "logits/rejected": -0.1431012600660324,
      "logps/chosen": -408.3069763183594,
      "logps/rejected": -326.5879821777344,
      "loss": 0.3363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3031954765319824,
      "rewards/margins": 1.4156159162521362,
      "rewards/rejected": -2.718811273574829,
      "step": 12503
    },
    {
      "epoch": 1.6362312914042692,
      "grad_norm": 12.713096618652344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4638623297214508,
      "logits/rejected": -0.5315832495689392,
      "logps/chosen": -542.281005859375,
      "logps/rejected": -539.9458618164062,
      "loss": 0.4584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9994255304336548,
      "rewards/margins": 1.0870795249938965,
      "rewards/rejected": -2.0865049362182617,
      "step": 12504
    },
    {
      "epoch": 1.636362149341621,
      "grad_norm": 7.828797817230225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19653506577014923,
      "logits/rejected": -0.3298097550868988,
      "logps/chosen": -586.0831909179688,
      "logps/rejected": -387.526123046875,
      "loss": 0.342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7034049034118652,
      "rewards/margins": 1.555604100227356,
      "rewards/rejected": -2.2590088844299316,
      "step": 12505
    },
    {
      "epoch": 1.6364930072789727,
      "grad_norm": 9.269880294799805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21735602617263794,
      "logits/rejected": -0.3910045623779297,
      "logps/chosen": -283.05157470703125,
      "logps/rejected": -346.2397155761719,
      "loss": 0.4809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.597373127937317,
      "rewards/margins": 1.505318284034729,
      "rewards/rejected": -3.102691650390625,
      "step": 12506
    },
    {
      "epoch": 1.6366238652163245,
      "grad_norm": 21.45920181274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6939201354980469,
      "logits/rejected": -0.7723207473754883,
      "logps/chosen": -680.1168823242188,
      "logps/rejected": -666.1290893554688,
      "loss": 1.357,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.267571210861206,
      "rewards/margins": 0.1008726954460144,
      "rewards/rejected": -2.3684439659118652,
      "step": 12507
    },
    {
      "epoch": 1.6367547231536763,
      "grad_norm": 14.375804901123047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5490021705627441,
      "logits/rejected": -0.657728910446167,
      "logps/chosen": -446.52374267578125,
      "logps/rejected": -327.60894775390625,
      "loss": 0.5182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9432604312896729,
      "rewards/margins": 1.2368550300598145,
      "rewards/rejected": -3.1801154613494873,
      "step": 12508
    },
    {
      "epoch": 1.636885581091028,
      "grad_norm": 5.204736709594727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3576700985431671,
      "logits/rejected": -0.3330657184123993,
      "logps/chosen": -311.1988525390625,
      "logps/rejected": -354.1338806152344,
      "loss": 0.2264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1088758707046509,
      "rewards/margins": 1.8298163414001465,
      "rewards/rejected": -2.9386918544769287,
      "step": 12509
    },
    {
      "epoch": 1.63701643902838,
      "grad_norm": 10.116875648498535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34174656867980957,
      "logits/rejected": -0.29241010546684265,
      "logps/chosen": -459.2998352050781,
      "logps/rejected": -481.4971008300781,
      "loss": 0.6159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.347688913345337,
      "rewards/margins": 1.0143394470214844,
      "rewards/rejected": -2.3620283603668213,
      "step": 12510
    },
    {
      "epoch": 1.6371472969657317,
      "grad_norm": 8.34582805633545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4356827139854431,
      "logits/rejected": -0.27996864914894104,
      "logps/chosen": -495.13311767578125,
      "logps/rejected": -480.2574462890625,
      "loss": 0.3569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.16909122467041,
      "rewards/margins": 1.283227801322937,
      "rewards/rejected": -3.4523191452026367,
      "step": 12511
    },
    {
      "epoch": 1.6372781549030835,
      "grad_norm": 7.1198506355285645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2021462470293045,
      "logits/rejected": -0.24204696714878082,
      "logps/chosen": -640.66845703125,
      "logps/rejected": -443.06243896484375,
      "loss": 0.4075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3682224154472351,
      "rewards/margins": 1.1637077331542969,
      "rewards/rejected": -1.5319302082061768,
      "step": 12512
    },
    {
      "epoch": 1.637409012840435,
      "grad_norm": 10.60452651977539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4375159442424774,
      "logits/rejected": -0.47062408924102783,
      "logps/chosen": -474.9321594238281,
      "logps/rejected": -450.2601623535156,
      "loss": 0.4782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6483763456344604,
      "rewards/margins": 1.2635080814361572,
      "rewards/rejected": -2.9118845462799072,
      "step": 12513
    },
    {
      "epoch": 1.6375398707777868,
      "grad_norm": 4.99200963973999,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052108172327280045,
      "logits/rejected": -0.43854638934135437,
      "logps/chosen": -498.1534423828125,
      "logps/rejected": -340.3314514160156,
      "loss": 0.1912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06322747468948364,
      "rewards/margins": 2.1143617630004883,
      "rewards/rejected": -2.0511341094970703,
      "step": 12514
    },
    {
      "epoch": 1.6376707287151386,
      "grad_norm": 5.241668224334717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06000208854675293,
      "logits/rejected": -0.22402425110340118,
      "logps/chosen": -432.5991516113281,
      "logps/rejected": -334.5322570800781,
      "loss": 0.2619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6308414340019226,
      "rewards/margins": 1.7905410528182983,
      "rewards/rejected": -2.421382427215576,
      "step": 12515
    },
    {
      "epoch": 1.6378015866524904,
      "grad_norm": 8.411624908447266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2049112617969513,
      "logits/rejected": -0.5900819301605225,
      "logps/chosen": -486.5027770996094,
      "logps/rejected": -418.5906066894531,
      "loss": 0.4133,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2729992866516113,
      "rewards/margins": 1.3780853748321533,
      "rewards/rejected": -2.6510846614837646,
      "step": 12516
    },
    {
      "epoch": 1.637932444589842,
      "grad_norm": 9.252726554870605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.061764463782310486,
      "logits/rejected": -0.12264008820056915,
      "logps/chosen": -513.60400390625,
      "logps/rejected": -604.887939453125,
      "loss": 0.4647,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3496676683425903,
      "rewards/margins": 1.3838794231414795,
      "rewards/rejected": -2.7335474491119385,
      "step": 12517
    },
    {
      "epoch": 1.6380633025271938,
      "grad_norm": 8.511661529541016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6625121235847473,
      "logits/rejected": -0.6922593116760254,
      "logps/chosen": -360.89593505859375,
      "logps/rejected": -394.64495849609375,
      "loss": 0.4314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6965973377227783,
      "rewards/margins": 0.8001276254653931,
      "rewards/rejected": -2.496724843978882,
      "step": 12518
    },
    {
      "epoch": 1.6381941604645456,
      "grad_norm": 12.734297752380371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0725904256105423,
      "logits/rejected": -0.3494129180908203,
      "logps/chosen": -563.5865478515625,
      "logps/rejected": -451.4549865722656,
      "loss": 0.6994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.765625238418579,
      "rewards/margins": 1.3908109664916992,
      "rewards/rejected": -3.1564362049102783,
      "step": 12519
    },
    {
      "epoch": 1.6383250184018974,
      "grad_norm": 19.0603084564209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.505914568901062,
      "logits/rejected": -0.43534213304519653,
      "logps/chosen": -556.3018798828125,
      "logps/rejected": -490.09722900390625,
      "loss": 1.0114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.365152359008789,
      "rewards/margins": 0.26456645131111145,
      "rewards/rejected": -3.629718780517578,
      "step": 12520
    },
    {
      "epoch": 1.6384558763392492,
      "grad_norm": 12.382479667663574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5497244000434875,
      "logits/rejected": -0.39273470640182495,
      "logps/chosen": -495.02783203125,
      "logps/rejected": -433.0560607910156,
      "loss": 0.4943,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8320102691650391,
      "rewards/margins": 1.3840831518173218,
      "rewards/rejected": -2.2160935401916504,
      "step": 12521
    },
    {
      "epoch": 1.638586734276601,
      "grad_norm": 9.762138366699219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5099664926528931,
      "logits/rejected": -0.7351827621459961,
      "logps/chosen": -349.150390625,
      "logps/rejected": -395.90386962890625,
      "loss": 0.6606,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.4024691581726074,
      "rewards/margins": 0.5673549175262451,
      "rewards/rejected": -2.9698243141174316,
      "step": 12522
    },
    {
      "epoch": 1.6387175922139527,
      "grad_norm": 11.294855117797852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009010391309857368,
      "logits/rejected": -0.28012630343437195,
      "logps/chosen": -482.7884826660156,
      "logps/rejected": -459.08721923828125,
      "loss": 0.5826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9300481677055359,
      "rewards/margins": 1.669372797012329,
      "rewards/rejected": -2.5994210243225098,
      "step": 12523
    },
    {
      "epoch": 1.6388484501513045,
      "grad_norm": 11.635980606079102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13555893301963806,
      "logits/rejected": -0.2959880232810974,
      "logps/chosen": -421.5376281738281,
      "logps/rejected": -378.1748962402344,
      "loss": 1.0142,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3386062383651733,
      "rewards/margins": 0.16222494840621948,
      "rewards/rejected": -1.5008312463760376,
      "step": 12524
    },
    {
      "epoch": 1.6389793080886563,
      "grad_norm": 9.003609657287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2661612927913666,
      "logits/rejected": -0.4031134247779846,
      "logps/chosen": -407.0018310546875,
      "logps/rejected": -347.06939697265625,
      "loss": 0.4058,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3447233438491821,
      "rewards/margins": 1.2588231563568115,
      "rewards/rejected": -2.603546380996704,
      "step": 12525
    },
    {
      "epoch": 1.6391101660260081,
      "grad_norm": 7.931041717529297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28584277629852295,
      "logits/rejected": -0.4315050542354584,
      "logps/chosen": -405.957763671875,
      "logps/rejected": -306.85906982421875,
      "loss": 0.3791,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4342608451843262,
      "rewards/margins": 1.2055844068527222,
      "rewards/rejected": -2.639845132827759,
      "step": 12526
    },
    {
      "epoch": 1.63924102396336,
      "grad_norm": 5.8784894943237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32347792387008667,
      "logits/rejected": -0.24469363689422607,
      "logps/chosen": -454.97698974609375,
      "logps/rejected": -364.70037841796875,
      "loss": 0.2514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.658737063407898,
      "rewards/margins": 2.078986167907715,
      "rewards/rejected": -2.7377233505249023,
      "step": 12527
    },
    {
      "epoch": 1.6393718819007117,
      "grad_norm": 6.027157783508301,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2975289225578308,
      "logits/rejected": -0.23083139955997467,
      "logps/chosen": -423.0950927734375,
      "logps/rejected": -449.625244140625,
      "loss": 0.3283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7252765893936157,
      "rewards/margins": 1.2777202129364014,
      "rewards/rejected": -2.0029966831207275,
      "step": 12528
    },
    {
      "epoch": 1.6395027398380633,
      "grad_norm": 7.179526329040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24751606583595276,
      "logits/rejected": -0.15500372648239136,
      "logps/chosen": -353.5061950683594,
      "logps/rejected": -368.5963134765625,
      "loss": 0.2713,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.509456992149353,
      "rewards/margins": 2.0404446125030518,
      "rewards/rejected": -3.5499017238616943,
      "step": 12529
    },
    {
      "epoch": 1.639633597775415,
      "grad_norm": 8.617431640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5017468333244324,
      "logits/rejected": -0.41369763016700745,
      "logps/chosen": -679.094482421875,
      "logps/rejected": -548.2050170898438,
      "loss": 0.322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.820852279663086,
      "rewards/margins": 1.470086693763733,
      "rewards/rejected": -3.2909390926361084,
      "step": 12530
    },
    {
      "epoch": 1.6397644557127669,
      "grad_norm": 9.983673095703125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16125066578388214,
      "logits/rejected": -0.24998930096626282,
      "logps/chosen": -412.73590087890625,
      "logps/rejected": -452.1912536621094,
      "loss": 0.5117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3665628433227539,
      "rewards/margins": 0.8151391744613647,
      "rewards/rejected": -1.1817020177841187,
      "step": 12531
    },
    {
      "epoch": 1.6398953136501184,
      "grad_norm": 8.189441680908203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25270259380340576,
      "logits/rejected": -0.5174799561500549,
      "logps/chosen": -393.2662353515625,
      "logps/rejected": -337.989990234375,
      "loss": 0.5806,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4218554496765137,
      "rewards/margins": 0.838726818561554,
      "rewards/rejected": -2.260582208633423,
      "step": 12532
    },
    {
      "epoch": 1.6400261715874702,
      "grad_norm": 9.201107025146484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41920092701911926,
      "logits/rejected": -0.6011232137680054,
      "logps/chosen": -734.0806884765625,
      "logps/rejected": -649.361083984375,
      "loss": 0.44,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6752235293388367,
      "rewards/margins": 1.0718904733657837,
      "rewards/rejected": -1.7471139430999756,
      "step": 12533
    },
    {
      "epoch": 1.640157029524822,
      "grad_norm": 6.16977071762085,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09195832908153534,
      "logits/rejected": -0.251991331577301,
      "logps/chosen": -572.556884765625,
      "logps/rejected": -503.47943115234375,
      "loss": 0.289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5661827325820923,
      "rewards/margins": 1.462672233581543,
      "rewards/rejected": -2.0288548469543457,
      "step": 12534
    },
    {
      "epoch": 1.6402878874621738,
      "grad_norm": 16.49123764038086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30222421884536743,
      "logits/rejected": -0.47772324085235596,
      "logps/chosen": -418.76904296875,
      "logps/rejected": -440.44573974609375,
      "loss": 1.0507,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7317405939102173,
      "rewards/margins": -0.0998525321483612,
      "rewards/rejected": -1.6318880319595337,
      "step": 12535
    },
    {
      "epoch": 1.6404187453995256,
      "grad_norm": 4.603791236877441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15692806243896484,
      "logits/rejected": -0.37590426206588745,
      "logps/chosen": -306.5221252441406,
      "logps/rejected": -456.3527526855469,
      "loss": 0.2262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1023343801498413,
      "rewards/margins": 1.7458422183990479,
      "rewards/rejected": -2.8481764793395996,
      "step": 12536
    },
    {
      "epoch": 1.6405496033368774,
      "grad_norm": 6.838206768035889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22399233281612396,
      "logits/rejected": -0.17771388590335846,
      "logps/chosen": -266.6002197265625,
      "logps/rejected": -300.812255859375,
      "loss": 0.3058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3313521146774292,
      "rewards/margins": 1.6013615131378174,
      "rewards/rejected": -2.932713747024536,
      "step": 12537
    },
    {
      "epoch": 1.6406804612742292,
      "grad_norm": 9.475272178649902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2557947635650635,
      "logits/rejected": -0.2446518987417221,
      "logps/chosen": -517.7329711914062,
      "logps/rejected": -591.756103515625,
      "loss": 0.6289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7075395584106445,
      "rewards/margins": 1.293961763381958,
      "rewards/rejected": -3.0015013217926025,
      "step": 12538
    },
    {
      "epoch": 1.640811319211581,
      "grad_norm": 2.409172773361206,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44718053936958313,
      "logits/rejected": -0.6913437843322754,
      "logps/chosen": -481.8182067871094,
      "logps/rejected": -469.9169006347656,
      "loss": 0.0827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3960254490375519,
      "rewards/margins": 3.036648750305176,
      "rewards/rejected": -3.4326741695404053,
      "step": 12539
    },
    {
      "epoch": 1.6409421771489328,
      "grad_norm": 14.312873840332031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11132384091615677,
      "logits/rejected": -0.19352445006370544,
      "logps/chosen": -436.45068359375,
      "logps/rejected": -506.3131408691406,
      "loss": 0.9626,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1716841459274292,
      "rewards/margins": 0.45137494802474976,
      "rewards/rejected": -1.6230591535568237,
      "step": 12540
    },
    {
      "epoch": 1.6410730350862845,
      "grad_norm": 6.340066909790039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3158726692199707,
      "logits/rejected": -0.38976404070854187,
      "logps/chosen": -376.2588806152344,
      "logps/rejected": -454.6487121582031,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.058278270065784454,
      "rewards/margins": 2.117628335952759,
      "rewards/rejected": -2.1759064197540283,
      "step": 12541
    },
    {
      "epoch": 1.6412038930236363,
      "grad_norm": 8.80767822265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3166157007217407,
      "logits/rejected": -0.28084686398506165,
      "logps/chosen": -561.3176879882812,
      "logps/rejected": -474.2794494628906,
      "loss": 0.4667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7892036437988281,
      "rewards/margins": 1.1961822509765625,
      "rewards/rejected": -1.9853861331939697,
      "step": 12542
    },
    {
      "epoch": 1.6413347509609881,
      "grad_norm": 8.992475509643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11197540163993835,
      "logits/rejected": -0.16603657603263855,
      "logps/chosen": -470.46722412109375,
      "logps/rejected": -502.6044921875,
      "loss": 0.5352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9055594205856323,
      "rewards/margins": 1.264407992362976,
      "rewards/rejected": -2.1699674129486084,
      "step": 12543
    },
    {
      "epoch": 1.6414656088983397,
      "grad_norm": 13.182079315185547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6784109473228455,
      "logits/rejected": -0.48151758313179016,
      "logps/chosen": -559.9746704101562,
      "logps/rejected": -540.3164672851562,
      "loss": 0.5349,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4469884634017944,
      "rewards/margins": 0.9941682815551758,
      "rewards/rejected": -2.4411568641662598,
      "step": 12544
    },
    {
      "epoch": 1.6415964668356915,
      "grad_norm": 12.087162971496582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24393600225448608,
      "logits/rejected": 0.0317329540848732,
      "logps/chosen": -408.0064697265625,
      "logps/rejected": -391.0970153808594,
      "loss": 0.4722,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4700610935688019,
      "rewards/margins": 1.8758348226547241,
      "rewards/rejected": -2.3458962440490723,
      "step": 12545
    },
    {
      "epoch": 1.6417273247730433,
      "grad_norm": 7.153952121734619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35547250509262085,
      "logits/rejected": -0.5717898011207581,
      "logps/chosen": -367.2462158203125,
      "logps/rejected": -326.0164794921875,
      "loss": 0.4027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0743833780288696,
      "rewards/margins": 1.2024227380752563,
      "rewards/rejected": -2.276806116104126,
      "step": 12546
    },
    {
      "epoch": 1.641858182710395,
      "grad_norm": 5.7166972160339355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5492985248565674,
      "logits/rejected": -0.459839791059494,
      "logps/chosen": -495.99639892578125,
      "logps/rejected": -419.94281005859375,
      "loss": 0.2277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9661857485771179,
      "rewards/margins": 1.8608086109161377,
      "rewards/rejected": -2.8269944190979004,
      "step": 12547
    },
    {
      "epoch": 1.6419890406477466,
      "grad_norm": 2.7185754776000977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24674271047115326,
      "logits/rejected": -0.49764180183410645,
      "logps/chosen": -576.5790405273438,
      "logps/rejected": -480.45233154296875,
      "loss": 0.0892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0015705182449892163,
      "rewards/margins": 3.2496140003204346,
      "rewards/rejected": -3.2480435371398926,
      "step": 12548
    },
    {
      "epoch": 1.6421198985850984,
      "grad_norm": 9.304896354675293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4683963656425476,
      "logits/rejected": -0.4913673400878906,
      "logps/chosen": -379.35498046875,
      "logps/rejected": -477.4446105957031,
      "loss": 0.4753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8512856960296631,
      "rewards/margins": 1.3283202648162842,
      "rewards/rejected": -2.1796059608459473,
      "step": 12549
    },
    {
      "epoch": 1.6422507565224502,
      "grad_norm": 11.52748966217041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27336180210113525,
      "logits/rejected": -0.1776898354291916,
      "logps/chosen": -427.32891845703125,
      "logps/rejected": -520.3289184570312,
      "loss": 0.4253,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7872971296310425,
      "rewards/margins": 1.3548173904418945,
      "rewards/rejected": -3.1421146392822266,
      "step": 12550
    },
    {
      "epoch": 1.642381614459802,
      "grad_norm": 11.559135437011719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5448354482650757,
      "logits/rejected": -0.32325994968414307,
      "logps/chosen": -404.3700256347656,
      "logps/rejected": -398.7430419921875,
      "loss": 0.5281,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9513472318649292,
      "rewards/margins": 1.1396522521972656,
      "rewards/rejected": -3.0909996032714844,
      "step": 12551
    },
    {
      "epoch": 1.6425124723971538,
      "grad_norm": 11.920774459838867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5396080613136292,
      "logits/rejected": -0.3273201882839203,
      "logps/chosen": -419.8782958984375,
      "logps/rejected": -440.5076599121094,
      "loss": 0.7708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.499772071838379,
      "rewards/margins": 1.0198938846588135,
      "rewards/rejected": -2.5196659564971924,
      "step": 12552
    },
    {
      "epoch": 1.6426433303345056,
      "grad_norm": 5.376676559448242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.059342533349990845,
      "logits/rejected": -0.1736779510974884,
      "logps/chosen": -523.1923217773438,
      "logps/rejected": -276.1785888671875,
      "loss": 0.2702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10783655941486359,
      "rewards/margins": 1.504633903503418,
      "rewards/rejected": -1.612470269203186,
      "step": 12553
    },
    {
      "epoch": 1.6427741882718574,
      "grad_norm": 8.559943199157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.50031578540802,
      "logits/rejected": -0.5158828496932983,
      "logps/chosen": -361.69378662109375,
      "logps/rejected": -312.62969970703125,
      "loss": 0.5676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4455924034118652,
      "rewards/margins": 1.1056381464004517,
      "rewards/rejected": -2.5512306690216064,
      "step": 12554
    },
    {
      "epoch": 1.6429050462092092,
      "grad_norm": 6.309361934661865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34902381896972656,
      "logits/rejected": -0.4652809500694275,
      "logps/chosen": -384.30718994140625,
      "logps/rejected": -476.0834045410156,
      "loss": 0.2482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9792051315307617,
      "rewards/margins": 1.4762901067733765,
      "rewards/rejected": -3.4554951190948486,
      "step": 12555
    },
    {
      "epoch": 1.643035904146561,
      "grad_norm": 18.690786361694336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16777350008487701,
      "logits/rejected": -0.3351978063583374,
      "logps/chosen": -599.1422729492188,
      "logps/rejected": -445.2474670410156,
      "loss": 0.4411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2117104530334473,
      "rewards/margins": 1.1798601150512695,
      "rewards/rejected": -3.391570568084717,
      "step": 12556
    },
    {
      "epoch": 1.6431667620839128,
      "grad_norm": 7.146245002746582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10059988498687744,
      "logits/rejected": -0.34559330344200134,
      "logps/chosen": -628.666015625,
      "logps/rejected": -411.5342712402344,
      "loss": 0.302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5659731030464172,
      "rewards/margins": 1.589747428894043,
      "rewards/rejected": -2.1557204723358154,
      "step": 12557
    },
    {
      "epoch": 1.6432976200212646,
      "grad_norm": 4.161668300628662,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43340182304382324,
      "logits/rejected": -0.4124932289123535,
      "logps/chosen": -514.4200439453125,
      "logps/rejected": -607.56884765625,
      "loss": 0.1611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5250858068466187,
      "rewards/margins": 2.177154541015625,
      "rewards/rejected": -2.702240467071533,
      "step": 12558
    },
    {
      "epoch": 1.6434284779586161,
      "grad_norm": 10.774014472961426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19591452181339264,
      "logits/rejected": -0.2962260842323303,
      "logps/chosen": -413.5818176269531,
      "logps/rejected": -543.8170776367188,
      "loss": 0.5371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5837653875350952,
      "rewards/margins": 0.5152061581611633,
      "rewards/rejected": -1.0989716053009033,
      "step": 12559
    },
    {
      "epoch": 1.643559335895968,
      "grad_norm": 11.612977027893066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10054478049278259,
      "logits/rejected": -0.18653541803359985,
      "logps/chosen": -473.3785095214844,
      "logps/rejected": -447.24951171875,
      "loss": 0.4643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7016459703445435,
      "rewards/margins": 0.8959000110626221,
      "rewards/rejected": -1.5975459814071655,
      "step": 12560
    },
    {
      "epoch": 1.6436901938333197,
      "grad_norm": 13.285810470581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23624229431152344,
      "logits/rejected": -0.5166997313499451,
      "logps/chosen": -690.8966064453125,
      "logps/rejected": -385.04632568359375,
      "loss": 0.4998,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3396992683410645,
      "rewards/margins": 1.121067762374878,
      "rewards/rejected": -2.4607670307159424,
      "step": 12561
    },
    {
      "epoch": 1.6438210517706715,
      "grad_norm": 9.80099868774414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2176831066608429,
      "logits/rejected": -0.6192506551742554,
      "logps/chosen": -548.2642211914062,
      "logps/rejected": -557.578125,
      "loss": 0.4645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.04988431930542,
      "rewards/margins": 1.4912736415863037,
      "rewards/rejected": -2.5411577224731445,
      "step": 12562
    },
    {
      "epoch": 1.643951909708023,
      "grad_norm": 7.630472660064697,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15740936994552612,
      "logits/rejected": -0.09888875484466553,
      "logps/chosen": -383.40936279296875,
      "logps/rejected": -390.90509033203125,
      "loss": 0.3309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8366467952728271,
      "rewards/margins": 1.845903992652893,
      "rewards/rejected": -2.6825509071350098,
      "step": 12563
    },
    {
      "epoch": 1.6440827676453749,
      "grad_norm": 6.673274517059326,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3907727599143982,
      "logits/rejected": -0.6256576776504517,
      "logps/chosen": -468.78216552734375,
      "logps/rejected": -503.9208984375,
      "loss": 0.3259,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8849779367446899,
      "rewards/margins": 2.4827170372009277,
      "rewards/rejected": -3.3676950931549072,
      "step": 12564
    },
    {
      "epoch": 1.6442136255827267,
      "grad_norm": 6.6848931312561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48600155115127563,
      "logits/rejected": -0.48254460096359253,
      "logps/chosen": -374.41058349609375,
      "logps/rejected": -539.9485473632812,
      "loss": 0.3533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7894456386566162,
      "rewards/margins": 0.9465296864509583,
      "rewards/rejected": -1.7359753847122192,
      "step": 12565
    },
    {
      "epoch": 1.6443444835200784,
      "grad_norm": 6.020401954650879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1046549379825592,
      "logits/rejected": -0.1837315857410431,
      "logps/chosen": -395.18536376953125,
      "logps/rejected": -343.42132568359375,
      "loss": 0.3716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8667696118354797,
      "rewards/margins": 1.1434919834136963,
      "rewards/rejected": -2.0102615356445312,
      "step": 12566
    },
    {
      "epoch": 1.6444753414574302,
      "grad_norm": 8.104019165039062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5002328157424927,
      "logits/rejected": -0.3836559057235718,
      "logps/chosen": -378.5187683105469,
      "logps/rejected": -398.9838562011719,
      "loss": 0.372,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7480760812759399,
      "rewards/margins": 1.187989592552185,
      "rewards/rejected": -1.936065673828125,
      "step": 12567
    },
    {
      "epoch": 1.644606199394782,
      "grad_norm": 7.917856693267822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18793170154094696,
      "logits/rejected": -0.25144675374031067,
      "logps/chosen": -425.5258483886719,
      "logps/rejected": -601.960205078125,
      "loss": 0.3387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.028318166732788,
      "rewards/margins": 1.4546358585357666,
      "rewards/rejected": -2.4829540252685547,
      "step": 12568
    },
    {
      "epoch": 1.6447370573321338,
      "grad_norm": 8.773282051086426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3344128429889679,
      "logits/rejected": -0.42372727394104004,
      "logps/chosen": -453.4841613769531,
      "logps/rejected": -364.09259033203125,
      "loss": 0.408,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6660551428794861,
      "rewards/margins": 1.140515923500061,
      "rewards/rejected": -1.806571125984192,
      "step": 12569
    },
    {
      "epoch": 1.6448679152694856,
      "grad_norm": 9.652892112731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2938377857208252,
      "logits/rejected": -0.5414128303527832,
      "logps/chosen": -386.76116943359375,
      "logps/rejected": -372.8141784667969,
      "loss": 0.397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3072919845581055,
      "rewards/margins": 1.8925020694732666,
      "rewards/rejected": -3.199794054031372,
      "step": 12570
    },
    {
      "epoch": 1.6449987732068374,
      "grad_norm": 10.83204460144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34970974922180176,
      "logits/rejected": -0.3763207197189331,
      "logps/chosen": -468.9163818359375,
      "logps/rejected": -507.4488830566406,
      "loss": 0.6655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7998287081718445,
      "rewards/margins": 0.40439778566360474,
      "rewards/rejected": -1.2042264938354492,
      "step": 12571
    },
    {
      "epoch": 1.6451296311441892,
      "grad_norm": 9.578994750976562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0736747682094574,
      "logits/rejected": -0.22066792845726013,
      "logps/chosen": -527.2222900390625,
      "logps/rejected": -626.6231689453125,
      "loss": 0.4595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08122426271438599,
      "rewards/margins": 0.9154729247093201,
      "rewards/rejected": -0.9966973066329956,
      "step": 12572
    },
    {
      "epoch": 1.645260489081541,
      "grad_norm": 8.464698791503906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09404358267784119,
      "logits/rejected": -0.13204512000083923,
      "logps/chosen": -583.5358276367188,
      "logps/rejected": -686.0242309570312,
      "loss": 0.3367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43614500761032104,
      "rewards/margins": 1.6590276956558228,
      "rewards/rejected": -2.09517240524292,
      "step": 12573
    },
    {
      "epoch": 1.6453913470188928,
      "grad_norm": 46.665138244628906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3078303337097168,
      "logits/rejected": -0.17516514658927917,
      "logps/chosen": -394.4730224609375,
      "logps/rejected": -683.4072265625,
      "loss": 0.8557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1351072788238525,
      "rewards/margins": 0.4094463288784027,
      "rewards/rejected": -1.544553518295288,
      "step": 12574
    },
    {
      "epoch": 1.6455222049562443,
      "grad_norm": 9.349716186523438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27046045660972595,
      "logits/rejected": -0.4191973805427551,
      "logps/chosen": -646.8482055664062,
      "logps/rejected": -522.0938720703125,
      "loss": 0.504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004947260022163391,
      "rewards/margins": 0.6912055611610413,
      "rewards/rejected": -0.6862583160400391,
      "step": 12575
    },
    {
      "epoch": 1.6456530628935961,
      "grad_norm": 8.793011665344238,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06127415597438812,
      "logits/rejected": -0.014977270737290382,
      "logps/chosen": -576.0247192382812,
      "logps/rejected": -406.26287841796875,
      "loss": 0.339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8677631616592407,
      "rewards/margins": 1.623619794845581,
      "rewards/rejected": -2.4913830757141113,
      "step": 12576
    },
    {
      "epoch": 1.645783920830948,
      "grad_norm": 9.335700988769531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24923959374427795,
      "logits/rejected": -0.32651621103286743,
      "logps/chosen": -458.9219665527344,
      "logps/rejected": -384.2788391113281,
      "loss": 0.6022,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5473086833953857,
      "rewards/margins": 0.5757181644439697,
      "rewards/rejected": -2.1230263710021973,
      "step": 12577
    },
    {
      "epoch": 1.6459147787682995,
      "grad_norm": 7.458637237548828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08528701961040497,
      "logits/rejected": -0.3274011015892029,
      "logps/chosen": -327.2091979980469,
      "logps/rejected": -530.3587646484375,
      "loss": 0.3509,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.801997184753418,
      "rewards/margins": 2.055697441101074,
      "rewards/rejected": -3.857694625854492,
      "step": 12578
    },
    {
      "epoch": 1.6460456367056513,
      "grad_norm": 5.692739009857178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42809805274009705,
      "logits/rejected": -0.3994086682796478,
      "logps/chosen": -435.7358703613281,
      "logps/rejected": -380.745361328125,
      "loss": 0.2757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9981270432472229,
      "rewards/margins": 1.4703989028930664,
      "rewards/rejected": -2.4685258865356445,
      "step": 12579
    },
    {
      "epoch": 1.646176494643003,
      "grad_norm": 5.193770885467529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046433135867118835,
      "logits/rejected": -0.2901964783668518,
      "logps/chosen": -406.11236572265625,
      "logps/rejected": -582.3203735351562,
      "loss": 0.2491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9061458110809326,
      "rewards/margins": 2.0784053802490234,
      "rewards/rejected": -2.9845516681671143,
      "step": 12580
    },
    {
      "epoch": 1.6463073525803549,
      "grad_norm": 8.337618827819824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3515288829803467,
      "logits/rejected": -0.3978422284126282,
      "logps/chosen": -400.9493103027344,
      "logps/rejected": -438.5994873046875,
      "loss": 0.491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1023448705673218,
      "rewards/margins": 1.30791437625885,
      "rewards/rejected": -2.410259485244751,
      "step": 12581
    },
    {
      "epoch": 1.6464382105177067,
      "grad_norm": 4.167516708374023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01606198027729988,
      "logits/rejected": -0.049461521208286285,
      "logps/chosen": -411.6067199707031,
      "logps/rejected": -393.82733154296875,
      "loss": 0.1595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6933180093765259,
      "rewards/margins": 2.2596805095672607,
      "rewards/rejected": -2.952998638153076,
      "step": 12582
    },
    {
      "epoch": 1.6465690684550585,
      "grad_norm": 6.784381866455078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23613956570625305,
      "logits/rejected": -0.24933457374572754,
      "logps/chosen": -424.3781433105469,
      "logps/rejected": -651.0811157226562,
      "loss": 0.3241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0695852041244507,
      "rewards/margins": 1.9759438037872314,
      "rewards/rejected": -3.0455288887023926,
      "step": 12583
    },
    {
      "epoch": 1.6466999263924103,
      "grad_norm": 8.923995018005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031715407967567444,
      "logits/rejected": 0.036660030484199524,
      "logps/chosen": -453.269775390625,
      "logps/rejected": -487.99664306640625,
      "loss": 0.5041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7823875546455383,
      "rewards/margins": 0.9388604760169983,
      "rewards/rejected": -1.7212481498718262,
      "step": 12584
    },
    {
      "epoch": 1.646830784329762,
      "grad_norm": 16.304758071899414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7737751007080078,
      "logits/rejected": -0.4617193937301636,
      "logps/chosen": -443.9805908203125,
      "logps/rejected": -459.374267578125,
      "loss": 0.9174,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.988126277923584,
      "rewards/margins": -0.09295231103897095,
      "rewards/rejected": -1.8951741456985474,
      "step": 12585
    },
    {
      "epoch": 1.6469616422671138,
      "grad_norm": 9.048657417297363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07733823359012604,
      "logits/rejected": -0.3399916887283325,
      "logps/chosen": -502.0428466796875,
      "logps/rejected": -543.6537475585938,
      "loss": 0.4725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6338390111923218,
      "rewards/margins": 0.9217014312744141,
      "rewards/rejected": -1.5555404424667358,
      "step": 12586
    },
    {
      "epoch": 1.6470925002044656,
      "grad_norm": 10.164824485778809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3193323016166687,
      "logits/rejected": -0.5972045660018921,
      "logps/chosen": -674.74462890625,
      "logps/rejected": -582.5023193359375,
      "loss": 0.3861,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3563595116138458,
      "rewards/margins": 1.2283267974853516,
      "rewards/rejected": -1.584686279296875,
      "step": 12587
    },
    {
      "epoch": 1.6472233581418174,
      "grad_norm": 18.92276954650879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4125484824180603,
      "logits/rejected": -0.5112971067428589,
      "logps/chosen": -575.2160034179688,
      "logps/rejected": -423.9825134277344,
      "loss": 1.2085,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3202974796295166,
      "rewards/margins": -0.024841412901878357,
      "rewards/rejected": -2.2954556941986084,
      "step": 12588
    },
    {
      "epoch": 1.6473542160791692,
      "grad_norm": 9.736922264099121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2725348472595215,
      "logits/rejected": -0.16051873564720154,
      "logps/chosen": -410.1860656738281,
      "logps/rejected": -424.19085693359375,
      "loss": 0.4567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9538484811782837,
      "rewards/margins": 0.8451783657073975,
      "rewards/rejected": -1.7990268468856812,
      "step": 12589
    },
    {
      "epoch": 1.6474850740165208,
      "grad_norm": 12.188599586486816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48931223154067993,
      "logits/rejected": -0.399827241897583,
      "logps/chosen": -266.71246337890625,
      "logps/rejected": -424.7592468261719,
      "loss": 0.662,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.395928144454956,
      "rewards/margins": 0.6067384481430054,
      "rewards/rejected": -3.002666711807251,
      "step": 12590
    },
    {
      "epoch": 1.6476159319538726,
      "grad_norm": 10.97309398651123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3570590615272522,
      "logits/rejected": -0.32795003056526184,
      "logps/chosen": -452.35211181640625,
      "logps/rejected": -459.1949157714844,
      "loss": 0.6716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.273694634437561,
      "rewards/margins": 0.45904868841171265,
      "rewards/rejected": -1.7327433824539185,
      "step": 12591
    },
    {
      "epoch": 1.6477467898912244,
      "grad_norm": 7.25496244430542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17046117782592773,
      "logits/rejected": -0.17951950430870056,
      "logps/chosen": -595.58056640625,
      "logps/rejected": -556.1716918945312,
      "loss": 0.2788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1493088006973267,
      "rewards/margins": 1.506291389465332,
      "rewards/rejected": -2.6556003093719482,
      "step": 12592
    },
    {
      "epoch": 1.6478776478285762,
      "grad_norm": 6.766441822052002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1576519012451172,
      "logits/rejected": -0.27263131737709045,
      "logps/chosen": -300.3736877441406,
      "logps/rejected": -406.8490905761719,
      "loss": 0.3686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1193804740905762,
      "rewards/margins": 1.0517122745513916,
      "rewards/rejected": -2.1710925102233887,
      "step": 12593
    },
    {
      "epoch": 1.6480085057659277,
      "grad_norm": 8.348185539245605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.369488388299942,
      "logits/rejected": -0.35600709915161133,
      "logps/chosen": -452.43292236328125,
      "logps/rejected": -430.9941711425781,
      "loss": 0.3109,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5739969611167908,
      "rewards/margins": 1.7473649978637695,
      "rewards/rejected": -2.321361780166626,
      "step": 12594
    },
    {
      "epoch": 1.6481393637032795,
      "grad_norm": 7.3881611824035645,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03530927747488022,
      "logits/rejected": -0.007923688739538193,
      "logps/chosen": -750.897705078125,
      "logps/rejected": -487.5525817871094,
      "loss": 0.289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7304367423057556,
      "rewards/margins": 1.5876902341842651,
      "rewards/rejected": -2.318126916885376,
      "step": 12595
    },
    {
      "epoch": 1.6482702216406313,
      "grad_norm": 7.6984543800354,
      "learning_rate": 5e-06,
      "logits/chosen": -0.350713849067688,
      "logits/rejected": -0.4737016260623932,
      "logps/chosen": -378.85736083984375,
      "logps/rejected": -395.50555419921875,
      "loss": 0.4577,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6683861613273621,
      "rewards/margins": 0.744663655757904,
      "rewards/rejected": -1.4130496978759766,
      "step": 12596
    },
    {
      "epoch": 1.648401079577983,
      "grad_norm": 5.722488880157471,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3689170181751251,
      "logits/rejected": -0.41685906052589417,
      "logps/chosen": -621.5826416015625,
      "logps/rejected": -654.300537109375,
      "loss": 0.2111,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0087001323699951,
      "rewards/margins": 2.609469175338745,
      "rewards/rejected": -3.6181697845458984,
      "step": 12597
    },
    {
      "epoch": 1.6485319375153349,
      "grad_norm": 7.984352111816406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43782714009284973,
      "logits/rejected": -0.1670863777399063,
      "logps/chosen": -560.6600341796875,
      "logps/rejected": -540.3510131835938,
      "loss": 0.3439,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11223605275154114,
      "rewards/margins": 1.2960913181304932,
      "rewards/rejected": -1.4083274602890015,
      "step": 12598
    },
    {
      "epoch": 1.6486627954526867,
      "grad_norm": 13.708582878112793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44966065883636475,
      "logits/rejected": -0.6129151582717896,
      "logps/chosen": -289.6675109863281,
      "logps/rejected": -619.1996459960938,
      "loss": 0.9902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9833993911743164,
      "rewards/margins": 0.1557026207447052,
      "rewards/rejected": -2.139101982116699,
      "step": 12599
    },
    {
      "epoch": 1.6487936533900385,
      "grad_norm": 6.812320232391357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2279149740934372,
      "logits/rejected": -0.23818358778953552,
      "logps/chosen": -407.1826477050781,
      "logps/rejected": -418.8552551269531,
      "loss": 0.3853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4947933554649353,
      "rewards/margins": 1.0682557821273804,
      "rewards/rejected": -1.5630491971969604,
      "step": 12600
    },
    {
      "epoch": 1.6489245113273903,
      "grad_norm": 10.712996482849121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014016306027770042,
      "logits/rejected": -0.07668852061033249,
      "logps/chosen": -458.8866882324219,
      "logps/rejected": -606.3156127929688,
      "loss": 0.5625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4081661105155945,
      "rewards/margins": 1.2525146007537842,
      "rewards/rejected": -1.6606807708740234,
      "step": 12601
    },
    {
      "epoch": 1.649055369264742,
      "grad_norm": 6.819516181945801,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47932350635528564,
      "logits/rejected": -0.3903057873249054,
      "logps/chosen": -338.0459899902344,
      "logps/rejected": -343.545166015625,
      "loss": 0.3265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9050569534301758,
      "rewards/margins": 1.5762434005737305,
      "rewards/rejected": -2.481300115585327,
      "step": 12602
    },
    {
      "epoch": 1.6491862272020938,
      "grad_norm": 4.3623223304748535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07725315541028976,
      "logits/rejected": -0.3140372037887573,
      "logps/chosen": -468.809814453125,
      "logps/rejected": -439.78955078125,
      "loss": 0.1847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6659613251686096,
      "rewards/margins": 2.184878349304199,
      "rewards/rejected": -2.850839614868164,
      "step": 12603
    },
    {
      "epoch": 1.6493170851394456,
      "grad_norm": 10.258706092834473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.583399772644043,
      "logits/rejected": -0.41132208704948425,
      "logps/chosen": -530.0838623046875,
      "logps/rejected": -551.26123046875,
      "loss": 0.7,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0160534381866455,
      "rewards/margins": 0.5032942295074463,
      "rewards/rejected": -1.5193475484848022,
      "step": 12604
    },
    {
      "epoch": 1.6494479430767972,
      "grad_norm": 12.723125457763672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3690156638622284,
      "logits/rejected": -0.4452006220817566,
      "logps/chosen": -404.4219665527344,
      "logps/rejected": -392.413330078125,
      "loss": 1.1112,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4140413999557495,
      "rewards/margins": 0.007492661476135254,
      "rewards/rejected": -1.4215338230133057,
      "step": 12605
    },
    {
      "epoch": 1.649578801014149,
      "grad_norm": 6.4807868003845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5054298639297485,
      "logits/rejected": -0.6177691221237183,
      "logps/chosen": -363.56292724609375,
      "logps/rejected": -410.2290954589844,
      "loss": 0.2257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5716405510902405,
      "rewards/margins": 2.409212589263916,
      "rewards/rejected": -2.9808530807495117,
      "step": 12606
    },
    {
      "epoch": 1.6497096589515008,
      "grad_norm": 15.190398216247559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34459200501441956,
      "logits/rejected": -0.7538019418716431,
      "logps/chosen": -529.0279541015625,
      "logps/rejected": -493.0090637207031,
      "loss": 0.8164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5292080640792847,
      "rewards/margins": 1.0504852533340454,
      "rewards/rejected": -2.57969331741333,
      "step": 12607
    },
    {
      "epoch": 1.6498405168888526,
      "grad_norm": 12.54089641571045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3248269855976105,
      "logits/rejected": -0.43292805552482605,
      "logps/chosen": -462.0706787109375,
      "logps/rejected": -599.7552490234375,
      "loss": 0.6074,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7630068063735962,
      "rewards/margins": 0.5049023628234863,
      "rewards/rejected": -2.267909288406372,
      "step": 12608
    },
    {
      "epoch": 1.6499713748262042,
      "grad_norm": 12.274052619934082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38507187366485596,
      "logits/rejected": -0.056567080318927765,
      "logps/chosen": -463.8307189941406,
      "logps/rejected": -364.40924072265625,
      "loss": 0.5114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45998191833496094,
      "rewards/margins": 1.2380746603012085,
      "rewards/rejected": -1.698056697845459,
      "step": 12609
    },
    {
      "epoch": 1.650102232763556,
      "grad_norm": 10.810402870178223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3175029456615448,
      "logits/rejected": -0.29894429445266724,
      "logps/chosen": -390.6351013183594,
      "logps/rejected": -516.8167724609375,
      "loss": 0.5509,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2277857065200806,
      "rewards/margins": 1.046499252319336,
      "rewards/rejected": -2.274285078048706,
      "step": 12610
    },
    {
      "epoch": 1.6502330907009077,
      "grad_norm": 10.60233211517334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.015358656644821167,
      "logits/rejected": -0.2542511820793152,
      "logps/chosen": -345.4477844238281,
      "logps/rejected": -476.7344055175781,
      "loss": 0.7083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1782015562057495,
      "rewards/margins": 0.38476675748825073,
      "rewards/rejected": -1.5629682540893555,
      "step": 12611
    },
    {
      "epoch": 1.6503639486382595,
      "grad_norm": 9.201699256896973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30603834986686707,
      "logits/rejected": -0.4284060001373291,
      "logps/chosen": -583.35302734375,
      "logps/rejected": -428.5199279785156,
      "loss": 0.3879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5259645581245422,
      "rewards/margins": 1.2133172750473022,
      "rewards/rejected": -1.7392817735671997,
      "step": 12612
    },
    {
      "epoch": 1.6504948065756113,
      "grad_norm": 4.7297492027282715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.328025221824646,
      "logits/rejected": -0.22941678762435913,
      "logps/chosen": -424.54510498046875,
      "logps/rejected": -346.7339172363281,
      "loss": 0.339,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5505812168121338,
      "rewards/margins": 1.6453752517700195,
      "rewards/rejected": -2.195956230163574,
      "step": 12613
    },
    {
      "epoch": 1.650625664512963,
      "grad_norm": 10.74988079071045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36538395285606384,
      "logits/rejected": -0.2957855463027954,
      "logps/chosen": -445.7147216796875,
      "logps/rejected": -548.5103759765625,
      "loss": 0.5922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3138589859008789,
      "rewards/margins": 0.4091610610485077,
      "rewards/rejected": -0.7230199575424194,
      "step": 12614
    },
    {
      "epoch": 1.650756522450315,
      "grad_norm": 11.004094123840332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2091837227344513,
      "logits/rejected": -0.02206236869096756,
      "logps/chosen": -455.722412109375,
      "logps/rejected": -401.5204772949219,
      "loss": 0.5037,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8936833143234253,
      "rewards/margins": 0.9908339977264404,
      "rewards/rejected": -2.8845176696777344,
      "step": 12615
    },
    {
      "epoch": 1.6508873803876667,
      "grad_norm": 7.1519904136657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11114749312400818,
      "logits/rejected": -0.25147947669029236,
      "logps/chosen": -456.5041198730469,
      "logps/rejected": -387.1506042480469,
      "loss": 0.3831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0626055002212524,
      "rewards/margins": 1.2878235578536987,
      "rewards/rejected": -2.350428819656372,
      "step": 12616
    },
    {
      "epoch": 1.6510182383250185,
      "grad_norm": 5.9825568199157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06794571876525879,
      "logits/rejected": -0.1102331280708313,
      "logps/chosen": -454.11724853515625,
      "logps/rejected": -442.70184326171875,
      "loss": 0.2959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8565912842750549,
      "rewards/margins": 1.5209542512893677,
      "rewards/rejected": -2.3775455951690674,
      "step": 12617
    },
    {
      "epoch": 1.6511490962623703,
      "grad_norm": 8.308417320251465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45303723216056824,
      "logits/rejected": -0.4643653333187103,
      "logps/chosen": -362.0912780761719,
      "logps/rejected": -501.4285888671875,
      "loss": 0.4595,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8567026853561401,
      "rewards/margins": 1.4851709604263306,
      "rewards/rejected": -2.3418736457824707,
      "step": 12618
    },
    {
      "epoch": 1.651279954199722,
      "grad_norm": 9.095148086547852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2639317512512207,
      "logits/rejected": -0.47456982731819153,
      "logps/chosen": -508.5304870605469,
      "logps/rejected": -528.0159912109375,
      "loss": 0.3776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9331469535827637,
      "rewards/margins": 2.1028032302856445,
      "rewards/rejected": -4.035950183868408,
      "step": 12619
    },
    {
      "epoch": 1.6514108121370736,
      "grad_norm": 11.41088581085205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3566494882106781,
      "logits/rejected": -0.19948413968086243,
      "logps/chosen": -381.0364074707031,
      "logps/rejected": -414.3027648925781,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2785837650299072,
      "rewards/margins": 0.5643187761306763,
      "rewards/rejected": -2.842902660369873,
      "step": 12620
    },
    {
      "epoch": 1.6515416700744254,
      "grad_norm": 7.652770042419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11440914124250412,
      "logits/rejected": -0.16306866705417633,
      "logps/chosen": -455.3903503417969,
      "logps/rejected": -479.92413330078125,
      "loss": 0.3431,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8357720375061035,
      "rewards/margins": 1.5191035270690918,
      "rewards/rejected": -2.3548755645751953,
      "step": 12621
    },
    {
      "epoch": 1.6516725280117772,
      "grad_norm": 11.995229721069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31108060479164124,
      "logits/rejected": -0.3644139766693115,
      "logps/chosen": -609.005859375,
      "logps/rejected": -464.211669921875,
      "loss": 0.7917,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48763561248779297,
      "rewards/margins": 0.5051729679107666,
      "rewards/rejected": -0.9928085207939148,
      "step": 12622
    },
    {
      "epoch": 1.651803385949129,
      "grad_norm": 10.120063781738281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06991957128047943,
      "logits/rejected": -0.17347866296768188,
      "logps/chosen": -515.1721801757812,
      "logps/rejected": -407.1963195800781,
      "loss": 0.4856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0103596448898315,
      "rewards/margins": 1.2600140571594238,
      "rewards/rejected": -2.270373582839966,
      "step": 12623
    },
    {
      "epoch": 1.6519342438864806,
      "grad_norm": 5.510203838348389,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40212076902389526,
      "logits/rejected": -0.6875881552696228,
      "logps/chosen": -453.9480895996094,
      "logps/rejected": -343.7918395996094,
      "loss": 0.2908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9355304837226868,
      "rewards/margins": 1.760205626487732,
      "rewards/rejected": -2.6957364082336426,
      "step": 12624
    },
    {
      "epoch": 1.6520651018238324,
      "grad_norm": 6.147453308105469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48959994316101074,
      "logits/rejected": -0.6844082474708557,
      "logps/chosen": -489.19970703125,
      "logps/rejected": -379.7484130859375,
      "loss": 0.2483,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9266414642333984,
      "rewards/margins": 1.897693395614624,
      "rewards/rejected": -2.8243348598480225,
      "step": 12625
    },
    {
      "epoch": 1.6521959597611842,
      "grad_norm": 13.069269180297852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23275995254516602,
      "logits/rejected": -0.25275471806526184,
      "logps/chosen": -513.1383056640625,
      "logps/rejected": -570.9447021484375,
      "loss": 0.6609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4733515977859497,
      "rewards/margins": 0.9399474859237671,
      "rewards/rejected": -2.4132988452911377,
      "step": 12626
    },
    {
      "epoch": 1.652326817698536,
      "grad_norm": 10.928668022155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0640951618552208,
      "logits/rejected": -0.4697890281677246,
      "logps/chosen": -495.9485168457031,
      "logps/rejected": -439.4138488769531,
      "loss": 0.4978,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7029427289962769,
      "rewards/margins": 0.9903593063354492,
      "rewards/rejected": -2.6933019161224365,
      "step": 12627
    },
    {
      "epoch": 1.6524576756358877,
      "grad_norm": 7.582618236541748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5212246775627136,
      "logits/rejected": -0.4914259612560272,
      "logps/chosen": -323.9322204589844,
      "logps/rejected": -457.0823974609375,
      "loss": 0.3398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.49297034740448,
      "rewards/margins": 1.3457099199295044,
      "rewards/rejected": -2.8386802673339844,
      "step": 12628
    },
    {
      "epoch": 1.6525885335732395,
      "grad_norm": 5.060673713684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3204900324344635,
      "logits/rejected": -0.36152029037475586,
      "logps/chosen": -530.3535766601562,
      "logps/rejected": -702.5383911132812,
      "loss": 0.1609,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5373420715332031,
      "rewards/margins": 3.4584319591522217,
      "rewards/rejected": -3.995774269104004,
      "step": 12629
    },
    {
      "epoch": 1.6527193915105913,
      "grad_norm": 10.506448745727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27435579895973206,
      "logits/rejected": -0.14987972378730774,
      "logps/chosen": -640.295166015625,
      "logps/rejected": -582.3093872070312,
      "loss": 0.3215,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5201976299285889,
      "rewards/margins": 1.7794488668441772,
      "rewards/rejected": -2.2996466159820557,
      "step": 12630
    },
    {
      "epoch": 1.6528502494479431,
      "grad_norm": 8.469624519348145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005787596106529236,
      "logits/rejected": 0.015114161185920238,
      "logps/chosen": -453.0324401855469,
      "logps/rejected": -373.09375,
      "loss": 0.3251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47071215510368347,
      "rewards/margins": 1.5116134881973267,
      "rewards/rejected": -1.9823256731033325,
      "step": 12631
    },
    {
      "epoch": 1.652981107385295,
      "grad_norm": 9.7483491897583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13529986143112183,
      "logits/rejected": -0.07197234779596329,
      "logps/chosen": -619.730712890625,
      "logps/rejected": -422.5792541503906,
      "loss": 0.4022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5616693496704102,
      "rewards/margins": 1.4616358280181885,
      "rewards/rejected": -2.0233051776885986,
      "step": 12632
    },
    {
      "epoch": 1.6531119653226467,
      "grad_norm": 10.335155487060547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027343451976776123,
      "logits/rejected": -0.00017700716853141785,
      "logps/chosen": -367.5185852050781,
      "logps/rejected": -378.8464050292969,
      "loss": 0.4822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3447797298431396,
      "rewards/margins": 1.0443576574325562,
      "rewards/rejected": -2.3891375064849854,
      "step": 12633
    },
    {
      "epoch": 1.6532428232599985,
      "grad_norm": 10.040080070495605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26889726519584656,
      "logits/rejected": -0.23658694326877594,
      "logps/chosen": -559.1204833984375,
      "logps/rejected": -419.5183410644531,
      "loss": 0.466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4100383520126343,
      "rewards/margins": 1.1107738018035889,
      "rewards/rejected": -2.5208122730255127,
      "step": 12634
    },
    {
      "epoch": 1.6533736811973503,
      "grad_norm": 8.223796844482422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1833535134792328,
      "logits/rejected": -0.42107677459716797,
      "logps/chosen": -570.0430908203125,
      "logps/rejected": -435.2515869140625,
      "loss": 0.2693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8995699286460876,
      "rewards/margins": 1.8224444389343262,
      "rewards/rejected": -2.7220141887664795,
      "step": 12635
    },
    {
      "epoch": 1.6535045391347019,
      "grad_norm": 3.2525782585144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2591283321380615,
      "logits/rejected": -0.08670179545879364,
      "logps/chosen": -582.053466796875,
      "logps/rejected": -609.6632690429688,
      "loss": 0.1028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1654815822839737,
      "rewards/margins": 2.622927665710449,
      "rewards/rejected": -2.4574460983276367,
      "step": 12636
    },
    {
      "epoch": 1.6536353970720536,
      "grad_norm": 8.605185508728027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14229416847229004,
      "logits/rejected": -0.16251523792743683,
      "logps/chosen": -459.065185546875,
      "logps/rejected": -577.8763427734375,
      "loss": 0.3984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8763401508331299,
      "rewards/margins": 1.594200611114502,
      "rewards/rejected": -2.470540761947632,
      "step": 12637
    },
    {
      "epoch": 1.6537662550094054,
      "grad_norm": 9.795501708984375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21875974535942078,
      "logits/rejected": -0.5352872610092163,
      "logps/chosen": -563.98486328125,
      "logps/rejected": -521.802734375,
      "loss": 0.4015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4918248653411865,
      "rewards/margins": 1.0410935878753662,
      "rewards/rejected": -2.532918691635132,
      "step": 12638
    },
    {
      "epoch": 1.6538971129467572,
      "grad_norm": 13.400555610656738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.156711608171463,
      "logits/rejected": -0.06421080231666565,
      "logps/chosen": -388.1304626464844,
      "logps/rejected": -353.4456787109375,
      "loss": 0.6837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2023319005966187,
      "rewards/margins": 0.49574682116508484,
      "rewards/rejected": -1.6980788707733154,
      "step": 12639
    },
    {
      "epoch": 1.6540279708841088,
      "grad_norm": 8.360053062438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2357160747051239,
      "logits/rejected": -0.3144019544124603,
      "logps/chosen": -334.0108337402344,
      "logps/rejected": -476.5329895019531,
      "loss": 0.3842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2275729179382324,
      "rewards/margins": 1.0177431106567383,
      "rewards/rejected": -2.2453160285949707,
      "step": 12640
    },
    {
      "epoch": 1.6541588288214606,
      "grad_norm": 8.681829452514648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1318872720003128,
      "logits/rejected": -0.15199409425258636,
      "logps/chosen": -510.739013671875,
      "logps/rejected": -399.577392578125,
      "loss": 0.3956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06909923255443573,
      "rewards/margins": 1.4429997205734253,
      "rewards/rejected": -1.5120989084243774,
      "step": 12641
    },
    {
      "epoch": 1.6542896867588124,
      "grad_norm": 4.931316375732422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18004091084003448,
      "logits/rejected": -0.3240112364292145,
      "logps/chosen": -339.43768310546875,
      "logps/rejected": -431.334716796875,
      "loss": 0.3423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8557016253471375,
      "rewards/margins": 1.8014719486236572,
      "rewards/rejected": -2.6571733951568604,
      "step": 12642
    },
    {
      "epoch": 1.6544205446961642,
      "grad_norm": 9.974096298217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07828061282634735,
      "logits/rejected": -0.25705084204673767,
      "logps/chosen": -604.4850463867188,
      "logps/rejected": -512.1782836914062,
      "loss": 0.5836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5443005561828613,
      "rewards/margins": 0.9161753058433533,
      "rewards/rejected": -2.4604759216308594,
      "step": 12643
    },
    {
      "epoch": 1.654551402633516,
      "grad_norm": 10.31576919555664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3156026005744934,
      "logits/rejected": -0.05013728886842728,
      "logps/chosen": -385.3974609375,
      "logps/rejected": -536.12890625,
      "loss": 0.5615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7615107297897339,
      "rewards/margins": 0.6633049845695496,
      "rewards/rejected": -2.4248158931732178,
      "step": 12644
    },
    {
      "epoch": 1.6546822605708678,
      "grad_norm": 6.729769229888916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.436998188495636,
      "logits/rejected": -0.3752977252006531,
      "logps/chosen": -302.990234375,
      "logps/rejected": -328.4731140136719,
      "loss": 0.4581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8383808135986328,
      "rewards/margins": 1.0330276489257812,
      "rewards/rejected": -1.871408462524414,
      "step": 12645
    },
    {
      "epoch": 1.6548131185082195,
      "grad_norm": 4.8926215171813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2344907969236374,
      "logits/rejected": -0.4684121012687683,
      "logps/chosen": -410.320068359375,
      "logps/rejected": -367.240478515625,
      "loss": 0.1822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8530895709991455,
      "rewards/margins": 1.7873971462249756,
      "rewards/rejected": -2.640486717224121,
      "step": 12646
    },
    {
      "epoch": 1.6549439764455713,
      "grad_norm": 7.259542465209961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23002737760543823,
      "logits/rejected": -0.44359177350997925,
      "logps/chosen": -369.93768310546875,
      "logps/rejected": -510.2596435546875,
      "loss": 0.3939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8332979679107666,
      "rewards/margins": 2.4236178398132324,
      "rewards/rejected": -3.256916046142578,
      "step": 12647
    },
    {
      "epoch": 1.6550748343829231,
      "grad_norm": 11.646930694580078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11522756516933441,
      "logits/rejected": -0.22415080666542053,
      "logps/chosen": -581.111083984375,
      "logps/rejected": -556.0015869140625,
      "loss": 0.5139,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7190843820571899,
      "rewards/margins": 1.2196385860443115,
      "rewards/rejected": -1.9387229681015015,
      "step": 12648
    },
    {
      "epoch": 1.655205692320275,
      "grad_norm": 9.547623634338379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14758023619651794,
      "logits/rejected": -0.4598287343978882,
      "logps/chosen": -389.7408447265625,
      "logps/rejected": -310.59320068359375,
      "loss": 0.6746,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.361919641494751,
      "rewards/margins": 0.6319679021835327,
      "rewards/rejected": -1.9938875436782837,
      "step": 12649
    },
    {
      "epoch": 1.6553365502576267,
      "grad_norm": 6.787718296051025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27282336354255676,
      "logits/rejected": -0.2722417712211609,
      "logps/chosen": -647.496826171875,
      "logps/rejected": -577.8858032226562,
      "loss": 0.2819,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9753733277320862,
      "rewards/margins": 2.1426379680633545,
      "rewards/rejected": -3.118011713027954,
      "step": 12650
    },
    {
      "epoch": 1.6554674081949783,
      "grad_norm": 11.95002555847168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3748338520526886,
      "logits/rejected": -0.34563201665878296,
      "logps/chosen": -484.31878662109375,
      "logps/rejected": -600.12060546875,
      "loss": 0.6159,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5751144289970398,
      "rewards/margins": 1.2431188821792603,
      "rewards/rejected": -1.8182332515716553,
      "step": 12651
    },
    {
      "epoch": 1.65559826613233,
      "grad_norm": 11.273024559020996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042924441397190094,
      "logits/rejected": -0.3067125976085663,
      "logps/chosen": -420.87353515625,
      "logps/rejected": -460.3554382324219,
      "loss": 0.5411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4400860071182251,
      "rewards/margins": 0.7570825815200806,
      "rewards/rejected": -1.1971685886383057,
      "step": 12652
    },
    {
      "epoch": 1.6557291240696819,
      "grad_norm": 8.5121431350708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12756498157978058,
      "logits/rejected": -0.1265818476676941,
      "logps/chosen": -545.0247802734375,
      "logps/rejected": -503.86444091796875,
      "loss": 0.3374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.003456488251686096,
      "rewards/margins": 1.927213430404663,
      "rewards/rejected": -1.9237569570541382,
      "step": 12653
    },
    {
      "epoch": 1.6558599820070337,
      "grad_norm": 13.43218994140625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.052749164402484894,
      "logits/rejected": -0.057951267808675766,
      "logps/chosen": -473.43267822265625,
      "logps/rejected": -541.943603515625,
      "loss": 0.8277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4845730066299438,
      "rewards/margins": 0.4522014260292053,
      "rewards/rejected": -1.936774492263794,
      "step": 12654
    },
    {
      "epoch": 1.6559908399443852,
      "grad_norm": 9.320898056030273,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06718669831752777,
      "logits/rejected": -0.05019117891788483,
      "logps/chosen": -676.8151245117188,
      "logps/rejected": -697.504150390625,
      "loss": 0.5108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8391861319541931,
      "rewards/margins": 1.2716081142425537,
      "rewards/rejected": -2.1107940673828125,
      "step": 12655
    },
    {
      "epoch": 1.656121697881737,
      "grad_norm": 12.885215759277344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22250306606292725,
      "logits/rejected": -0.3126904368400574,
      "logps/chosen": -665.0217895507812,
      "logps/rejected": -473.9745178222656,
      "loss": 0.6289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.279531478881836,
      "rewards/margins": 1.0617053508758545,
      "rewards/rejected": -2.3412368297576904,
      "step": 12656
    },
    {
      "epoch": 1.6562525558190888,
      "grad_norm": 8.51948356628418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4004567265510559,
      "logits/rejected": -0.4535645544528961,
      "logps/chosen": -359.6056213378906,
      "logps/rejected": -375.1527099609375,
      "loss": 0.4521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0083808898925781,
      "rewards/margins": 1.3992687463760376,
      "rewards/rejected": -2.4076497554779053,
      "step": 12657
    },
    {
      "epoch": 1.6563834137564406,
      "grad_norm": 7.1222124099731445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0470234714448452,
      "logits/rejected": -0.1433931142091751,
      "logps/chosen": -335.9031982421875,
      "logps/rejected": -404.2867431640625,
      "loss": 0.4146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6275247931480408,
      "rewards/margins": 1.164583683013916,
      "rewards/rejected": -1.7921085357666016,
      "step": 12658
    },
    {
      "epoch": 1.6565142716937924,
      "grad_norm": 9.720318794250488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014766556210815907,
      "logits/rejected": -0.05683209374547005,
      "logps/chosen": -408.257080078125,
      "logps/rejected": -482.1884765625,
      "loss": 0.6295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4855802059173584,
      "rewards/margins": 1.5407774448394775,
      "rewards/rejected": -3.026357650756836,
      "step": 12659
    },
    {
      "epoch": 1.6566451296311442,
      "grad_norm": 9.382129669189453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08733584731817245,
      "logits/rejected": -0.3017953336238861,
      "logps/chosen": -513.9262084960938,
      "logps/rejected": -621.2284545898438,
      "loss": 0.4437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0413051843643188,
      "rewards/margins": 1.0747014284133911,
      "rewards/rejected": -2.116006851196289,
      "step": 12660
    },
    {
      "epoch": 1.656775987568496,
      "grad_norm": 9.141594886779785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0741683766245842,
      "logits/rejected": -0.1898246705532074,
      "logps/chosen": -671.899658203125,
      "logps/rejected": -382.4248046875,
      "loss": 0.4087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9532368183135986,
      "rewards/margins": 0.9229393005371094,
      "rewards/rejected": -1.8761762380599976,
      "step": 12661
    },
    {
      "epoch": 1.6569068455058478,
      "grad_norm": 5.107543468475342,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4296359419822693,
      "logits/rejected": -0.4493955969810486,
      "logps/chosen": -308.1341552734375,
      "logps/rejected": -388.83062744140625,
      "loss": 0.3382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9297374486923218,
      "rewards/margins": 1.5180048942565918,
      "rewards/rejected": -2.447742223739624,
      "step": 12662
    },
    {
      "epoch": 1.6570377034431996,
      "grad_norm": 9.325867652893066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06188332289457321,
      "logits/rejected": 0.013666053302586079,
      "logps/chosen": -481.48040771484375,
      "logps/rejected": -461.5505676269531,
      "loss": 0.686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0135124921798706,
      "rewards/margins": 0.6613719463348389,
      "rewards/rejected": -1.674884557723999,
      "step": 12663
    },
    {
      "epoch": 1.6571685613805514,
      "grad_norm": 8.156237602233887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21208131313323975,
      "logits/rejected": -0.22026196122169495,
      "logps/chosen": -475.23065185546875,
      "logps/rejected": -390.8515625,
      "loss": 0.3857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3833014965057373,
      "rewards/margins": 1.2449541091918945,
      "rewards/rejected": -2.628255605697632,
      "step": 12664
    },
    {
      "epoch": 1.6572994193179031,
      "grad_norm": 9.59875202178955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.146515890955925,
      "logits/rejected": 0.05865485221147537,
      "logps/chosen": -327.39813232421875,
      "logps/rejected": -434.27239990234375,
      "loss": 0.4771,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2645230293273926,
      "rewards/margins": 0.7928810119628906,
      "rewards/rejected": -2.057404041290283,
      "step": 12665
    },
    {
      "epoch": 1.6574302772552547,
      "grad_norm": 6.897101402282715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20945586264133453,
      "logits/rejected": -0.4621598720550537,
      "logps/chosen": -377.133056640625,
      "logps/rejected": -364.89404296875,
      "loss": 0.2652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2666536569595337,
      "rewards/margins": 2.105424642562866,
      "rewards/rejected": -3.3720779418945312,
      "step": 12666
    },
    {
      "epoch": 1.6575611351926065,
      "grad_norm": 10.434142112731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15159174799919128,
      "logits/rejected": -0.10498614609241486,
      "logps/chosen": -573.7106323242188,
      "logps/rejected": -595.8382568359375,
      "loss": 0.4753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8241239786148071,
      "rewards/margins": 0.9876346588134766,
      "rewards/rejected": -2.811758518218994,
      "step": 12667
    },
    {
      "epoch": 1.6576919931299583,
      "grad_norm": 7.092102527618408,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23315644264221191,
      "logits/rejected": -0.17317678034305573,
      "logps/chosen": -445.26751708984375,
      "logps/rejected": -565.0537109375,
      "loss": 0.4006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8634241223335266,
      "rewards/margins": 0.9093238711357117,
      "rewards/rejected": -1.7727478742599487,
      "step": 12668
    },
    {
      "epoch": 1.65782285106731,
      "grad_norm": 7.635714054107666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2950974106788635,
      "logits/rejected": -0.28548485040664673,
      "logps/chosen": -502.7168273925781,
      "logps/rejected": -510.8995056152344,
      "loss": 0.5903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5368334054946899,
      "rewards/margins": 1.021889328956604,
      "rewards/rejected": -1.558722734451294,
      "step": 12669
    },
    {
      "epoch": 1.6579537090046617,
      "grad_norm": 8.663435935974121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3403942584991455,
      "logits/rejected": -0.27832669019699097,
      "logps/chosen": -400.9754333496094,
      "logps/rejected": -459.05865478515625,
      "loss": 0.4934,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6060854196548462,
      "rewards/margins": 1.1629711389541626,
      "rewards/rejected": -1.7690565586090088,
      "step": 12670
    },
    {
      "epoch": 1.6580845669420134,
      "grad_norm": 9.287151336669922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4005005657672882,
      "logits/rejected": -0.4781820774078369,
      "logps/chosen": -425.77001953125,
      "logps/rejected": -433.8913269042969,
      "loss": 0.3637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3404598236083984,
      "rewards/margins": 3.8369741439819336,
      "rewards/rejected": -5.177433967590332,
      "step": 12671
    },
    {
      "epoch": 1.6582154248793652,
      "grad_norm": 11.018235206604004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16301196813583374,
      "logits/rejected": -0.24276646971702576,
      "logps/chosen": -436.47906494140625,
      "logps/rejected": -392.5325927734375,
      "loss": 0.6537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.808971643447876,
      "rewards/margins": 0.5219566822052002,
      "rewards/rejected": -2.330928087234497,
      "step": 12672
    },
    {
      "epoch": 1.658346282816717,
      "grad_norm": 5.8000264167785645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47820499539375305,
      "logits/rejected": -0.41324934363365173,
      "logps/chosen": -435.3548583984375,
      "logps/rejected": -412.153076171875,
      "loss": 0.3576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0678895711898804,
      "rewards/margins": 1.1921610832214355,
      "rewards/rejected": -2.2600505352020264,
      "step": 12673
    },
    {
      "epoch": 1.6584771407540688,
      "grad_norm": 10.70875358581543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40994757413864136,
      "logits/rejected": -0.5493832230567932,
      "logps/chosen": -438.7214660644531,
      "logps/rejected": -434.18572998046875,
      "loss": 0.6799,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4770742654800415,
      "rewards/margins": 0.8381760120391846,
      "rewards/rejected": -2.3152501583099365,
      "step": 12674
    },
    {
      "epoch": 1.6586079986914206,
      "grad_norm": 8.123619079589844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3638404309749603,
      "logits/rejected": -0.6215912103652954,
      "logps/chosen": -575.9479370117188,
      "logps/rejected": -647.13818359375,
      "loss": 0.3673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.157650351524353,
      "rewards/margins": 1.5001240968704224,
      "rewards/rejected": -2.6577744483947754,
      "step": 12675
    },
    {
      "epoch": 1.6587388566287724,
      "grad_norm": 6.068550109863281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.564873218536377,
      "logits/rejected": -0.3560214042663574,
      "logps/chosen": -471.1007385253906,
      "logps/rejected": -427.5486755371094,
      "loss": 0.3419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9903767108917236,
      "rewards/margins": 1.428479790687561,
      "rewards/rejected": -2.418856620788574,
      "step": 12676
    },
    {
      "epoch": 1.6588697145661242,
      "grad_norm": 21.46733283996582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.508227527141571,
      "logits/rejected": -0.6454159617424011,
      "logps/chosen": -429.57586669921875,
      "logps/rejected": -366.81890869140625,
      "loss": 1.065,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.425037145614624,
      "rewards/margins": 0.05156439542770386,
      "rewards/rejected": -2.4766016006469727,
      "step": 12677
    },
    {
      "epoch": 1.659000572503476,
      "grad_norm": 11.686561584472656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1241600513458252,
      "logits/rejected": -0.14943882822990417,
      "logps/chosen": -402.4283752441406,
      "logps/rejected": -378.3172912597656,
      "loss": 0.562,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0802561044692993,
      "rewards/margins": 1.2538009881973267,
      "rewards/rejected": -2.334057092666626,
      "step": 12678
    },
    {
      "epoch": 1.6591314304408278,
      "grad_norm": 6.095625400543213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09175144135951996,
      "logits/rejected": -0.204023540019989,
      "logps/chosen": -502.2532958984375,
      "logps/rejected": -451.6829833984375,
      "loss": 0.2879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2524162530899048,
      "rewards/margins": 2.0162346363067627,
      "rewards/rejected": -2.268651008605957,
      "step": 12679
    },
    {
      "epoch": 1.6592622883781796,
      "grad_norm": 9.321060180664062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25157904624938965,
      "logits/rejected": -0.1716102808713913,
      "logps/chosen": -401.58966064453125,
      "logps/rejected": -485.173583984375,
      "loss": 0.5514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9259979724884033,
      "rewards/margins": 1.5462384223937988,
      "rewards/rejected": -2.472236394882202,
      "step": 12680
    },
    {
      "epoch": 1.6593931463155314,
      "grad_norm": 4.24302864074707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2084742784500122,
      "logits/rejected": -0.09628957509994507,
      "logps/chosen": -373.30609130859375,
      "logps/rejected": -393.1112365722656,
      "loss": 0.1517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3489694595336914,
      "rewards/margins": 2.6115176677703857,
      "rewards/rejected": -2.960486888885498,
      "step": 12681
    },
    {
      "epoch": 1.659524004252883,
      "grad_norm": 9.993975639343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20975621044635773,
      "logits/rejected": -0.2581876814365387,
      "logps/chosen": -410.3039245605469,
      "logps/rejected": -417.34600830078125,
      "loss": 0.5351,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1118289232254028,
      "rewards/margins": 0.9885444641113281,
      "rewards/rejected": -2.1003735065460205,
      "step": 12682
    },
    {
      "epoch": 1.6596548621902347,
      "grad_norm": 7.294676303863525,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1558009684085846,
      "logits/rejected": -0.24165768921375275,
      "logps/chosen": -696.23486328125,
      "logps/rejected": -649.366455078125,
      "loss": 0.3313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7979530096054077,
      "rewards/margins": 1.5510555505752563,
      "rewards/rejected": -2.349008560180664,
      "step": 12683
    },
    {
      "epoch": 1.6597857201275865,
      "grad_norm": 5.967861652374268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15191352367401123,
      "logits/rejected": -0.27767500281333923,
      "logps/chosen": -580.6549072265625,
      "logps/rejected": -487.9734191894531,
      "loss": 0.2593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0794808864593506,
      "rewards/margins": 1.6028082370758057,
      "rewards/rejected": -2.6822891235351562,
      "step": 12684
    },
    {
      "epoch": 1.659916578064938,
      "grad_norm": 5.187668323516846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5295571684837341,
      "logits/rejected": -0.732012927532196,
      "logps/chosen": -301.2230529785156,
      "logps/rejected": -555.4449462890625,
      "loss": 0.3034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6753298044204712,
      "rewards/margins": 3.194535732269287,
      "rewards/rejected": -3.869865894317627,
      "step": 12685
    },
    {
      "epoch": 1.6600474360022899,
      "grad_norm": 8.104283332824707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009343376383185387,
      "logits/rejected": -0.027147889137268066,
      "logps/chosen": -451.7536315917969,
      "logps/rejected": -420.6754455566406,
      "loss": 0.3102,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3284740447998047,
      "rewards/margins": 2.1810975074768066,
      "rewards/rejected": -2.5095715522766113,
      "step": 12686
    },
    {
      "epoch": 1.6601782939396417,
      "grad_norm": 14.172130584716797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1215057224035263,
      "logits/rejected": -0.373701274394989,
      "logps/chosen": -429.40936279296875,
      "logps/rejected": -457.01055908203125,
      "loss": 0.8549,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.6123721599578857,
      "rewards/margins": 0.34734946489334106,
      "rewards/rejected": -2.959721803665161,
      "step": 12687
    },
    {
      "epoch": 1.6603091518769935,
      "grad_norm": 11.383607864379883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43092864751815796,
      "logits/rejected": -0.41091272234916687,
      "logps/chosen": -297.2007141113281,
      "logps/rejected": -318.2073974609375,
      "loss": 0.8154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8573048114776611,
      "rewards/margins": 0.2153446227312088,
      "rewards/rejected": -2.0726494789123535,
      "step": 12688
    },
    {
      "epoch": 1.6604400098143453,
      "grad_norm": 12.55634880065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4606700837612152,
      "logits/rejected": -0.47144433856010437,
      "logps/chosen": -371.6740417480469,
      "logps/rejected": -497.45013427734375,
      "loss": 0.671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3622889518737793,
      "rewards/margins": 0.5227491855621338,
      "rewards/rejected": -1.885038137435913,
      "step": 12689
    },
    {
      "epoch": 1.660570867751697,
      "grad_norm": 7.198885440826416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17542356252670288,
      "logits/rejected": -0.5160572528839111,
      "logps/chosen": -400.4659118652344,
      "logps/rejected": -524.0845336914062,
      "loss": 0.2827,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13366547226905823,
      "rewards/margins": 2.067460536956787,
      "rewards/rejected": -2.2011260986328125,
      "step": 12690
    },
    {
      "epoch": 1.6607017256890488,
      "grad_norm": 13.442704200744629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19972606003284454,
      "logits/rejected": -0.2898916006088257,
      "logps/chosen": -301.5815734863281,
      "logps/rejected": -562.3958740234375,
      "loss": 0.5716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6261317729949951,
      "rewards/margins": 1.0968139171600342,
      "rewards/rejected": -2.7229456901550293,
      "step": 12691
    },
    {
      "epoch": 1.6608325836264006,
      "grad_norm": 5.036990165710449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1616930365562439,
      "logits/rejected": -0.27161550521850586,
      "logps/chosen": -627.709228515625,
      "logps/rejected": -473.2431640625,
      "loss": 0.1952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8598420023918152,
      "rewards/margins": 2.1222805976867676,
      "rewards/rejected": -2.9821224212646484,
      "step": 12692
    },
    {
      "epoch": 1.6609634415637524,
      "grad_norm": 9.782328605651855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32475337386131287,
      "logits/rejected": -0.22962437570095062,
      "logps/chosen": -438.8436279296875,
      "logps/rejected": -424.33001708984375,
      "loss": 0.495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0010778903961182,
      "rewards/margins": 0.744553804397583,
      "rewards/rejected": -1.745631456375122,
      "step": 12693
    },
    {
      "epoch": 1.6610942995011042,
      "grad_norm": 5.853771686553955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3972247838973999,
      "logits/rejected": -0.2587956190109253,
      "logps/chosen": -535.706298828125,
      "logps/rejected": -411.968994140625,
      "loss": 0.2331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0535533428192139,
      "rewards/margins": 1.536313772201538,
      "rewards/rejected": -2.589867115020752,
      "step": 12694
    },
    {
      "epoch": 1.661225157438456,
      "grad_norm": 9.18360710144043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5027268528938293,
      "logits/rejected": -0.5710530281066895,
      "logps/chosen": -413.1356201171875,
      "logps/rejected": -442.0390625,
      "loss": 0.3826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1812442541122437,
      "rewards/margins": 1.5008015632629395,
      "rewards/rejected": -2.6820459365844727,
      "step": 12695
    },
    {
      "epoch": 1.6613560153758078,
      "grad_norm": 8.479262351989746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04354608803987503,
      "logits/rejected": -0.2965414822101593,
      "logps/chosen": -437.5784912109375,
      "logps/rejected": -435.4466552734375,
      "loss": 0.4367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.683242678642273,
      "rewards/margins": 1.239990234375,
      "rewards/rejected": -2.9232330322265625,
      "step": 12696
    },
    {
      "epoch": 1.6614868733131594,
      "grad_norm": 11.411971092224121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22934065759181976,
      "logits/rejected": -0.1969178318977356,
      "logps/chosen": -443.24749755859375,
      "logps/rejected": -389.0328369140625,
      "loss": 0.5792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1842362880706787,
      "rewards/margins": 0.8725082874298096,
      "rewards/rejected": -2.0567445755004883,
      "step": 12697
    },
    {
      "epoch": 1.6616177312505112,
      "grad_norm": 12.08512020111084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07238072156906128,
      "logits/rejected": -0.2015809714794159,
      "logps/chosen": -640.2678833007812,
      "logps/rejected": -596.0374145507812,
      "loss": 0.5326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6818908452987671,
      "rewards/margins": 1.5810673236846924,
      "rewards/rejected": -2.262958288192749,
      "step": 12698
    },
    {
      "epoch": 1.661748589187863,
      "grad_norm": 9.14267349243164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6526218056678772,
      "logits/rejected": -0.7177069187164307,
      "logps/chosen": -293.57318115234375,
      "logps/rejected": -296.15020751953125,
      "loss": 0.5422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.562450885772705,
      "rewards/margins": 0.7138268947601318,
      "rewards/rejected": -2.276278018951416,
      "step": 12699
    },
    {
      "epoch": 1.6618794471252147,
      "grad_norm": 11.909870147705078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22357748448848724,
      "logits/rejected": -0.35479578375816345,
      "logps/chosen": -457.2020263671875,
      "logps/rejected": -348.2599792480469,
      "loss": 0.7753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.732029676437378,
      "rewards/margins": 0.27277305722236633,
      "rewards/rejected": -2.004802703857422,
      "step": 12700
    },
    {
      "epoch": 1.6620103050625663,
      "grad_norm": 3.9106333255767822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15576975047588348,
      "logits/rejected": -0.07959102094173431,
      "logps/chosen": -434.6546936035156,
      "logps/rejected": -593.11865234375,
      "loss": 0.2006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5647627115249634,
      "rewards/margins": 1.7868361473083496,
      "rewards/rejected": -2.3515989780426025,
      "step": 12701
    },
    {
      "epoch": 1.662141162999918,
      "grad_norm": 10.503205299377441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20522509515285492,
      "logits/rejected": -0.17827481031417847,
      "logps/chosen": -736.9363403320312,
      "logps/rejected": -557.0635986328125,
      "loss": 0.5809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2962837219238281,
      "rewards/margins": 1.5349124670028687,
      "rewards/rejected": -1.8311963081359863,
      "step": 12702
    },
    {
      "epoch": 1.66227202093727,
      "grad_norm": 4.428946495056152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33459362387657166,
      "logits/rejected": -0.5961870551109314,
      "logps/chosen": -379.76092529296875,
      "logps/rejected": -396.4307861328125,
      "loss": 0.3071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2560378909111023,
      "rewards/margins": 2.474202871322632,
      "rewards/rejected": -2.730240821838379,
      "step": 12703
    },
    {
      "epoch": 1.6624028788746217,
      "grad_norm": 10.097070693969727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22737203538417816,
      "logits/rejected": -0.39751356840133667,
      "logps/chosen": -562.8551635742188,
      "logps/rejected": -651.36962890625,
      "loss": 0.3453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.412000298500061,
      "rewards/margins": 1.9158161878585815,
      "rewards/rejected": -3.3278164863586426,
      "step": 12704
    },
    {
      "epoch": 1.6625337368119735,
      "grad_norm": 9.278915405273438,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025943616405129433,
      "logits/rejected": -0.03245963156223297,
      "logps/chosen": -291.1591491699219,
      "logps/rejected": -339.27642822265625,
      "loss": 0.5639,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5073747634887695,
      "rewards/margins": 0.7781112790107727,
      "rewards/rejected": -1.285486102104187,
      "step": 12705
    },
    {
      "epoch": 1.6626645947493253,
      "grad_norm": 13.884222030639648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14489498734474182,
      "logits/rejected": -0.20256638526916504,
      "logps/chosen": -494.5145263671875,
      "logps/rejected": -429.275390625,
      "loss": 0.9678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7134028077125549,
      "rewards/margins": 1.140154242515564,
      "rewards/rejected": -1.8535571098327637,
      "step": 12706
    },
    {
      "epoch": 1.662795452686677,
      "grad_norm": 4.828352928161621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20282498002052307,
      "logits/rejected": -0.2990961968898773,
      "logps/chosen": -401.2584228515625,
      "logps/rejected": -346.8879699707031,
      "loss": 0.1978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1262054443359375,
      "rewards/margins": 1.9360778331756592,
      "rewards/rejected": -1.8098723888397217,
      "step": 12707
    },
    {
      "epoch": 1.6629263106240288,
      "grad_norm": 7.216242790222168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6749439835548401,
      "logits/rejected": -0.24431128799915314,
      "logps/chosen": -348.4924621582031,
      "logps/rejected": -423.0018005371094,
      "loss": 0.4269,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40822258591651917,
      "rewards/margins": 0.8666425347328186,
      "rewards/rejected": -1.2748652696609497,
      "step": 12708
    },
    {
      "epoch": 1.6630571685613806,
      "grad_norm": 8.328264236450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17886210978031158,
      "logits/rejected": -0.23668980598449707,
      "logps/chosen": -662.4228515625,
      "logps/rejected": -591.7388916015625,
      "loss": 0.3303,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2610597610473633,
      "rewards/margins": 1.2797785997390747,
      "rewards/rejected": -1.540838360786438,
      "step": 12709
    },
    {
      "epoch": 1.6631880264987324,
      "grad_norm": 7.005903244018555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13363005220890045,
      "logits/rejected": -0.13867595791816711,
      "logps/chosen": -332.44451904296875,
      "logps/rejected": -310.0067138671875,
      "loss": 0.3358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9004802703857422,
      "rewards/margins": 0.9683166742324829,
      "rewards/rejected": -2.8687968254089355,
      "step": 12710
    },
    {
      "epoch": 1.6633188844360842,
      "grad_norm": 5.049031734466553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07168856263160706,
      "logits/rejected": -0.18517577648162842,
      "logps/chosen": -644.6744384765625,
      "logps/rejected": -527.8311767578125,
      "loss": 0.1984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2010326385498047,
      "rewards/margins": 1.7486536502838135,
      "rewards/rejected": -1.9496862888336182,
      "step": 12711
    },
    {
      "epoch": 1.6634497423734358,
      "grad_norm": 6.486782073974609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41333407163619995,
      "logits/rejected": -0.4216468334197998,
      "logps/chosen": -361.93560791015625,
      "logps/rejected": -282.09185791015625,
      "loss": 0.3898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9160588979721069,
      "rewards/margins": 0.9773738384246826,
      "rewards/rejected": -1.8934327363967896,
      "step": 12712
    },
    {
      "epoch": 1.6635806003107876,
      "grad_norm": 12.94322395324707,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007504038512706757,
      "logits/rejected": 0.03938905522227287,
      "logps/chosen": -383.45941162109375,
      "logps/rejected": -583.4172973632812,
      "loss": 0.8135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8310285806655884,
      "rewards/margins": 0.5781545639038086,
      "rewards/rejected": -1.4091830253601074,
      "step": 12713
    },
    {
      "epoch": 1.6637114582481394,
      "grad_norm": 9.631572723388672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5511044859886169,
      "logits/rejected": -0.4717841148376465,
      "logps/chosen": -557.941650390625,
      "logps/rejected": -370.58856201171875,
      "loss": 0.3917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8831435441970825,
      "rewards/margins": 1.03600013256073,
      "rewards/rejected": -2.9191436767578125,
      "step": 12714
    },
    {
      "epoch": 1.6638423161854912,
      "grad_norm": 5.762731075286865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1011311486363411,
      "logits/rejected": -0.19978411495685577,
      "logps/chosen": -494.580810546875,
      "logps/rejected": -314.1147766113281,
      "loss": 0.2764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07987270504236221,
      "rewards/margins": 1.3086901903152466,
      "rewards/rejected": -1.3885630369186401,
      "step": 12715
    },
    {
      "epoch": 1.6639731741228427,
      "grad_norm": 7.703710079193115,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08839914202690125,
      "logits/rejected": 0.09223411977291107,
      "logps/chosen": -322.8914794921875,
      "logps/rejected": -450.56256103515625,
      "loss": 0.2463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2682781517505646,
      "rewards/margins": 2.947449207305908,
      "rewards/rejected": -3.2157273292541504,
      "step": 12716
    },
    {
      "epoch": 1.6641040320601945,
      "grad_norm": 4.785571575164795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19864831864833832,
      "logits/rejected": -0.2623646855354309,
      "logps/chosen": -531.8801879882812,
      "logps/rejected": -473.35162353515625,
      "loss": 0.2094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21188277006149292,
      "rewards/margins": 2.3155763149261475,
      "rewards/rejected": -2.527459144592285,
      "step": 12717
    },
    {
      "epoch": 1.6642348899975463,
      "grad_norm": 6.11516809463501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5059606432914734,
      "logits/rejected": -0.34412020444869995,
      "logps/chosen": -408.10076904296875,
      "logps/rejected": -322.3790588378906,
      "loss": 0.2551,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28975632786750793,
      "rewards/margins": 1.8409919738769531,
      "rewards/rejected": -2.1307482719421387,
      "step": 12718
    },
    {
      "epoch": 1.664365747934898,
      "grad_norm": 8.759510040283203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018280029296875,
      "logits/rejected": -0.054820701479911804,
      "logps/chosen": -1029.3868408203125,
      "logps/rejected": -686.947021484375,
      "loss": 0.4029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1550643891096115,
      "rewards/margins": 1.3266348838806152,
      "rewards/rejected": -1.1715705394744873,
      "step": 12719
    },
    {
      "epoch": 1.66449660587225,
      "grad_norm": 9.905630111694336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1470765322446823,
      "logits/rejected": -0.15531551837921143,
      "logps/chosen": -316.940185546875,
      "logps/rejected": -354.21661376953125,
      "loss": 0.7174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9992722868919373,
      "rewards/margins": 0.3262266218662262,
      "rewards/rejected": -1.3254989385604858,
      "step": 12720
    },
    {
      "epoch": 1.6646274638096017,
      "grad_norm": 9.494893074035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1580408215522766,
      "logits/rejected": -0.09219151735305786,
      "logps/chosen": -369.6461486816406,
      "logps/rejected": -578.502685546875,
      "loss": 0.5136,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09265999495983124,
      "rewards/margins": 1.3136268854141235,
      "rewards/rejected": -1.4062869548797607,
      "step": 12721
    },
    {
      "epoch": 1.6647583217469535,
      "grad_norm": 6.514061450958252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0567571222782135,
      "logits/rejected": -0.24812890589237213,
      "logps/chosen": -535.1624145507812,
      "logps/rejected": -428.1795654296875,
      "loss": 0.2254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.460791826248169,
      "rewards/margins": 1.9582500457763672,
      "rewards/rejected": -3.419041633605957,
      "step": 12722
    },
    {
      "epoch": 1.6648891796843053,
      "grad_norm": 6.279077053070068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5106584429740906,
      "logits/rejected": -0.2317383885383606,
      "logps/chosen": -465.69775390625,
      "logps/rejected": -595.3787841796875,
      "loss": 0.2064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.863728940486908,
      "rewards/margins": 2.5932226181030273,
      "rewards/rejected": -3.456951379776001,
      "step": 12723
    },
    {
      "epoch": 1.665020037621657,
      "grad_norm": 7.105111122131348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5599369406700134,
      "logits/rejected": -0.40949657559394836,
      "logps/chosen": -391.7679443359375,
      "logps/rejected": -424.1187744140625,
      "loss": 0.3106,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22150860726833344,
      "rewards/margins": 1.5920099020004272,
      "rewards/rejected": -1.8135185241699219,
      "step": 12724
    },
    {
      "epoch": 1.6651508955590089,
      "grad_norm": 9.441431045532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06521116942167282,
      "logits/rejected": -0.02634109929203987,
      "logps/chosen": -450.9185485839844,
      "logps/rejected": -429.1710205078125,
      "loss": 0.417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.927921712398529,
      "rewards/margins": 1.3343178033828735,
      "rewards/rejected": -2.262239456176758,
      "step": 12725
    },
    {
      "epoch": 1.6652817534963607,
      "grad_norm": 6.749337673187256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13750365376472473,
      "logits/rejected": -0.16205626726150513,
      "logps/chosen": -660.3948974609375,
      "logps/rejected": -547.0089721679688,
      "loss": 0.344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7628803253173828,
      "rewards/margins": 1.2846593856811523,
      "rewards/rejected": -2.047539710998535,
      "step": 12726
    },
    {
      "epoch": 1.6654126114337124,
      "grad_norm": 8.611064910888672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22339364886283875,
      "logits/rejected": -0.4929303824901581,
      "logps/chosen": -456.4847412109375,
      "logps/rejected": -430.5726013183594,
      "loss": 0.4682,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8754515051841736,
      "rewards/margins": 1.7620434761047363,
      "rewards/rejected": -2.6374948024749756,
      "step": 12727
    },
    {
      "epoch": 1.665543469371064,
      "grad_norm": 6.943753719329834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38494959473609924,
      "logits/rejected": -0.3895583152770996,
      "logps/chosen": -579.7626953125,
      "logps/rejected": -463.8844909667969,
      "loss": 0.3022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0853021144866943,
      "rewards/margins": 1.6826796531677246,
      "rewards/rejected": -2.767981767654419,
      "step": 12728
    },
    {
      "epoch": 1.6656743273084158,
      "grad_norm": 9.001044273376465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3231811225414276,
      "logits/rejected": -0.4039936661720276,
      "logps/chosen": -400.2158508300781,
      "logps/rejected": -424.2433776855469,
      "loss": 0.2681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06163233518600464,
      "rewards/margins": 2.2441418170928955,
      "rewards/rejected": -2.182509660720825,
      "step": 12729
    },
    {
      "epoch": 1.6658051852457676,
      "grad_norm": 12.018585205078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6336970329284668,
      "logits/rejected": -0.378410667181015,
      "logps/chosen": -426.20977783203125,
      "logps/rejected": -339.287109375,
      "loss": 0.7372,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7889885902404785,
      "rewards/margins": 0.7312438488006592,
      "rewards/rejected": -2.5202324390411377,
      "step": 12730
    },
    {
      "epoch": 1.6659360431831192,
      "grad_norm": 12.907797813415527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3220435678958893,
      "logits/rejected": -0.1925622820854187,
      "logps/chosen": -437.852783203125,
      "logps/rejected": -351.8536682128906,
      "loss": 0.7093,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0372852087020874,
      "rewards/margins": 0.6463603973388672,
      "rewards/rejected": -1.6836457252502441,
      "step": 12731
    },
    {
      "epoch": 1.666066901120471,
      "grad_norm": 7.231508731842041,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07001282274723053,
      "logits/rejected": -0.21868599951267242,
      "logps/chosen": -672.8734741210938,
      "logps/rejected": -597.3487548828125,
      "loss": 0.1943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3117990791797638,
      "rewards/margins": 2.6393463611602783,
      "rewards/rejected": -2.951145648956299,
      "step": 12732
    },
    {
      "epoch": 1.6661977590578227,
      "grad_norm": 10.64985466003418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1938447654247284,
      "logits/rejected": -0.16452769935131073,
      "logps/chosen": -457.96673583984375,
      "logps/rejected": -369.3080139160156,
      "loss": 0.692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.871515154838562,
      "rewards/margins": 0.6991609334945679,
      "rewards/rejected": -2.57067608833313,
      "step": 12733
    },
    {
      "epoch": 1.6663286169951745,
      "grad_norm": 3.8805813789367676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09131529927253723,
      "logits/rejected": -0.3068350553512573,
      "logps/chosen": -420.82000732421875,
      "logps/rejected": -316.7449645996094,
      "loss": 0.1732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6845380663871765,
      "rewards/margins": 2.194586753845215,
      "rewards/rejected": -2.879124879837036,
      "step": 12734
    },
    {
      "epoch": 1.6664594749325263,
      "grad_norm": 7.0179243087768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2416663020849228,
      "logits/rejected": -0.18116748332977295,
      "logps/chosen": -545.7399291992188,
      "logps/rejected": -491.79736328125,
      "loss": 0.2855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13948440551757812,
      "rewards/margins": 2.2481436729431152,
      "rewards/rejected": -2.3876283168792725,
      "step": 12735
    },
    {
      "epoch": 1.6665903328698781,
      "grad_norm": 11.200228691101074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38064953684806824,
      "logits/rejected": -0.45210376381874084,
      "logps/chosen": -375.66827392578125,
      "logps/rejected": -339.9497985839844,
      "loss": 0.4733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3204509019851685,
      "rewards/margins": 1.2250585556030273,
      "rewards/rejected": -2.5455095767974854,
      "step": 12736
    },
    {
      "epoch": 1.66672119080723,
      "grad_norm": 7.5728230476379395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22233517467975616,
      "logits/rejected": -0.2327984869480133,
      "logps/chosen": -429.6661376953125,
      "logps/rejected": -437.3363952636719,
      "loss": 0.2507,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8413205742835999,
      "rewards/margins": 1.9137459993362427,
      "rewards/rejected": -2.7550666332244873,
      "step": 12737
    },
    {
      "epoch": 1.6668520487445817,
      "grad_norm": 9.59073543548584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25245019793510437,
      "logits/rejected": -0.38399451971054077,
      "logps/chosen": -448.6004943847656,
      "logps/rejected": -547.51904296875,
      "loss": 0.4878,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8900211453437805,
      "rewards/margins": 1.881476640701294,
      "rewards/rejected": -2.7714974880218506,
      "step": 12738
    },
    {
      "epoch": 1.6669829066819335,
      "grad_norm": 12.611288070678711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2810294032096863,
      "logits/rejected": -0.29495561122894287,
      "logps/chosen": -489.8583984375,
      "logps/rejected": -410.5768737792969,
      "loss": 0.7321,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9485061764717102,
      "rewards/margins": 0.001800544559955597,
      "rewards/rejected": -0.95030677318573,
      "step": 12739
    },
    {
      "epoch": 1.6671137646192853,
      "grad_norm": 6.9297709465026855,
      "learning_rate": 5e-06,
      "logits/chosen": 0.048443689942359924,
      "logits/rejected": 0.24459339678287506,
      "logps/chosen": -431.3730773925781,
      "logps/rejected": -425.98272705078125,
      "loss": 0.3746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8980394601821899,
      "rewards/margins": 1.3555970191955566,
      "rewards/rejected": -2.253636360168457,
      "step": 12740
    },
    {
      "epoch": 1.667244622556637,
      "grad_norm": 12.759456634521484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3088589310646057,
      "logits/rejected": -0.3340032398700714,
      "logps/chosen": -737.9078369140625,
      "logps/rejected": -529.0357055664062,
      "loss": 0.7795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8854705691337585,
      "rewards/margins": 0.6132656335830688,
      "rewards/rejected": -1.4987361431121826,
      "step": 12741
    },
    {
      "epoch": 1.6673754804939889,
      "grad_norm": 13.04110336303711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5997379422187805,
      "logits/rejected": -0.612363874912262,
      "logps/chosen": -407.7489318847656,
      "logps/rejected": -321.37451171875,
      "loss": 0.6395,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3081371784210205,
      "rewards/margins": 0.7250651121139526,
      "rewards/rejected": -2.0332021713256836,
      "step": 12742
    },
    {
      "epoch": 1.6675063384313404,
      "grad_norm": 4.247158050537109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2583528161048889,
      "logits/rejected": -0.24815435707569122,
      "logps/chosen": -478.947021484375,
      "logps/rejected": -518.4445190429688,
      "loss": 0.1558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20735891163349152,
      "rewards/margins": 2.6310536861419678,
      "rewards/rejected": -2.423694610595703,
      "step": 12743
    },
    {
      "epoch": 1.6676371963686922,
      "grad_norm": 10.550349235534668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.332760751247406,
      "logits/rejected": -0.4051210582256317,
      "logps/chosen": -342.4665222167969,
      "logps/rejected": -371.85052490234375,
      "loss": 0.7883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9615726470947266,
      "rewards/margins": 0.06765693426132202,
      "rewards/rejected": -1.0292296409606934,
      "step": 12744
    },
    {
      "epoch": 1.667768054306044,
      "grad_norm": 4.910179138183594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2510819733142853,
      "logits/rejected": -0.3524235188961029,
      "logps/chosen": -515.0120849609375,
      "logps/rejected": -450.371337890625,
      "loss": 0.1532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8543967008590698,
      "rewards/margins": 2.1518313884735107,
      "rewards/rejected": -3.00622820854187,
      "step": 12745
    },
    {
      "epoch": 1.6678989122433958,
      "grad_norm": 13.941730499267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15240982174873352,
      "logits/rejected": -0.12859371304512024,
      "logps/chosen": -347.9411315917969,
      "logps/rejected": -325.11810302734375,
      "loss": 0.9713,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5349478125572205,
      "rewards/margins": 0.3359760642051697,
      "rewards/rejected": -0.8709238171577454,
      "step": 12746
    },
    {
      "epoch": 1.6680297701807474,
      "grad_norm": 7.0915937423706055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23994043469429016,
      "logits/rejected": -0.46801531314849854,
      "logps/chosen": -656.1644897460938,
      "logps/rejected": -738.5245361328125,
      "loss": 0.2705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6383897662162781,
      "rewards/margins": 1.8556292057037354,
      "rewards/rejected": -2.4940192699432373,
      "step": 12747
    },
    {
      "epoch": 1.6681606281180992,
      "grad_norm": 7.647133827209473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.083967424929142,
      "logits/rejected": -0.07266761362552643,
      "logps/chosen": -317.807373046875,
      "logps/rejected": -398.441162109375,
      "loss": 0.3619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8962174654006958,
      "rewards/margins": 1.4717919826507568,
      "rewards/rejected": -2.368009328842163,
      "step": 12748
    },
    {
      "epoch": 1.668291486055451,
      "grad_norm": 7.820015907287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5373290181159973,
      "logits/rejected": -0.31231689453125,
      "logps/chosen": -389.38555908203125,
      "logps/rejected": -511.95220947265625,
      "loss": 0.5014,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8279109597206116,
      "rewards/margins": 1.2884621620178223,
      "rewards/rejected": -2.116373062133789,
      "step": 12749
    },
    {
      "epoch": 1.6684223439928028,
      "grad_norm": 6.345727920532227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34136343002319336,
      "logits/rejected": -0.1694958359003067,
      "logps/chosen": -285.6226501464844,
      "logps/rejected": -434.09002685546875,
      "loss": 0.2953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4084757566452026,
      "rewards/margins": 1.8407841920852661,
      "rewards/rejected": -3.2492599487304688,
      "step": 12750
    },
    {
      "epoch": 1.6685532019301546,
      "grad_norm": 7.463007926940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26891711354255676,
      "logits/rejected": -0.42358824610710144,
      "logps/chosen": -548.5081787109375,
      "logps/rejected": -514.8942260742188,
      "loss": 0.2507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.012366876006126404,
      "rewards/margins": 1.6554431915283203,
      "rewards/rejected": -1.667810082435608,
      "step": 12751
    },
    {
      "epoch": 1.6686840598675063,
      "grad_norm": 8.71101188659668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.041340235620737076,
      "logits/rejected": -0.14425958693027496,
      "logps/chosen": -525.452392578125,
      "logps/rejected": -632.1043090820312,
      "loss": 0.3585,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1816596984863281,
      "rewards/margins": 1.5152515172958374,
      "rewards/rejected": -2.696911334991455,
      "step": 12752
    },
    {
      "epoch": 1.6688149178048581,
      "grad_norm": 6.4344096183776855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04457675665616989,
      "logits/rejected": -0.11162099242210388,
      "logps/chosen": -660.4119873046875,
      "logps/rejected": -691.047119140625,
      "loss": 0.2952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2257641553878784,
      "rewards/margins": 1.9103032350540161,
      "rewards/rejected": -3.1360673904418945,
      "step": 12753
    },
    {
      "epoch": 1.66894577574221,
      "grad_norm": 12.921375274658203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23778803646564484,
      "logits/rejected": -0.3086068034172058,
      "logps/chosen": -523.4737548828125,
      "logps/rejected": -402.5596618652344,
      "loss": 0.9249,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2749816179275513,
      "rewards/margins": 0.034085631370544434,
      "rewards/rejected": -1.3090671300888062,
      "step": 12754
    },
    {
      "epoch": 1.6690766336795617,
      "grad_norm": 13.665399551391602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4989561438560486,
      "logits/rejected": -0.1905713677406311,
      "logps/chosen": -529.0642700195312,
      "logps/rejected": -545.40478515625,
      "loss": 0.5086,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2399901151657104,
      "rewards/margins": 1.092419981956482,
      "rewards/rejected": -2.3324100971221924,
      "step": 12755
    },
    {
      "epoch": 1.6692074916169135,
      "grad_norm": 5.04881477355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0597393661737442,
      "logits/rejected": -0.31051695346832275,
      "logps/chosen": -541.076416015625,
      "logps/rejected": -574.37890625,
      "loss": 0.1632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1700117588043213,
      "rewards/margins": 2.5841140747070312,
      "rewards/rejected": -3.7541255950927734,
      "step": 12756
    },
    {
      "epoch": 1.6693383495542653,
      "grad_norm": 5.47047758102417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5285916924476624,
      "logits/rejected": -0.5797739624977112,
      "logps/chosen": -390.347900390625,
      "logps/rejected": -461.2361755371094,
      "loss": 0.2624,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18677271902561188,
      "rewards/margins": 2.2977418899536133,
      "rewards/rejected": -2.4845149517059326,
      "step": 12757
    },
    {
      "epoch": 1.6694692074916169,
      "grad_norm": 8.57578182220459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40661555528640747,
      "logits/rejected": -0.13938890397548676,
      "logps/chosen": -299.16119384765625,
      "logps/rejected": -391.5133056640625,
      "loss": 0.4462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0407053232192993,
      "rewards/margins": 1.1459234952926636,
      "rewards/rejected": -2.186628818511963,
      "step": 12758
    },
    {
      "epoch": 1.6696000654289687,
      "grad_norm": 7.168542861938477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06974214315414429,
      "logits/rejected": -0.07767864316701889,
      "logps/chosen": -399.7618713378906,
      "logps/rejected": -496.2762451171875,
      "loss": 0.3081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40484005212783813,
      "rewards/margins": 2.107701301574707,
      "rewards/rejected": -2.5125415325164795,
      "step": 12759
    },
    {
      "epoch": 1.6697309233663205,
      "grad_norm": 10.725247383117676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15207995474338531,
      "logits/rejected": -0.23257897794246674,
      "logps/chosen": -456.2696838378906,
      "logps/rejected": -485.61810302734375,
      "loss": 0.6297,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7791521549224854,
      "rewards/margins": 1.0121673345565796,
      "rewards/rejected": -1.791319489479065,
      "step": 12760
    },
    {
      "epoch": 1.6698617813036722,
      "grad_norm": 8.484128952026367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09468945860862732,
      "logits/rejected": -0.3101872205734253,
      "logps/chosen": -536.5165405273438,
      "logps/rejected": -412.4434509277344,
      "loss": 0.372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31405583024024963,
      "rewards/margins": 1.3605372905731201,
      "rewards/rejected": -1.674593210220337,
      "step": 12761
    },
    {
      "epoch": 1.6699926392410238,
      "grad_norm": 10.344926834106445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1423165500164032,
      "logits/rejected": -0.3700520992279053,
      "logps/chosen": -583.529052734375,
      "logps/rejected": -466.4024658203125,
      "loss": 0.4659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6628923416137695,
      "rewards/margins": 1.9084101915359497,
      "rewards/rejected": -2.571302652359009,
      "step": 12762
    },
    {
      "epoch": 1.6701234971783756,
      "grad_norm": 9.005354881286621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009298548102378845,
      "logits/rejected": -0.15407125651836395,
      "logps/chosen": -343.1837158203125,
      "logps/rejected": -439.60296630859375,
      "loss": 0.7688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3248045444488525,
      "rewards/margins": 0.32936134934425354,
      "rewards/rejected": -1.6541659832000732,
      "step": 12763
    },
    {
      "epoch": 1.6702543551157274,
      "grad_norm": 15.384866714477539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11885397881269455,
      "logits/rejected": -0.2163861244916916,
      "logps/chosen": -395.6398620605469,
      "logps/rejected": -444.51580810546875,
      "loss": 0.8546,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4162591695785522,
      "rewards/margins": -0.16142094135284424,
      "rewards/rejected": -1.254838228225708,
      "step": 12764
    },
    {
      "epoch": 1.6703852130530792,
      "grad_norm": 11.355351448059082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037596385926008224,
      "logits/rejected": 0.0732228010892868,
      "logps/chosen": -441.3990478515625,
      "logps/rejected": -442.8299865722656,
      "loss": 0.7476,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6360385417938232,
      "rewards/margins": 0.13343890011310577,
      "rewards/rejected": -0.7694774270057678,
      "step": 12765
    },
    {
      "epoch": 1.670516070990431,
      "grad_norm": 8.033563613891602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2475314885377884,
      "logits/rejected": -0.4162459969520569,
      "logps/chosen": -324.7674560546875,
      "logps/rejected": -397.89703369140625,
      "loss": 0.426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6367766857147217,
      "rewards/margins": 1.4822394847869873,
      "rewards/rejected": -2.119016170501709,
      "step": 12766
    },
    {
      "epoch": 1.6706469289277828,
      "grad_norm": 10.890469551086426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10956132411956787,
      "logits/rejected": -0.1342964917421341,
      "logps/chosen": -772.0478515625,
      "logps/rejected": -443.49359130859375,
      "loss": 0.4286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10125847160816193,
      "rewards/margins": 1.3787142038345337,
      "rewards/rejected": -1.4799726009368896,
      "step": 12767
    },
    {
      "epoch": 1.6707777868651346,
      "grad_norm": 5.619182586669922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48783352971076965,
      "logits/rejected": -0.4602983891963959,
      "logps/chosen": -298.64581298828125,
      "logps/rejected": -347.2022705078125,
      "loss": 0.2521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22488442063331604,
      "rewards/margins": 2.325387716293335,
      "rewards/rejected": -2.550271987915039,
      "step": 12768
    },
    {
      "epoch": 1.6709086448024864,
      "grad_norm": 5.194945335388184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046018294990062714,
      "logits/rejected": -0.21641571819782257,
      "logps/chosen": -732.683349609375,
      "logps/rejected": -518.7926635742188,
      "loss": 0.2403,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38815879821777344,
      "rewards/margins": 2.094844102859497,
      "rewards/rejected": -2.4830026626586914,
      "step": 12769
    },
    {
      "epoch": 1.6710395027398381,
      "grad_norm": 6.067774295806885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25299692153930664,
      "logits/rejected": -0.291176438331604,
      "logps/chosen": -478.53436279296875,
      "logps/rejected": -439.78192138671875,
      "loss": 0.2923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48609888553619385,
      "rewards/margins": 1.6774311065673828,
      "rewards/rejected": -2.163529872894287,
      "step": 12770
    },
    {
      "epoch": 1.67117036067719,
      "grad_norm": 8.62109661102295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23055925965309143,
      "logits/rejected": -0.42148590087890625,
      "logps/chosen": -549.4800415039062,
      "logps/rejected": -589.5402221679688,
      "loss": 0.3161,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1967111825942993,
      "rewards/margins": 2.280968427658081,
      "rewards/rejected": -3.4776792526245117,
      "step": 12771
    },
    {
      "epoch": 1.6713012186145417,
      "grad_norm": 11.381661415100098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5669810175895691,
      "logits/rejected": -0.3809009790420532,
      "logps/chosen": -500.9505615234375,
      "logps/rejected": -404.5592956542969,
      "loss": 0.4464,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1252182722091675,
      "rewards/margins": 1.6216964721679688,
      "rewards/rejected": -2.746914863586426,
      "step": 12772
    },
    {
      "epoch": 1.6714320765518933,
      "grad_norm": 4.868966579437256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30340415239334106,
      "logits/rejected": -0.06390649825334549,
      "logps/chosen": -465.2906799316406,
      "logps/rejected": -572.0726318359375,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4642027020454407,
      "rewards/margins": 2.7193312644958496,
      "rewards/rejected": -3.1835341453552246,
      "step": 12773
    },
    {
      "epoch": 1.671562934489245,
      "grad_norm": 9.031854629516602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22994288802146912,
      "logits/rejected": -0.23439909517765045,
      "logps/chosen": -426.3450012207031,
      "logps/rejected": -429.1759033203125,
      "loss": 0.5523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8872629404067993,
      "rewards/margins": 0.8285097479820251,
      "rewards/rejected": -1.7157726287841797,
      "step": 12774
    },
    {
      "epoch": 1.6716937924265969,
      "grad_norm": 4.902348041534424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3966595530509949,
      "logits/rejected": -0.6307924389839172,
      "logps/chosen": -471.0934753417969,
      "logps/rejected": -409.1904602050781,
      "loss": 0.1771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7091941833496094,
      "rewards/margins": 2.112396240234375,
      "rewards/rejected": -2.8215906620025635,
      "step": 12775
    },
    {
      "epoch": 1.6718246503639487,
      "grad_norm": 10.104480743408203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4097689092159271,
      "logits/rejected": -0.3538797199726105,
      "logps/chosen": -415.78515625,
      "logps/rejected": -436.5542297363281,
      "loss": 0.5904,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6113373041152954,
      "rewards/margins": 1.5086884498596191,
      "rewards/rejected": -2.120025634765625,
      "step": 12776
    },
    {
      "epoch": 1.6719555083013002,
      "grad_norm": 8.026850700378418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2638511657714844,
      "logits/rejected": -0.19862934947013855,
      "logps/chosen": -401.24462890625,
      "logps/rejected": -420.8721923828125,
      "loss": 0.4045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31026896834373474,
      "rewards/margins": 1.142788052558899,
      "rewards/rejected": -1.4530569314956665,
      "step": 12777
    },
    {
      "epoch": 1.672086366238652,
      "grad_norm": 10.409926414489746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2095961570739746,
      "logits/rejected": -0.28634172677993774,
      "logps/chosen": -232.76535034179688,
      "logps/rejected": -290.68280029296875,
      "loss": 0.432,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.360454797744751,
      "rewards/margins": 1.1701029539108276,
      "rewards/rejected": -2.530557870864868,
      "step": 12778
    },
    {
      "epoch": 1.6722172241760038,
      "grad_norm": 9.551652908325195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03801689296960831,
      "logits/rejected": -0.06670509278774261,
      "logps/chosen": -461.9419860839844,
      "logps/rejected": -385.6691589355469,
      "loss": 0.571,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8287473320960999,
      "rewards/margins": 0.6528517603874207,
      "rewards/rejected": -1.481598973274231,
      "step": 12779
    },
    {
      "epoch": 1.6723480821133556,
      "grad_norm": 6.382769584655762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39827799797058105,
      "logits/rejected": -0.5063769817352295,
      "logps/chosen": -394.9134521484375,
      "logps/rejected": -499.9799499511719,
      "loss": 0.3359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44668787717819214,
      "rewards/margins": 1.51350998878479,
      "rewards/rejected": -1.9601976871490479,
      "step": 12780
    },
    {
      "epoch": 1.6724789400507074,
      "grad_norm": 4.638802528381348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05322989076375961,
      "logits/rejected": -0.23657134175300598,
      "logps/chosen": -406.76513671875,
      "logps/rejected": -406.7731628417969,
      "loss": 0.2026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7424525022506714,
      "rewards/margins": 1.7916290760040283,
      "rewards/rejected": -2.53408145904541,
      "step": 12781
    },
    {
      "epoch": 1.6726097979880592,
      "grad_norm": 14.386730194091797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12705358862876892,
      "logits/rejected": -0.03383825719356537,
      "logps/chosen": -466.5645751953125,
      "logps/rejected": -494.364501953125,
      "loss": 0.7259,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0161912441253662,
      "rewards/margins": 0.8223028182983398,
      "rewards/rejected": -1.8384941816329956,
      "step": 12782
    },
    {
      "epoch": 1.672740655925411,
      "grad_norm": 9.041023254394531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29632502794265747,
      "logits/rejected": -0.38115620613098145,
      "logps/chosen": -515.7291259765625,
      "logps/rejected": -513.0094604492188,
      "loss": 0.5715,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5511003732681274,
      "rewards/margins": 1.4245126247406006,
      "rewards/rejected": -1.9756131172180176,
      "step": 12783
    },
    {
      "epoch": 1.6728715138627628,
      "grad_norm": 5.579288959503174,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0653340220451355,
      "logits/rejected": -0.344304621219635,
      "logps/chosen": -522.4197387695312,
      "logps/rejected": -387.9176025390625,
      "loss": 0.2752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.008432373404502869,
      "rewards/margins": 1.7606801986694336,
      "rewards/rejected": -1.7522478103637695,
      "step": 12784
    },
    {
      "epoch": 1.6730023718001146,
      "grad_norm": 11.203917503356934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29725584387779236,
      "logits/rejected": -0.2702377140522003,
      "logps/chosen": -358.3563232421875,
      "logps/rejected": -502.7396240234375,
      "loss": 0.4844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5813036561012268,
      "rewards/margins": 1.4184341430664062,
      "rewards/rejected": -1.9997377395629883,
      "step": 12785
    },
    {
      "epoch": 1.6731332297374664,
      "grad_norm": 9.892621040344238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5175172686576843,
      "logits/rejected": -0.5145689845085144,
      "logps/chosen": -482.9583435058594,
      "logps/rejected": -604.598388671875,
      "loss": 0.72,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6688373684883118,
      "rewards/margins": 1.0015724897384644,
      "rewards/rejected": -1.6704097986221313,
      "step": 12786
    },
    {
      "epoch": 1.6732640876748182,
      "grad_norm": 8.477598190307617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3253381848335266,
      "logits/rejected": -0.31014105677604675,
      "logps/chosen": -402.72265625,
      "logps/rejected": -445.295166015625,
      "loss": 0.4104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3590143918991089,
      "rewards/margins": 1.5322327613830566,
      "rewards/rejected": -2.891247034072876,
      "step": 12787
    },
    {
      "epoch": 1.67339494561217,
      "grad_norm": 4.636229515075684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1518712341785431,
      "logits/rejected": 0.09169881790876389,
      "logps/chosen": -396.03118896484375,
      "logps/rejected": -449.1042175292969,
      "loss": 0.3571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.380287230014801,
      "rewards/margins": 2.0606727600097656,
      "rewards/rejected": -2.440960168838501,
      "step": 12788
    },
    {
      "epoch": 1.6735258035495215,
      "grad_norm": 4.1381916999816895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1316715031862259,
      "logits/rejected": -0.33075278997421265,
      "logps/chosen": -594.647705078125,
      "logps/rejected": -442.2556457519531,
      "loss": 0.1331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9308719038963318,
      "rewards/margins": 3.0390541553497314,
      "rewards/rejected": -2.108182191848755,
      "step": 12789
    },
    {
      "epoch": 1.6736566614868733,
      "grad_norm": 3.664046287536621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20456214249134064,
      "logits/rejected": -0.47173112630844116,
      "logps/chosen": -415.8876953125,
      "logps/rejected": -311.6134033203125,
      "loss": 0.178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6211355328559875,
      "rewards/margins": 2.6694235801696777,
      "rewards/rejected": -3.2905588150024414,
      "step": 12790
    },
    {
      "epoch": 1.673787519424225,
      "grad_norm": 14.885292053222656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6985805630683899,
      "logits/rejected": -0.4045625627040863,
      "logps/chosen": -355.9150695800781,
      "logps/rejected": -411.51904296875,
      "loss": 0.5926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4047479629516602,
      "rewards/margins": 1.0443458557128906,
      "rewards/rejected": -2.449093818664551,
      "step": 12791
    },
    {
      "epoch": 1.673918377361577,
      "grad_norm": 7.976291179656982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00768589973449707,
      "logits/rejected": -0.20109249651432037,
      "logps/chosen": -388.93316650390625,
      "logps/rejected": -310.0697326660156,
      "loss": 0.4057,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5314176082611084,
      "rewards/margins": 0.9671739935874939,
      "rewards/rejected": -2.498591661453247,
      "step": 12792
    },
    {
      "epoch": 1.6740492352989285,
      "grad_norm": 7.416320323944092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.033073991537094116,
      "logits/rejected": -0.08500231802463531,
      "logps/chosen": -306.2798156738281,
      "logps/rejected": -475.53118896484375,
      "loss": 0.3849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5813684463500977,
      "rewards/margins": 1.300718903541565,
      "rewards/rejected": -1.8820874691009521,
      "step": 12793
    },
    {
      "epoch": 1.6741800932362803,
      "grad_norm": 9.703986167907715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25946009159088135,
      "logits/rejected": -0.29458165168762207,
      "logps/chosen": -324.6985168457031,
      "logps/rejected": -532.1463623046875,
      "loss": 0.3734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7577248811721802,
      "rewards/margins": 1.4547271728515625,
      "rewards/rejected": -3.2124521732330322,
      "step": 12794
    },
    {
      "epoch": 1.674310951173632,
      "grad_norm": 6.053508758544922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06615103781223297,
      "logits/rejected": -0.1418398767709732,
      "logps/chosen": -597.3076782226562,
      "logps/rejected": -442.41802978515625,
      "loss": 0.2886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13390350341796875,
      "rewards/margins": 1.3543505668640137,
      "rewards/rejected": -1.220447063446045,
      "step": 12795
    },
    {
      "epoch": 1.6744418091109838,
      "grad_norm": 15.420076370239258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08109445869922638,
      "logits/rejected": -0.05373917892575264,
      "logps/chosen": -659.6158447265625,
      "logps/rejected": -441.8489074707031,
      "loss": 0.83,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18706858158111572,
      "rewards/margins": 0.8117170333862305,
      "rewards/rejected": -0.9987855553627014,
      "step": 12796
    },
    {
      "epoch": 1.6745726670483356,
      "grad_norm": 12.994820594787598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20498822629451752,
      "logits/rejected": -0.3691987097263336,
      "logps/chosen": -395.41162109375,
      "logps/rejected": -371.92724609375,
      "loss": 0.7605,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2228537797927856,
      "rewards/margins": 0.7328833341598511,
      "rewards/rejected": -1.9557373523712158,
      "step": 12797
    },
    {
      "epoch": 1.6747035249856874,
      "grad_norm": 9.113850593566895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.244713693857193,
      "logits/rejected": -0.27625104784965515,
      "logps/chosen": -394.71246337890625,
      "logps/rejected": -424.3773193359375,
      "loss": 0.5384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7046121954917908,
      "rewards/margins": 1.6746540069580078,
      "rewards/rejected": -2.3792662620544434,
      "step": 12798
    },
    {
      "epoch": 1.6748343829230392,
      "grad_norm": 8.589014053344727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2688734531402588,
      "logits/rejected": -0.30630192160606384,
      "logps/chosen": -466.530029296875,
      "logps/rejected": -424.49169921875,
      "loss": 0.4363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9997403025627136,
      "rewards/margins": 2.3524487018585205,
      "rewards/rejected": -3.352189064025879,
      "step": 12799
    },
    {
      "epoch": 1.674965240860391,
      "grad_norm": 4.005672931671143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37066540122032166,
      "logits/rejected": -0.32853883504867554,
      "logps/chosen": -339.71148681640625,
      "logps/rejected": -401.69873046875,
      "loss": 0.2962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.752602219581604,
      "rewards/margins": 1.695242166519165,
      "rewards/rejected": -2.4478447437286377,
      "step": 12800
    },
    {
      "epoch": 1.6750960987977428,
      "grad_norm": 6.792181491851807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006779663264751434,
      "logits/rejected": 0.31710320711135864,
      "logps/chosen": -451.1094055175781,
      "logps/rejected": -476.3323059082031,
      "loss": 0.298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4544258117675781,
      "rewards/margins": 1.6851251125335693,
      "rewards/rejected": -2.1395509243011475,
      "step": 12801
    },
    {
      "epoch": 1.6752269567350946,
      "grad_norm": 14.250694274902344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18527449667453766,
      "logits/rejected": -0.3075944781303406,
      "logps/chosen": -461.6044616699219,
      "logps/rejected": -532.8477783203125,
      "loss": 0.8007,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8394476175308228,
      "rewards/margins": 0.672378659248352,
      "rewards/rejected": -1.5118262767791748,
      "step": 12802
    },
    {
      "epoch": 1.6753578146724464,
      "grad_norm": 13.710260391235352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11990826576948166,
      "logits/rejected": -0.38724592328071594,
      "logps/chosen": -445.72149658203125,
      "logps/rejected": -508.439453125,
      "loss": 0.4221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0260089635849,
      "rewards/margins": 2.0645711421966553,
      "rewards/rejected": -3.090580463409424,
      "step": 12803
    },
    {
      "epoch": 1.675488672609798,
      "grad_norm": 3.7355823516845703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11533626914024353,
      "logits/rejected": -0.24111361801624298,
      "logps/chosen": -691.2988891601562,
      "logps/rejected": -626.0306396484375,
      "loss": 0.1222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14070892333984375,
      "rewards/margins": 2.687994956970215,
      "rewards/rejected": -2.8287038803100586,
      "step": 12804
    },
    {
      "epoch": 1.6756195305471497,
      "grad_norm": 7.992196559906006,
      "learning_rate": 5e-06,
      "logits/chosen": 0.055345721542835236,
      "logits/rejected": -0.35469651222229004,
      "logps/chosen": -572.8541259765625,
      "logps/rejected": -456.35638427734375,
      "loss": 0.3022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37476879358291626,
      "rewards/margins": 1.7540061473846436,
      "rewards/rejected": -2.128775119781494,
      "step": 12805
    },
    {
      "epoch": 1.6757503884845015,
      "grad_norm": 4.71540641784668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07200414687395096,
      "logits/rejected": 0.07747713476419449,
      "logps/chosen": -702.19384765625,
      "logps/rejected": -639.3734741210938,
      "loss": 0.1579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15509873628616333,
      "rewards/margins": 2.692751169204712,
      "rewards/rejected": -2.8478498458862305,
      "step": 12806
    },
    {
      "epoch": 1.6758812464218533,
      "grad_norm": 8.30917739868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47083747386932373,
      "logits/rejected": -0.6010528802871704,
      "logps/chosen": -519.7127685546875,
      "logps/rejected": -457.9691162109375,
      "loss": 0.3954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6326797008514404,
      "rewards/margins": 1.645493745803833,
      "rewards/rejected": -2.2781734466552734,
      "step": 12807
    },
    {
      "epoch": 1.676012104359205,
      "grad_norm": 14.492026329040527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2805720567703247,
      "logits/rejected": -0.26632529497146606,
      "logps/chosen": -638.2606811523438,
      "logps/rejected": -549.172607421875,
      "loss": 0.795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3531039357185364,
      "rewards/margins": 0.9326063394546509,
      "rewards/rejected": -1.285710334777832,
      "step": 12808
    },
    {
      "epoch": 1.6761429622965567,
      "grad_norm": 11.096907615661621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2158472239971161,
      "logits/rejected": -0.18734821677207947,
      "logps/chosen": -497.7696228027344,
      "logps/rejected": -513.836181640625,
      "loss": 0.5614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9816133975982666,
      "rewards/margins": 0.8121408224105835,
      "rewards/rejected": -1.7937541007995605,
      "step": 12809
    },
    {
      "epoch": 1.6762738202339085,
      "grad_norm": 6.00819206237793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16022220253944397,
      "logits/rejected": -0.21637026965618134,
      "logps/chosen": -487.070068359375,
      "logps/rejected": -352.15985107421875,
      "loss": 0.2297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5416067242622375,
      "rewards/margins": 2.1565322875976562,
      "rewards/rejected": -2.698139190673828,
      "step": 12810
    },
    {
      "epoch": 1.6764046781712603,
      "grad_norm": 8.809993743896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3248146176338196,
      "logits/rejected": -0.1467733532190323,
      "logps/chosen": -394.8911437988281,
      "logps/rejected": -459.4537353515625,
      "loss": 0.3744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2624781131744385,
      "rewards/margins": 1.9348909854888916,
      "rewards/rejected": -3.197369337081909,
      "step": 12811
    },
    {
      "epoch": 1.676535536108612,
      "grad_norm": 6.01209831237793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3172907829284668,
      "logits/rejected": -0.25589659810066223,
      "logps/chosen": -345.63482666015625,
      "logps/rejected": -420.458984375,
      "loss": 0.2899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9902303814888,
      "rewards/margins": 1.6549116373062134,
      "rewards/rejected": -2.645142078399658,
      "step": 12812
    },
    {
      "epoch": 1.6766663940459638,
      "grad_norm": 5.854462146759033,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17861279845237732,
      "logits/rejected": -0.22861172258853912,
      "logps/chosen": -393.4505615234375,
      "logps/rejected": -403.82666015625,
      "loss": 0.3559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3941795229911804,
      "rewards/margins": 1.1196215152740479,
      "rewards/rejected": -1.5138009786605835,
      "step": 12813
    },
    {
      "epoch": 1.6767972519833156,
      "grad_norm": 9.49495792388916,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05240297690033913,
      "logits/rejected": -0.03372744843363762,
      "logps/chosen": -471.5322570800781,
      "logps/rejected": -462.7909240722656,
      "loss": 0.5655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7073736190795898,
      "rewards/margins": 1.2894951105117798,
      "rewards/rejected": -1.99686861038208,
      "step": 12814
    },
    {
      "epoch": 1.6769281099206674,
      "grad_norm": 12.598013877868652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2186332792043686,
      "logits/rejected": -0.3777392506599426,
      "logps/chosen": -398.495849609375,
      "logps/rejected": -308.9317626953125,
      "loss": 0.8758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5746407508850098,
      "rewards/margins": 0.02716195583343506,
      "rewards/rejected": -1.6018027067184448,
      "step": 12815
    },
    {
      "epoch": 1.6770589678580192,
      "grad_norm": 8.224756240844727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021784011274576187,
      "logits/rejected": -0.05697765201330185,
      "logps/chosen": -525.2664184570312,
      "logps/rejected": -539.976318359375,
      "loss": 0.4317,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19190675020217896,
      "rewards/margins": 1.2367922067642212,
      "rewards/rejected": -1.4286987781524658,
      "step": 12816
    },
    {
      "epoch": 1.677189825795371,
      "grad_norm": 14.300860404968262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19223526120185852,
      "logits/rejected": -0.35089290142059326,
      "logps/chosen": -216.02685546875,
      "logps/rejected": -413.05511474609375,
      "loss": 0.6963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6200863122940063,
      "rewards/margins": 1.6896867752075195,
      "rewards/rejected": -3.3097732067108154,
      "step": 12817
    },
    {
      "epoch": 1.6773206837327228,
      "grad_norm": 7.768774509429932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12216009944677353,
      "logits/rejected": -0.12888455390930176,
      "logps/chosen": -512.094970703125,
      "logps/rejected": -374.55029296875,
      "loss": 0.3331,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34424132108688354,
      "rewards/margins": 2.147770881652832,
      "rewards/rejected": -2.4920122623443604,
      "step": 12818
    },
    {
      "epoch": 1.6774515416700744,
      "grad_norm": 5.835766315460205,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02355952560901642,
      "logits/rejected": -0.11842299997806549,
      "logps/chosen": -546.0463256835938,
      "logps/rejected": -393.26409912109375,
      "loss": 0.2855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.146900936961174,
      "rewards/margins": 1.889176368713379,
      "rewards/rejected": -1.7422754764556885,
      "step": 12819
    },
    {
      "epoch": 1.6775823996074262,
      "grad_norm": 7.471240997314453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0024405494332313538,
      "logits/rejected": 0.07317055016756058,
      "logps/chosen": -327.12957763671875,
      "logps/rejected": -397.3558349609375,
      "loss": 0.4592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01625155657529831,
      "rewards/margins": 1.163354516029358,
      "rewards/rejected": -1.1796059608459473,
      "step": 12820
    },
    {
      "epoch": 1.677713257544778,
      "grad_norm": 14.904400825500488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1945350617170334,
      "logits/rejected": 0.038348156958818436,
      "logps/chosen": -364.2818908691406,
      "logps/rejected": -395.15362548828125,
      "loss": 0.9156,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9961029291152954,
      "rewards/margins": -0.07894289493560791,
      "rewards/rejected": -0.9171600937843323,
      "step": 12821
    },
    {
      "epoch": 1.6778441154821297,
      "grad_norm": 7.529302597045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31574827432632446,
      "logits/rejected": -0.32762622833251953,
      "logps/chosen": -374.64453125,
      "logps/rejected": -376.08575439453125,
      "loss": 0.2864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32899075746536255,
      "rewards/margins": 1.989226222038269,
      "rewards/rejected": -2.3182170391082764,
      "step": 12822
    },
    {
      "epoch": 1.6779749734194813,
      "grad_norm": 9.452951431274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13800989091396332,
      "logits/rejected": -0.1696631908416748,
      "logps/chosen": -584.288330078125,
      "logps/rejected": -593.664306640625,
      "loss": 0.3808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22329579293727875,
      "rewards/margins": 1.6881330013275146,
      "rewards/rejected": -1.911428689956665,
      "step": 12823
    },
    {
      "epoch": 1.678105831356833,
      "grad_norm": 6.375288009643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5897981524467468,
      "logits/rejected": -0.5588513612747192,
      "logps/chosen": -374.0704345703125,
      "logps/rejected": -301.9869384765625,
      "loss": 0.4202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.695133626461029,
      "rewards/margins": 1.5155736207962036,
      "rewards/rejected": -2.210706949234009,
      "step": 12824
    },
    {
      "epoch": 1.678236689294185,
      "grad_norm": 10.727754592895508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2793813943862915,
      "logits/rejected": -0.3169713020324707,
      "logps/chosen": -251.45751953125,
      "logps/rejected": -265.9798889160156,
      "loss": 0.5974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5637677907943726,
      "rewards/margins": 0.7495322227478027,
      "rewards/rejected": -2.313300132751465,
      "step": 12825
    },
    {
      "epoch": 1.6783675472315367,
      "grad_norm": 9.943930625915527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17997148633003235,
      "logits/rejected": -0.10045915096998215,
      "logps/chosen": -448.4432678222656,
      "logps/rejected": -405.915771484375,
      "loss": 0.3951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7928850650787354,
      "rewards/margins": 1.0644176006317139,
      "rewards/rejected": -1.8573026657104492,
      "step": 12826
    },
    {
      "epoch": 1.6784984051688885,
      "grad_norm": 8.975625038146973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0032863393425941467,
      "logits/rejected": -0.12593838572502136,
      "logps/chosen": -322.1373291015625,
      "logps/rejected": -325.66424560546875,
      "loss": 0.427,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5244312286376953,
      "rewards/margins": 0.9430685043334961,
      "rewards/rejected": -2.4674997329711914,
      "step": 12827
    },
    {
      "epoch": 1.6786292631062403,
      "grad_norm": 5.975149631500244,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08539847284555435,
      "logits/rejected": -0.12765175104141235,
      "logps/chosen": -530.9319458007812,
      "logps/rejected": -441.14752197265625,
      "loss": 0.3196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4472540020942688,
      "rewards/margins": 1.4225633144378662,
      "rewards/rejected": -1.8698173761367798,
      "step": 12828
    },
    {
      "epoch": 1.678760121043592,
      "grad_norm": 10.906814575195312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09200634807348251,
      "logits/rejected": -0.02149277552962303,
      "logps/chosen": -610.895263671875,
      "logps/rejected": -402.45477294921875,
      "loss": 0.541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3900253772735596,
      "rewards/margins": 0.4965582489967346,
      "rewards/rejected": -1.8865835666656494,
      "step": 12829
    },
    {
      "epoch": 1.6788909789809439,
      "grad_norm": 15.424761772155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09199626743793488,
      "logits/rejected": -0.18145392835140228,
      "logps/chosen": -529.127685546875,
      "logps/rejected": -448.3143310546875,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.521519660949707,
      "rewards/margins": 0.995697557926178,
      "rewards/rejected": -2.5172171592712402,
      "step": 12830
    },
    {
      "epoch": 1.6790218369182957,
      "grad_norm": 10.454850196838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07107934355735779,
      "logits/rejected": 0.14588221907615662,
      "logps/chosen": -439.15679931640625,
      "logps/rejected": -535.2215576171875,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8168414831161499,
      "rewards/margins": 1.1790393590927124,
      "rewards/rejected": -1.9958808422088623,
      "step": 12831
    },
    {
      "epoch": 1.6791526948556474,
      "grad_norm": 10.568943977355957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05724290758371353,
      "logits/rejected": -0.28548988699913025,
      "logps/chosen": -476.3985595703125,
      "logps/rejected": -542.48876953125,
      "loss": 0.4708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4409689903259277,
      "rewards/margins": 0.7655670046806335,
      "rewards/rejected": -2.206536054611206,
      "step": 12832
    },
    {
      "epoch": 1.6792835527929992,
      "grad_norm": 11.869569778442383,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3111621141433716,
      "logits/rejected": 0.1723431646823883,
      "logps/chosen": -645.3087158203125,
      "logps/rejected": -503.45001220703125,
      "loss": 0.5479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4147079586982727,
      "rewards/margins": 1.0375317335128784,
      "rewards/rejected": -1.4522396326065063,
      "step": 12833
    },
    {
      "epoch": 1.679414410730351,
      "grad_norm": 9.693819046020508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019981447607278824,
      "logits/rejected": -0.15816497802734375,
      "logps/chosen": -665.1231689453125,
      "logps/rejected": -538.8508911132812,
      "loss": 0.4872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5191875100135803,
      "rewards/margins": 0.7505871057510376,
      "rewards/rejected": -1.2697744369506836,
      "step": 12834
    },
    {
      "epoch": 1.6795452686677026,
      "grad_norm": 5.432539463043213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27448463439941406,
      "logits/rejected": -0.23917782306671143,
      "logps/chosen": -537.4925537109375,
      "logps/rejected": -560.7548217773438,
      "loss": 0.1971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6422061920166016,
      "rewards/margins": 2.3789658546447754,
      "rewards/rejected": -1.7367596626281738,
      "step": 12835
    },
    {
      "epoch": 1.6796761266050544,
      "grad_norm": 5.316415786743164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19002318382263184,
      "logits/rejected": -0.17898832261562347,
      "logps/chosen": -465.9204406738281,
      "logps/rejected": -387.806640625,
      "loss": 0.2893,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2824157476425171,
      "rewards/margins": 1.6480997800827026,
      "rewards/rejected": -1.3656837940216064,
      "step": 12836
    },
    {
      "epoch": 1.6798069845424062,
      "grad_norm": 3.590379476547241,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31994831562042236,
      "logits/rejected": -0.22872963547706604,
      "logps/chosen": -459.6326904296875,
      "logps/rejected": -417.01019287109375,
      "loss": 0.1395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41235312819480896,
      "rewards/margins": 2.5085206031799316,
      "rewards/rejected": -2.9208736419677734,
      "step": 12837
    },
    {
      "epoch": 1.6799378424797577,
      "grad_norm": 11.449261665344238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3983379304409027,
      "logits/rejected": -0.38976675271987915,
      "logps/chosen": -427.805419921875,
      "logps/rejected": -381.70770263671875,
      "loss": 0.6821,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1860729455947876,
      "rewards/margins": 1.1614439487457275,
      "rewards/rejected": -2.3475165367126465,
      "step": 12838
    },
    {
      "epoch": 1.6800687004171095,
      "grad_norm": 6.421660423278809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.012960023246705532,
      "logits/rejected": -0.02380239963531494,
      "logps/chosen": -345.7933349609375,
      "logps/rejected": -308.73095703125,
      "loss": 0.3515,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6421173214912415,
      "rewards/margins": 1.2321372032165527,
      "rewards/rejected": -1.8742544651031494,
      "step": 12839
    },
    {
      "epoch": 1.6801995583544613,
      "grad_norm": 7.378627777099609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19046416878700256,
      "logits/rejected": -0.29736730456352234,
      "logps/chosen": -306.2406311035156,
      "logps/rejected": -353.0793151855469,
      "loss": 0.2702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40397894382476807,
      "rewards/margins": 1.637730360031128,
      "rewards/rejected": -2.0417091846466064,
      "step": 12840
    },
    {
      "epoch": 1.6803304162918131,
      "grad_norm": 17.07555389404297,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2498008906841278,
      "logits/rejected": 0.20866860449314117,
      "logps/chosen": -594.8177490234375,
      "logps/rejected": -640.94873046875,
      "loss": 1.2995,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0571563243865967,
      "rewards/margins": -0.048912808299064636,
      "rewards/rejected": -1.0082435607910156,
      "step": 12841
    },
    {
      "epoch": 1.680461274229165,
      "grad_norm": 9.525604248046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05683637410402298,
      "logits/rejected": -0.12422463297843933,
      "logps/chosen": -308.9787292480469,
      "logps/rejected": -376.76361083984375,
      "loss": 0.4458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1136887073516846,
      "rewards/margins": 1.241828203201294,
      "rewards/rejected": -2.3555166721343994,
      "step": 12842
    },
    {
      "epoch": 1.6805921321665167,
      "grad_norm": 7.965705871582031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3534674644470215,
      "logits/rejected": -0.20851415395736694,
      "logps/chosen": -452.48419189453125,
      "logps/rejected": -585.5596313476562,
      "loss": 0.2768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9550051093101501,
      "rewards/margins": 1.4407716989517212,
      "rewards/rejected": -2.3957767486572266,
      "step": 12843
    },
    {
      "epoch": 1.6807229901038685,
      "grad_norm": 7.185003280639648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17229776084423065,
      "logits/rejected": -0.08448221534490585,
      "logps/chosen": -535.257080078125,
      "logps/rejected": -388.5650329589844,
      "loss": 0.3449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31177347898483276,
      "rewards/margins": 1.3519362211227417,
      "rewards/rejected": -1.6637096405029297,
      "step": 12844
    },
    {
      "epoch": 1.6808538480412203,
      "grad_norm": 7.519027233123779,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1548839509487152,
      "logits/rejected": 0.020673222839832306,
      "logps/chosen": -564.7108154296875,
      "logps/rejected": -524.0245361328125,
      "loss": 0.4011,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06105421483516693,
      "rewards/margins": 1.1382368803024292,
      "rewards/rejected": -1.1992911100387573,
      "step": 12845
    },
    {
      "epoch": 1.680984705978572,
      "grad_norm": 10.175905227661133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29406049847602844,
      "logits/rejected": -0.36382102966308594,
      "logps/chosen": -414.22430419921875,
      "logps/rejected": -427.9646301269531,
      "loss": 0.5344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9629157781600952,
      "rewards/margins": 0.901267409324646,
      "rewards/rejected": -1.8641833066940308,
      "step": 12846
    },
    {
      "epoch": 1.6811155639159239,
      "grad_norm": 12.881915092468262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28016647696495056,
      "logits/rejected": -0.3989759087562561,
      "logps/chosen": -400.7292785644531,
      "logps/rejected": -328.5820617675781,
      "loss": 0.6881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47301793098449707,
      "rewards/margins": 0.3586973547935486,
      "rewards/rejected": -0.8317152857780457,
      "step": 12847
    },
    {
      "epoch": 1.6812464218532757,
      "grad_norm": 6.236656665802002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009570833295583725,
      "logits/rejected": -0.0688503235578537,
      "logps/chosen": -528.3909912109375,
      "logps/rejected": -397.81072998046875,
      "loss": 0.308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6721475124359131,
      "rewards/margins": 1.4137102365493774,
      "rewards/rejected": -0.7415626645088196,
      "step": 12848
    },
    {
      "epoch": 1.6813772797906275,
      "grad_norm": 11.288359642028809,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10121749341487885,
      "logits/rejected": -0.2041085809469223,
      "logps/chosen": -601.3167114257812,
      "logps/rejected": -520.177734375,
      "loss": 0.6681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15386009216308594,
      "rewards/margins": 1.438369870185852,
      "rewards/rejected": -1.5922298431396484,
      "step": 12849
    },
    {
      "epoch": 1.681508137727979,
      "grad_norm": 14.182936668395996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3278640806674957,
      "logits/rejected": -0.1496603786945343,
      "logps/chosen": -485.42572021484375,
      "logps/rejected": -335.1195373535156,
      "loss": 0.6319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3479768931865692,
      "rewards/margins": 0.5124185085296631,
      "rewards/rejected": -0.8603954911231995,
      "step": 12850
    },
    {
      "epoch": 1.6816389956653308,
      "grad_norm": 11.29204273223877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.006672360002994537,
      "logits/rejected": 0.11546412855386734,
      "logps/chosen": -494.8741760253906,
      "logps/rejected": -506.82586669921875,
      "loss": 0.6685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8773903846740723,
      "rewards/margins": 0.45217448472976685,
      "rewards/rejected": -1.3295648097991943,
      "step": 12851
    },
    {
      "epoch": 1.6817698536026826,
      "grad_norm": 11.97269344329834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27933621406555176,
      "logits/rejected": -0.27878493070602417,
      "logps/chosen": -627.0286254882812,
      "logps/rejected": -578.2827758789062,
      "loss": 0.605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2957439422607422,
      "rewards/margins": 0.7911351919174194,
      "rewards/rejected": -1.0868792533874512,
      "step": 12852
    },
    {
      "epoch": 1.6819007115400344,
      "grad_norm": 14.680119514465332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04478711262345314,
      "logits/rejected": -0.1024923175573349,
      "logps/chosen": -284.6625671386719,
      "logps/rejected": -357.0325622558594,
      "loss": 0.4432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40931734442710876,
      "rewards/margins": 1.2738815546035767,
      "rewards/rejected": -1.6831989288330078,
      "step": 12853
    },
    {
      "epoch": 1.682031569477386,
      "grad_norm": 8.458094596862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39170923829078674,
      "logits/rejected": -0.3061924874782562,
      "logps/chosen": -403.96490478515625,
      "logps/rejected": -405.6826171875,
      "loss": 0.4482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5802565217018127,
      "rewards/margins": 1.1792749166488647,
      "rewards/rejected": -1.7595314979553223,
      "step": 12854
    },
    {
      "epoch": 1.6821624274147378,
      "grad_norm": 8.05402660369873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18184660375118256,
      "logits/rejected": -0.10793368518352509,
      "logps/chosen": -484.30975341796875,
      "logps/rejected": -470.717529296875,
      "loss": 0.363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.441358745098114,
      "rewards/margins": 1.2394617795944214,
      "rewards/rejected": -1.6808205842971802,
      "step": 12855
    },
    {
      "epoch": 1.6822932853520896,
      "grad_norm": 10.566764831542969,
      "learning_rate": 5e-06,
      "logits/chosen": 0.27323830127716064,
      "logits/rejected": 0.166310653090477,
      "logps/chosen": -414.58355712890625,
      "logps/rejected": -433.9585266113281,
      "loss": 0.598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8850088119506836,
      "rewards/margins": 0.9115670919418335,
      "rewards/rejected": -1.796575903892517,
      "step": 12856
    },
    {
      "epoch": 1.6824241432894413,
      "grad_norm": 9.740013122558594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40565457940101624,
      "logits/rejected": -0.392640084028244,
      "logps/chosen": -345.5886535644531,
      "logps/rejected": -451.6269836425781,
      "loss": 0.454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4965473413467407,
      "rewards/margins": 1.2983136177062988,
      "rewards/rejected": -1.79486083984375,
      "step": 12857
    },
    {
      "epoch": 1.6825550012267931,
      "grad_norm": 6.009622097015381,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0652814656496048,
      "logits/rejected": 0.034334149211645126,
      "logps/chosen": -461.5439453125,
      "logps/rejected": -518.2425537109375,
      "loss": 0.3682,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6550334095954895,
      "rewards/margins": 1.3984041213989258,
      "rewards/rejected": -0.7433708310127258,
      "step": 12858
    },
    {
      "epoch": 1.682685859164145,
      "grad_norm": 10.362842559814453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1894644945859909,
      "logits/rejected": -0.3031613230705261,
      "logps/chosen": -441.3324279785156,
      "logps/rejected": -442.63458251953125,
      "loss": 0.5454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5820707082748413,
      "rewards/margins": 1.0802503824234009,
      "rewards/rejected": -1.6623210906982422,
      "step": 12859
    },
    {
      "epoch": 1.6828167171014967,
      "grad_norm": 9.342583656311035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4911700487136841,
      "logits/rejected": -0.3228318691253662,
      "logps/chosen": -465.0667724609375,
      "logps/rejected": -470.2397155761719,
      "loss": 0.4666,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7728468179702759,
      "rewards/margins": 0.7549715042114258,
      "rewards/rejected": -1.5278183221817017,
      "step": 12860
    },
    {
      "epoch": 1.6829475750388485,
      "grad_norm": 8.113357543945312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2198909968137741,
      "logits/rejected": -0.5505325198173523,
      "logps/chosen": -529.4976806640625,
      "logps/rejected": -537.4774169921875,
      "loss": 0.3445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24219760298728943,
      "rewards/margins": 1.6056429147720337,
      "rewards/rejected": -1.8478405475616455,
      "step": 12861
    },
    {
      "epoch": 1.6830784329762003,
      "grad_norm": 5.909122467041016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1759626865386963,
      "logits/rejected": -0.05404692143201828,
      "logps/chosen": -769.2281494140625,
      "logps/rejected": -404.73590087890625,
      "loss": 0.3733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24684861302375793,
      "rewards/margins": 2.594003915786743,
      "rewards/rejected": -2.3471553325653076,
      "step": 12862
    },
    {
      "epoch": 1.683209290913552,
      "grad_norm": 4.170773983001709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2577522397041321,
      "logits/rejected": 0.12807203829288483,
      "logps/chosen": -466.33856201171875,
      "logps/rejected": -555.0864868164062,
      "loss": 0.1482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.44273605942726135,
      "rewards/margins": 2.527942657470703,
      "rewards/rejected": -2.9706785678863525,
      "step": 12863
    },
    {
      "epoch": 1.6833401488509039,
      "grad_norm": 10.149014472961426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04194263368844986,
      "logits/rejected": -0.07120613008737564,
      "logps/chosen": -453.91815185546875,
      "logps/rejected": -417.82952880859375,
      "loss": 0.5899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3670839071273804,
      "rewards/margins": 0.9573439359664917,
      "rewards/rejected": -2.324427843093872,
      "step": 12864
    },
    {
      "epoch": 1.6834710067882555,
      "grad_norm": 10.289440155029297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07225671410560608,
      "logits/rejected": -0.0867118388414383,
      "logps/chosen": -335.9378356933594,
      "logps/rejected": -348.7666320800781,
      "loss": 0.4833,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9483048915863037,
      "rewards/margins": 0.861234188079834,
      "rewards/rejected": -2.8095386028289795,
      "step": 12865
    },
    {
      "epoch": 1.6836018647256072,
      "grad_norm": 7.768951416015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02996774949133396,
      "logits/rejected": 0.008531192317605019,
      "logps/chosen": -702.70556640625,
      "logps/rejected": -524.5186767578125,
      "loss": 0.4318,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.159836545586586,
      "rewards/margins": 1.1982277631759644,
      "rewards/rejected": -1.3580642938613892,
      "step": 12866
    },
    {
      "epoch": 1.683732722662959,
      "grad_norm": 10.019972801208496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05017416179180145,
      "logits/rejected": -0.07874912023544312,
      "logps/chosen": -309.4278259277344,
      "logps/rejected": -382.103515625,
      "loss": 0.5267,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1096112728118896,
      "rewards/margins": 1.1675058603286743,
      "rewards/rejected": -2.2771172523498535,
      "step": 12867
    },
    {
      "epoch": 1.6838635806003108,
      "grad_norm": 10.899459838867188,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010062207467854023,
      "logits/rejected": -0.2200711965560913,
      "logps/chosen": -452.968994140625,
      "logps/rejected": -468.3436584472656,
      "loss": 0.6141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1214319467544556,
      "rewards/margins": 1.3635931015014648,
      "rewards/rejected": -2.48502516746521,
      "step": 12868
    },
    {
      "epoch": 1.6839944385376624,
      "grad_norm": 8.353798866271973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26511693000793457,
      "logits/rejected": -0.5164142847061157,
      "logps/chosen": -414.51226806640625,
      "logps/rejected": -461.1209716796875,
      "loss": 0.479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4893455505371094,
      "rewards/margins": 1.7403733730316162,
      "rewards/rejected": -3.2297189235687256,
      "step": 12869
    },
    {
      "epoch": 1.6841252964750142,
      "grad_norm": 7.3128814697265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2542479634284973,
      "logits/rejected": -0.23385608196258545,
      "logps/chosen": -496.77667236328125,
      "logps/rejected": -414.00091552734375,
      "loss": 0.522,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.023389071226119995,
      "rewards/margins": 0.8595566153526306,
      "rewards/rejected": -0.882945716381073,
      "step": 12870
    },
    {
      "epoch": 1.684256154412366,
      "grad_norm": 9.75057315826416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1001322865486145,
      "logits/rejected": -0.2252776175737381,
      "logps/chosen": -505.4146728515625,
      "logps/rejected": -450.5550842285156,
      "loss": 0.4573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3784186840057373,
      "rewards/margins": 0.7658833265304565,
      "rewards/rejected": -2.1443018913269043,
      "step": 12871
    },
    {
      "epoch": 1.6843870123497178,
      "grad_norm": 8.670936584472656,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17937274277210236,
      "logits/rejected": -0.10662506520748138,
      "logps/chosen": -594.4656372070312,
      "logps/rejected": -446.40252685546875,
      "loss": 0.4679,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22756138443946838,
      "rewards/margins": 1.4069340229034424,
      "rewards/rejected": -1.6344953775405884,
      "step": 12872
    },
    {
      "epoch": 1.6845178702870696,
      "grad_norm": 6.784157752990723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.219360813498497,
      "logits/rejected": -0.24958619475364685,
      "logps/chosen": -467.3891296386719,
      "logps/rejected": -459.1611022949219,
      "loss": 0.3104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11320212483406067,
      "rewards/margins": 1.8032536506652832,
      "rewards/rejected": -1.9164557456970215,
      "step": 12873
    },
    {
      "epoch": 1.6846487282244214,
      "grad_norm": 8.471504211425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17282460629940033,
      "logits/rejected": -0.2511085867881775,
      "logps/chosen": -422.4908142089844,
      "logps/rejected": -362.303955078125,
      "loss": 0.5716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28409507870674133,
      "rewards/margins": 0.8177728056907654,
      "rewards/rejected": -1.101867914199829,
      "step": 12874
    },
    {
      "epoch": 1.6847795861617731,
      "grad_norm": 13.394647598266602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1702049970626831,
      "logits/rejected": -0.11467989534139633,
      "logps/chosen": -534.8870849609375,
      "logps/rejected": -471.06365966796875,
      "loss": 0.5828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1538228988647461,
      "rewards/margins": 1.5315247774124146,
      "rewards/rejected": -1.6853477954864502,
      "step": 12875
    },
    {
      "epoch": 1.684910444099125,
      "grad_norm": 10.806944847106934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04778457060456276,
      "logits/rejected": -0.11823201924562454,
      "logps/chosen": -431.168212890625,
      "logps/rejected": -382.1986999511719,
      "loss": 0.698,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.03414249420166,
      "rewards/margins": 0.376626193523407,
      "rewards/rejected": -2.410768508911133,
      "step": 12876
    },
    {
      "epoch": 1.6850413020364767,
      "grad_norm": 4.730177402496338,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10335159301757812,
      "logits/rejected": -0.1603531837463379,
      "logps/chosen": -394.254150390625,
      "logps/rejected": -425.46661376953125,
      "loss": 0.2496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1494941711425781,
      "rewards/margins": 1.9286059141159058,
      "rewards/rejected": -3.0781002044677734,
      "step": 12877
    },
    {
      "epoch": 1.6851721599738285,
      "grad_norm": 6.490392684936523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3508993685245514,
      "logits/rejected": -0.5290839672088623,
      "logps/chosen": -411.8598937988281,
      "logps/rejected": -479.90057373046875,
      "loss": 0.3244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.148982048034668,
      "rewards/margins": 1.332207441329956,
      "rewards/rejected": -2.481189489364624,
      "step": 12878
    },
    {
      "epoch": 1.6853030179111803,
      "grad_norm": 7.306067943572998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5610104203224182,
      "logits/rejected": -0.4850669801235199,
      "logps/chosen": -535.3032836914062,
      "logps/rejected": -458.0576477050781,
      "loss": 0.3582,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9510725140571594,
      "rewards/margins": 1.2081621885299683,
      "rewards/rejected": -2.1592347621917725,
      "step": 12879
    },
    {
      "epoch": 1.685433875848532,
      "grad_norm": 9.459985733032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2779442071914673,
      "logits/rejected": -0.26121893525123596,
      "logps/chosen": -495.1114807128906,
      "logps/rejected": -329.73944091796875,
      "loss": 0.567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36833104491233826,
      "rewards/margins": 1.4693126678466797,
      "rewards/rejected": -1.8376436233520508,
      "step": 12880
    },
    {
      "epoch": 1.6855647337858837,
      "grad_norm": 8.136646270751953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3786522150039673,
      "logits/rejected": -0.2514626383781433,
      "logps/chosen": -368.0255126953125,
      "logps/rejected": -580.125732421875,
      "loss": 0.3917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3580505847930908,
      "rewards/margins": 1.4101587533950806,
      "rewards/rejected": -2.768209218978882,
      "step": 12881
    },
    {
      "epoch": 1.6856955917232355,
      "grad_norm": 7.2221198081970215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021034985780715942,
      "logits/rejected": -0.10275551676750183,
      "logps/chosen": -507.338623046875,
      "logps/rejected": -562.9577026367188,
      "loss": 0.4182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.635370671749115,
      "rewards/margins": 0.9678220152854919,
      "rewards/rejected": -1.6031928062438965,
      "step": 12882
    },
    {
      "epoch": 1.6858264496605873,
      "grad_norm": 7.019614219665527,
      "learning_rate": 5e-06,
      "logits/chosen": 0.008776858448982239,
      "logits/rejected": -0.04751739650964737,
      "logps/chosen": -569.0256958007812,
      "logps/rejected": -628.4915771484375,
      "loss": 0.3623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2100391387939453,
      "rewards/margins": 1.1285582780838013,
      "rewards/rejected": -2.338597297668457,
      "step": 12883
    },
    {
      "epoch": 1.6859573075979388,
      "grad_norm": 12.066901206970215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1711479276418686,
      "logits/rejected": -0.35368961095809937,
      "logps/chosen": -700.9769287109375,
      "logps/rejected": -639.6953125,
      "loss": 0.6831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5902088284492493,
      "rewards/margins": 1.5868817567825317,
      "rewards/rejected": -2.177090644836426,
      "step": 12884
    },
    {
      "epoch": 1.6860881655352906,
      "grad_norm": 6.254439353942871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07050946354866028,
      "logits/rejected": -0.16451075673103333,
      "logps/chosen": -671.7584838867188,
      "logps/rejected": -609.8108520507812,
      "loss": 0.267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43103235960006714,
      "rewards/margins": 1.6705338954925537,
      "rewards/rejected": -2.1015663146972656,
      "step": 12885
    },
    {
      "epoch": 1.6862190234726424,
      "grad_norm": 7.763919830322266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23790881037712097,
      "logits/rejected": -0.4282628893852234,
      "logps/chosen": -517.8253173828125,
      "logps/rejected": -395.76629638671875,
      "loss": 0.4536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7056298851966858,
      "rewards/margins": 1.0546376705169678,
      "rewards/rejected": -1.7602676153182983,
      "step": 12886
    },
    {
      "epoch": 1.6863498814099942,
      "grad_norm": 3.8797457218170166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09743782877922058,
      "logits/rejected": -0.2775888442993164,
      "logps/chosen": -422.5504455566406,
      "logps/rejected": -428.07537841796875,
      "loss": 0.139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31737783551216125,
      "rewards/margins": 3.5056817531585693,
      "rewards/rejected": -3.1883037090301514,
      "step": 12887
    },
    {
      "epoch": 1.686480739347346,
      "grad_norm": 4.115653991699219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3993053436279297,
      "logits/rejected": -0.18949231505393982,
      "logps/chosen": -424.69586181640625,
      "logps/rejected": -543.816162109375,
      "loss": 0.1859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7434030771255493,
      "rewards/margins": 2.2875614166259766,
      "rewards/rejected": -3.0309643745422363,
      "step": 12888
    },
    {
      "epoch": 1.6866115972846978,
      "grad_norm": 7.860937118530273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13274112343788147,
      "logits/rejected": -0.27543210983276367,
      "logps/chosen": -296.5105285644531,
      "logps/rejected": -522.1255493164062,
      "loss": 0.3394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0244226455688477,
      "rewards/margins": 1.7620460987091064,
      "rewards/rejected": -2.786468505859375,
      "step": 12889
    },
    {
      "epoch": 1.6867424552220496,
      "grad_norm": 10.486796379089355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044561631977558136,
      "logits/rejected": -0.14262069761753082,
      "logps/chosen": -418.72528076171875,
      "logps/rejected": -327.56561279296875,
      "loss": 0.528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7056896090507507,
      "rewards/margins": 1.1170202493667603,
      "rewards/rejected": -1.8227097988128662,
      "step": 12890
    },
    {
      "epoch": 1.6868733131594014,
      "grad_norm": 4.449361801147461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17273655533790588,
      "logits/rejected": -0.3087902367115021,
      "logps/chosen": -514.0787353515625,
      "logps/rejected": -511.4833679199219,
      "loss": 0.1646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14238056540489197,
      "rewards/margins": 2.030913829803467,
      "rewards/rejected": -2.1732943058013916,
      "step": 12891
    },
    {
      "epoch": 1.6870041710967532,
      "grad_norm": 9.068781852722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053193822503089905,
      "logits/rejected": -0.044519755989313126,
      "logps/chosen": -487.27142333984375,
      "logps/rejected": -418.4933776855469,
      "loss": 0.5718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6802152991294861,
      "rewards/margins": 0.8660569190979004,
      "rewards/rejected": -1.5462721586227417,
      "step": 12892
    },
    {
      "epoch": 1.687135029034105,
      "grad_norm": 16.15869140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28901323676109314,
      "logits/rejected": -0.38443464040756226,
      "logps/chosen": -610.9962158203125,
      "logps/rejected": -440.4136657714844,
      "loss": 0.5953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9278628826141357,
      "rewards/margins": 0.9261767268180847,
      "rewards/rejected": -1.8540395498275757,
      "step": 12893
    },
    {
      "epoch": 1.6872658869714567,
      "grad_norm": 6.696908950805664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11177752912044525,
      "logits/rejected": -0.21452094614505768,
      "logps/chosen": -411.81396484375,
      "logps/rejected": -504.8653259277344,
      "loss": 0.3416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8623754978179932,
      "rewards/margins": 1.5623334646224976,
      "rewards/rejected": -2.424708843231201,
      "step": 12894
    },
    {
      "epoch": 1.6873967449088085,
      "grad_norm": 4.292905330657959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18094296753406525,
      "logits/rejected": -0.06908387690782547,
      "logps/chosen": -355.99652099609375,
      "logps/rejected": -493.15997314453125,
      "loss": 0.1848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25997722148895264,
      "rewards/margins": 2.9579081535339355,
      "rewards/rejected": -3.2178854942321777,
      "step": 12895
    },
    {
      "epoch": 1.68752760284616,
      "grad_norm": 2.906616449356079,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2627238929271698,
      "logits/rejected": -0.3227275013923645,
      "logps/chosen": -368.6132507324219,
      "logps/rejected": -380.4233703613281,
      "loss": 0.0951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.46384376287460327,
      "rewards/margins": 2.829887628555298,
      "rewards/rejected": -3.293731689453125,
      "step": 12896
    },
    {
      "epoch": 1.687658460783512,
      "grad_norm": 14.735167503356934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4882405698299408,
      "logits/rejected": -0.44551968574523926,
      "logps/chosen": -368.1855773925781,
      "logps/rejected": -375.4696044921875,
      "loss": 0.889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3332198858261108,
      "rewards/margins": 0.36606839299201965,
      "rewards/rejected": -1.6992883682250977,
      "step": 12897
    },
    {
      "epoch": 1.6877893187208637,
      "grad_norm": 8.115612030029297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1973796933889389,
      "logits/rejected": -0.23229026794433594,
      "logps/chosen": -247.0585479736328,
      "logps/rejected": -353.14385986328125,
      "loss": 0.437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.742598533630371,
      "rewards/margins": 0.8285204768180847,
      "rewards/rejected": -2.5711190700531006,
      "step": 12898
    },
    {
      "epoch": 1.6879201766582155,
      "grad_norm": 20.2406005859375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.019799422472715378,
      "logits/rejected": -0.13713619112968445,
      "logps/chosen": -353.16888427734375,
      "logps/rejected": -370.3003845214844,
      "loss": 1.0143,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.350008487701416,
      "rewards/margins": 0.12591630220413208,
      "rewards/rejected": -2.4759249687194824,
      "step": 12899
    },
    {
      "epoch": 1.688051034595567,
      "grad_norm": 10.008817672729492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1501099169254303,
      "logits/rejected": -0.11936599016189575,
      "logps/chosen": -498.46527099609375,
      "logps/rejected": -620.616943359375,
      "loss": 0.5047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1472828388214111,
      "rewards/margins": 1.1186431646347046,
      "rewards/rejected": -2.265925884246826,
      "step": 12900
    },
    {
      "epoch": 1.6881818925329188,
      "grad_norm": 8.96274471282959,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03259339928627014,
      "logits/rejected": -0.18529653549194336,
      "logps/chosen": -554.5072631835938,
      "logps/rejected": -403.40283203125,
      "loss": 0.4891,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10358984023332596,
      "rewards/margins": 1.6474523544311523,
      "rewards/rejected": -1.5438623428344727,
      "step": 12901
    },
    {
      "epoch": 1.6883127504702706,
      "grad_norm": 15.53529167175293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07911816984415054,
      "logits/rejected": -0.22146093845367432,
      "logps/chosen": -499.7317199707031,
      "logps/rejected": -422.26141357421875,
      "loss": 1.0054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3082165718078613,
      "rewards/margins": 0.9714488387107849,
      "rewards/rejected": -2.279665470123291,
      "step": 12902
    },
    {
      "epoch": 1.6884436084076224,
      "grad_norm": 5.351613521575928,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010917797684669495,
      "logits/rejected": -0.07183153182268143,
      "logps/chosen": -453.1908264160156,
      "logps/rejected": -450.370361328125,
      "loss": 0.29,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6939520239830017,
      "rewards/margins": 2.064552068710327,
      "rewards/rejected": -2.7585041522979736,
      "step": 12903
    },
    {
      "epoch": 1.6885744663449742,
      "grad_norm": 6.776058673858643,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007808692753314972,
      "logits/rejected": -0.10841865837574005,
      "logps/chosen": -489.7963562011719,
      "logps/rejected": -411.2260437011719,
      "loss": 0.381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08424471318721771,
      "rewards/margins": 1.3272678852081299,
      "rewards/rejected": -1.4115127325057983,
      "step": 12904
    },
    {
      "epoch": 1.688705324282326,
      "grad_norm": 8.138636589050293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30165761709213257,
      "logits/rejected": -0.13936568796634674,
      "logps/chosen": -432.5622253417969,
      "logps/rejected": -452.8014831542969,
      "loss": 0.4511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8197649717330933,
      "rewards/margins": 1.0796620845794678,
      "rewards/rejected": -1.899427056312561,
      "step": 12905
    },
    {
      "epoch": 1.6888361822196778,
      "grad_norm": 9.118730545043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16098535060882568,
      "logits/rejected": -0.3555211126804352,
      "logps/chosen": -566.4573364257812,
      "logps/rejected": -386.7616271972656,
      "loss": 0.5154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2808550000190735,
      "rewards/margins": 1.4429142475128174,
      "rewards/rejected": -1.7237690687179565,
      "step": 12906
    },
    {
      "epoch": 1.6889670401570296,
      "grad_norm": 8.365901947021484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25454071164131165,
      "logits/rejected": -0.2183777391910553,
      "logps/chosen": -490.650146484375,
      "logps/rejected": -459.8680114746094,
      "loss": 0.4598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9513072967529297,
      "rewards/margins": 1.2053574323654175,
      "rewards/rejected": -2.1566648483276367,
      "step": 12907
    },
    {
      "epoch": 1.6890978980943814,
      "grad_norm": 9.734540939331055,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025041043758392334,
      "logits/rejected": -0.04777069389820099,
      "logps/chosen": -358.13983154296875,
      "logps/rejected": -433.6544494628906,
      "loss": 0.5039,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5165327191352844,
      "rewards/margins": 2.7311789989471436,
      "rewards/rejected": -3.2477118968963623,
      "step": 12908
    },
    {
      "epoch": 1.6892287560317332,
      "grad_norm": 3.4325637817382812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19292625784873962,
      "logits/rejected": -0.30185550451278687,
      "logps/chosen": -483.1591796875,
      "logps/rejected": -680.0281982421875,
      "loss": 0.1267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2767958641052246,
      "rewards/margins": 2.8156509399414062,
      "rewards/rejected": -4.092447280883789,
      "step": 12909
    },
    {
      "epoch": 1.689359613969085,
      "grad_norm": 7.110013008117676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2259742170572281,
      "logits/rejected": -0.4952499568462372,
      "logps/chosen": -625.775146484375,
      "logps/rejected": -431.0874938964844,
      "loss": 0.3811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5402442812919617,
      "rewards/margins": 2.148247241973877,
      "rewards/rejected": -2.6884915828704834,
      "step": 12910
    },
    {
      "epoch": 1.6894904719064365,
      "grad_norm": 6.758903503417969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4001358151435852,
      "logits/rejected": -0.39472609758377075,
      "logps/chosen": -579.126708984375,
      "logps/rejected": -511.254150390625,
      "loss": 0.3732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6136127710342407,
      "rewards/margins": 1.2362083196640015,
      "rewards/rejected": -1.8498210906982422,
      "step": 12911
    },
    {
      "epoch": 1.6896213298437883,
      "grad_norm": 9.362507820129395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23625919222831726,
      "logits/rejected": -0.3682984709739685,
      "logps/chosen": -511.7651062011719,
      "logps/rejected": -450.52508544921875,
      "loss": 0.3485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.932865560054779,
      "rewards/margins": 2.3563389778137207,
      "rewards/rejected": -3.2892045974731445,
      "step": 12912
    },
    {
      "epoch": 1.6897521877811401,
      "grad_norm": 7.572751998901367,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06800135225057602,
      "logits/rejected": -0.0781494677066803,
      "logps/chosen": -586.8822631835938,
      "logps/rejected": -563.4340209960938,
      "loss": 0.3162,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.305785596370697,
      "rewards/margins": 2.3006856441497803,
      "rewards/rejected": -1.994900107383728,
      "step": 12913
    },
    {
      "epoch": 1.689883045718492,
      "grad_norm": 7.480891227722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10278581827878952,
      "logits/rejected": -0.2505854070186615,
      "logps/chosen": -482.7231750488281,
      "logps/rejected": -595.36083984375,
      "loss": 0.2732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7350034713745117,
      "rewards/margins": 1.9072203636169434,
      "rewards/rejected": -2.642223596572876,
      "step": 12914
    },
    {
      "epoch": 1.6900139036558435,
      "grad_norm": 8.093317985534668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18110725283622742,
      "logits/rejected": -0.031194590032100677,
      "logps/chosen": -450.4757080078125,
      "logps/rejected": -465.8627624511719,
      "loss": 0.3855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4370439052581787,
      "rewards/margins": 1.735105037689209,
      "rewards/rejected": -2.172149181365967,
      "step": 12915
    },
    {
      "epoch": 1.6901447615931953,
      "grad_norm": 12.099913597106934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09315541386604309,
      "logits/rejected": -0.09695038199424744,
      "logps/chosen": -379.78973388671875,
      "logps/rejected": -562.0584106445312,
      "loss": 0.7628,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2259703874588013,
      "rewards/margins": 1.1709580421447754,
      "rewards/rejected": -2.396928548812866,
      "step": 12916
    },
    {
      "epoch": 1.690275619530547,
      "grad_norm": 11.343234062194824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1538511961698532,
      "logits/rejected": -0.3080480098724365,
      "logps/chosen": -429.9208984375,
      "logps/rejected": -467.5254821777344,
      "loss": 0.5429,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8448562026023865,
      "rewards/margins": 0.8844923973083496,
      "rewards/rejected": -1.7293486595153809,
      "step": 12917
    },
    {
      "epoch": 1.6904064774678988,
      "grad_norm": 20.50745964050293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40138283371925354,
      "logits/rejected": -0.4654591679573059,
      "logps/chosen": -482.2920837402344,
      "logps/rejected": -423.099365234375,
      "loss": 1.0018,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4842090606689453,
      "rewards/margins": 0.28094369173049927,
      "rewards/rejected": -1.7651524543762207,
      "step": 12918
    },
    {
      "epoch": 1.6905373354052506,
      "grad_norm": 11.744404792785645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38914909958839417,
      "logits/rejected": -0.04871740564703941,
      "logps/chosen": -367.2106628417969,
      "logps/rejected": -481.836669921875,
      "loss": 0.6422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3519551753997803,
      "rewards/margins": 0.8481752872467041,
      "rewards/rejected": -2.2001304626464844,
      "step": 12919
    },
    {
      "epoch": 1.6906681933426024,
      "grad_norm": 17.554346084594727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3689001202583313,
      "logits/rejected": -0.4594706892967224,
      "logps/chosen": -563.20849609375,
      "logps/rejected": -380.724853515625,
      "loss": 1.0858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0062520503997803,
      "rewards/margins": 0.70566725730896,
      "rewards/rejected": -2.7119193077087402,
      "step": 12920
    },
    {
      "epoch": 1.6907990512799542,
      "grad_norm": 11.284141540527344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07632185518741608,
      "logits/rejected": 0.15077857673168182,
      "logps/chosen": -432.424560546875,
      "logps/rejected": -481.3147888183594,
      "loss": 0.5852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8041963577270508,
      "rewards/margins": 1.6813158988952637,
      "rewards/rejected": -2.4855117797851562,
      "step": 12921
    },
    {
      "epoch": 1.690929909217306,
      "grad_norm": 13.746167182922363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25148069858551025,
      "logits/rejected": -0.040358766913414,
      "logps/chosen": -440.948974609375,
      "logps/rejected": -428.47613525390625,
      "loss": 0.835,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5555518865585327,
      "rewards/margins": 0.06948815286159515,
      "rewards/rejected": -1.625040054321289,
      "step": 12922
    },
    {
      "epoch": 1.6910607671546578,
      "grad_norm": 9.215536117553711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30688658356666565,
      "logits/rejected": -0.14557777345180511,
      "logps/chosen": -396.74261474609375,
      "logps/rejected": -463.402099609375,
      "loss": 0.6291,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0839213132858276,
      "rewards/margins": 0.6718764305114746,
      "rewards/rejected": -1.7557976245880127,
      "step": 12923
    },
    {
      "epoch": 1.6911916250920096,
      "grad_norm": 9.014655113220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24547439813613892,
      "logits/rejected": -0.1468791365623474,
      "logps/chosen": -527.372802734375,
      "logps/rejected": -631.284423828125,
      "loss": 0.412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027775391936302185,
      "rewards/margins": 1.5152812004089355,
      "rewards/rejected": -1.5430564880371094,
      "step": 12924
    },
    {
      "epoch": 1.6913224830293614,
      "grad_norm": 7.464653015136719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2328803539276123,
      "logits/rejected": -0.24473696947097778,
      "logps/chosen": -515.0178833007812,
      "logps/rejected": -445.7597961425781,
      "loss": 0.5075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0954294204711914,
      "rewards/margins": 0.7083828449249268,
      "rewards/rejected": -1.8038121461868286,
      "step": 12925
    },
    {
      "epoch": 1.691453340966713,
      "grad_norm": 11.462106704711914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.008256535977125168,
      "logits/rejected": -0.2652137875556946,
      "logps/chosen": -632.684326171875,
      "logps/rejected": -449.7171325683594,
      "loss": 0.5199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0026251077651978,
      "rewards/margins": 1.846093773841858,
      "rewards/rejected": -2.8487188816070557,
      "step": 12926
    },
    {
      "epoch": 1.6915841989040648,
      "grad_norm": 8.176702499389648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1465483158826828,
      "logits/rejected": -0.2700347900390625,
      "logps/chosen": -608.4027709960938,
      "logps/rejected": -571.1689453125,
      "loss": 0.3889,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4273326396942139,
      "rewards/margins": 1.0132927894592285,
      "rewards/rejected": -2.4406251907348633,
      "step": 12927
    },
    {
      "epoch": 1.6917150568414165,
      "grad_norm": 6.667159557342529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4855486750602722,
      "logits/rejected": -0.443045049905777,
      "logps/chosen": -521.55029296875,
      "logps/rejected": -457.56036376953125,
      "loss": 0.3216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0613878965377808,
      "rewards/margins": 1.7975139617919922,
      "rewards/rejected": -2.8589015007019043,
      "step": 12928
    },
    {
      "epoch": 1.6918459147787683,
      "grad_norm": 13.554730415344238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22886362671852112,
      "logits/rejected": -0.26423364877700806,
      "logps/chosen": -444.9123840332031,
      "logps/rejected": -438.8310241699219,
      "loss": 0.7512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.381940484046936,
      "rewards/margins": 0.839313805103302,
      "rewards/rejected": -2.221254348754883,
      "step": 12929
    },
    {
      "epoch": 1.69197677271612,
      "grad_norm": 6.530801296234131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2182057499885559,
      "logits/rejected": -0.46734619140625,
      "logps/chosen": -460.75555419921875,
      "logps/rejected": -369.562744140625,
      "loss": 0.4041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8328496813774109,
      "rewards/margins": 1.1506474018096924,
      "rewards/rejected": -1.983497142791748,
      "step": 12930
    },
    {
      "epoch": 1.6921076306534717,
      "grad_norm": 6.084815502166748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23196211457252502,
      "logits/rejected": -0.28694841265678406,
      "logps/chosen": -392.3953552246094,
      "logps/rejected": -384.23895263671875,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19159239530563354,
      "rewards/margins": 1.550349235534668,
      "rewards/rejected": -1.7419416904449463,
      "step": 12931
    },
    {
      "epoch": 1.6922384885908235,
      "grad_norm": 6.372363090515137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2816048562526703,
      "logits/rejected": -0.32754260301589966,
      "logps/chosen": -351.52557373046875,
      "logps/rejected": -344.962890625,
      "loss": 0.2532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6393028497695923,
      "rewards/margins": 1.7256603240966797,
      "rewards/rejected": -2.3649630546569824,
      "step": 12932
    },
    {
      "epoch": 1.6923693465281753,
      "grad_norm": 6.6529340744018555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16503044962882996,
      "logits/rejected": -0.08281736075878143,
      "logps/chosen": -318.60723876953125,
      "logps/rejected": -320.3114929199219,
      "loss": 0.3524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1529003232717514,
      "rewards/margins": 0.9814592599868774,
      "rewards/rejected": -0.8285588622093201,
      "step": 12933
    },
    {
      "epoch": 1.692500204465527,
      "grad_norm": 7.165975093841553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07374317944049835,
      "logits/rejected": -0.07606932520866394,
      "logps/chosen": -485.3149108886719,
      "logps/rejected": -648.9630737304688,
      "loss": 0.421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3291994333267212,
      "rewards/margins": 1.4172312021255493,
      "rewards/rejected": -1.746430516242981,
      "step": 12934
    },
    {
      "epoch": 1.6926310624028789,
      "grad_norm": 10.744221687316895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2600351572036743,
      "logits/rejected": -0.36280184984207153,
      "logps/chosen": -436.95721435546875,
      "logps/rejected": -509.9677429199219,
      "loss": 0.5261,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4339187145233154,
      "rewards/margins": 1.3933874368667603,
      "rewards/rejected": -2.8273062705993652,
      "step": 12935
    },
    {
      "epoch": 1.6927619203402307,
      "grad_norm": 9.964649200439453,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03931861370801926,
      "logits/rejected": 0.031069371849298477,
      "logps/chosen": -364.75860595703125,
      "logps/rejected": -396.238525390625,
      "loss": 0.4184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5683441162109375,
      "rewards/margins": 1.5091631412506104,
      "rewards/rejected": -2.077507257461548,
      "step": 12936
    },
    {
      "epoch": 1.6928927782775824,
      "grad_norm": 11.045342445373535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20477721095085144,
      "logits/rejected": 0.13157282769680023,
      "logps/chosen": -456.72467041015625,
      "logps/rejected": -352.270751953125,
      "loss": 0.5539,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49414730072021484,
      "rewards/margins": 0.8675233125686646,
      "rewards/rejected": -1.361670732498169,
      "step": 12937
    },
    {
      "epoch": 1.6930236362149342,
      "grad_norm": 11.647915840148926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13854797184467316,
      "logits/rejected": -0.20117224752902985,
      "logps/chosen": -474.70947265625,
      "logps/rejected": -358.4241943359375,
      "loss": 0.4939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6814991235733032,
      "rewards/margins": 1.297469139099121,
      "rewards/rejected": -1.9789683818817139,
      "step": 12938
    },
    {
      "epoch": 1.693154494152286,
      "grad_norm": 10.046497344970703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.004477962851524353,
      "logits/rejected": -0.30771398544311523,
      "logps/chosen": -549.0155639648438,
      "logps/rejected": -525.0966186523438,
      "loss": 0.8033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7558265328407288,
      "rewards/margins": 2.3884334564208984,
      "rewards/rejected": -3.1442601680755615,
      "step": 12939
    },
    {
      "epoch": 1.6932853520896378,
      "grad_norm": 11.916701316833496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17762194573879242,
      "logits/rejected": -0.42741715908050537,
      "logps/chosen": -493.6746826171875,
      "logps/rejected": -503.4386901855469,
      "loss": 0.604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6715139150619507,
      "rewards/margins": 0.9759538173675537,
      "rewards/rejected": -2.647467851638794,
      "step": 12940
    },
    {
      "epoch": 1.6934162100269896,
      "grad_norm": 10.527252197265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024821331724524498,
      "logits/rejected": -0.2654728591442108,
      "logps/chosen": -572.508544921875,
      "logps/rejected": -390.5436096191406,
      "loss": 0.5391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4874693751335144,
      "rewards/margins": 1.3803441524505615,
      "rewards/rejected": -1.8678133487701416,
      "step": 12941
    },
    {
      "epoch": 1.6935470679643412,
      "grad_norm": 7.398212909698486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18190990388393402,
      "logits/rejected": -0.23742364346981049,
      "logps/chosen": -448.7091064453125,
      "logps/rejected": -510.2173156738281,
      "loss": 0.4645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9894765615463257,
      "rewards/margins": 1.0722951889038086,
      "rewards/rejected": -2.0617716312408447,
      "step": 12942
    },
    {
      "epoch": 1.693677925901693,
      "grad_norm": 10.475320816040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07188793271780014,
      "logits/rejected": -0.1480041891336441,
      "logps/chosen": -642.5515747070312,
      "logps/rejected": -473.6746826171875,
      "loss": 0.5434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7408180236816406,
      "rewards/margins": 1.2598206996917725,
      "rewards/rejected": -2.000638723373413,
      "step": 12943
    },
    {
      "epoch": 1.6938087838390448,
      "grad_norm": 12.037027359008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012633740901947021,
      "logits/rejected": 0.10991404950618744,
      "logps/chosen": -261.79278564453125,
      "logps/rejected": -386.996826171875,
      "loss": 0.7714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.282741904258728,
      "rewards/margins": 0.013771042227745056,
      "rewards/rejected": -1.2965129613876343,
      "step": 12944
    },
    {
      "epoch": 1.6939396417763966,
      "grad_norm": 8.26731014251709,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11802096664905548,
      "logits/rejected": -0.011898268014192581,
      "logps/chosen": -681.94384765625,
      "logps/rejected": -524.122314453125,
      "loss": 0.4503,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04386254400014877,
      "rewards/margins": 1.3355438709259033,
      "rewards/rejected": -1.3794063329696655,
      "step": 12945
    },
    {
      "epoch": 1.6940704997137481,
      "grad_norm": 6.451496124267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09759946912527084,
      "logits/rejected": -0.2407519519329071,
      "logps/chosen": -461.91363525390625,
      "logps/rejected": -384.86572265625,
      "loss": 0.3591,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6891267895698547,
      "rewards/margins": 1.2418293952941895,
      "rewards/rejected": -1.9309561252593994,
      "step": 12946
    },
    {
      "epoch": 1.6942013576511,
      "grad_norm": 7.505417346954346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17830759286880493,
      "logits/rejected": -0.2689027190208435,
      "logps/chosen": -375.88775634765625,
      "logps/rejected": -386.2899169921875,
      "loss": 0.5029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11150246858596802,
      "rewards/margins": 1.074233889579773,
      "rewards/rejected": -1.1857362985610962,
      "step": 12947
    },
    {
      "epoch": 1.6943322155884517,
      "grad_norm": 9.867347717285156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08675824105739594,
      "logits/rejected": -0.0861644595861435,
      "logps/chosen": -683.2232666015625,
      "logps/rejected": -675.7631225585938,
      "loss": 0.4143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6265392303466797,
      "rewards/margins": 1.5886125564575195,
      "rewards/rejected": -2.215151786804199,
      "step": 12948
    },
    {
      "epoch": 1.6944630735258035,
      "grad_norm": 8.406655311584473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03828302025794983,
      "logits/rejected": -0.0004001148045063019,
      "logps/chosen": -544.8485717773438,
      "logps/rejected": -474.041748046875,
      "loss": 0.2917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5114032626152039,
      "rewards/margins": 1.686221718788147,
      "rewards/rejected": -2.197624921798706,
      "step": 12949
    },
    {
      "epoch": 1.6945939314631553,
      "grad_norm": 5.606235980987549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7379243969917297,
      "logits/rejected": -0.6950773000717163,
      "logps/chosen": -425.5224914550781,
      "logps/rejected": -625.4472045898438,
      "loss": 0.1705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07359564304351807,
      "rewards/margins": 2.8518471717834473,
      "rewards/rejected": -2.925442695617676,
      "step": 12950
    },
    {
      "epoch": 1.694724789400507,
      "grad_norm": 6.5028076171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22948025166988373,
      "logits/rejected": -0.1667202264070511,
      "logps/chosen": -343.466552734375,
      "logps/rejected": -369.53204345703125,
      "loss": 0.3846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6436913013458252,
      "rewards/margins": 0.9223960638046265,
      "rewards/rejected": -2.566087484359741,
      "step": 12951
    },
    {
      "epoch": 1.6948556473378589,
      "grad_norm": 9.332953453063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01390121877193451,
      "logits/rejected": 0.04251013696193695,
      "logps/chosen": -402.13079833984375,
      "logps/rejected": -356.3056640625,
      "loss": 0.5401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7973849773406982,
      "rewards/margins": 0.6429312229156494,
      "rewards/rejected": -1.4403162002563477,
      "step": 12952
    },
    {
      "epoch": 1.6949865052752107,
      "grad_norm": 10.3407621383667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10564438253641129,
      "logits/rejected": -0.07637529820203781,
      "logps/chosen": -568.7734375,
      "logps/rejected": -538.4508056640625,
      "loss": 0.6319,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3086429834365845,
      "rewards/margins": 0.6541393399238586,
      "rewards/rejected": -0.9627823233604431,
      "step": 12953
    },
    {
      "epoch": 1.6951173632125625,
      "grad_norm": 5.45052433013916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3144783079624176,
      "logits/rejected": -0.21922898292541504,
      "logps/chosen": -435.272216796875,
      "logps/rejected": -349.6039123535156,
      "loss": 0.2772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2769782543182373,
      "rewards/margins": 1.4143166542053223,
      "rewards/rejected": -2.6912949085235596,
      "step": 12954
    },
    {
      "epoch": 1.6952482211499142,
      "grad_norm": 5.66648006439209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4689022898674011,
      "logits/rejected": -0.385403037071228,
      "logps/chosen": -346.04241943359375,
      "logps/rejected": -480.0237731933594,
      "loss": 0.279,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.34726640582084656,
      "rewards/margins": 2.3604202270507812,
      "rewards/rejected": -2.013153553009033,
      "step": 12955
    },
    {
      "epoch": 1.695379079087266,
      "grad_norm": 7.143862247467041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1378466784954071,
      "logits/rejected": -0.33339908719062805,
      "logps/chosen": -497.0607604980469,
      "logps/rejected": -456.05615234375,
      "loss": 0.3142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4106571078300476,
      "rewards/margins": 1.323090672492981,
      "rewards/rejected": -1.7337477207183838,
      "step": 12956
    },
    {
      "epoch": 1.6955099370246176,
      "grad_norm": 7.057770252227783,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08127909898757935,
      "logits/rejected": -0.09911568462848663,
      "logps/chosen": -348.56756591796875,
      "logps/rejected": -346.1934814453125,
      "loss": 0.3819,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4075685143470764,
      "rewards/margins": 1.1745164394378662,
      "rewards/rejected": -1.5820848941802979,
      "step": 12957
    },
    {
      "epoch": 1.6956407949619694,
      "grad_norm": 13.00786304473877,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19144375622272491,
      "logits/rejected": -0.038997575640678406,
      "logps/chosen": -455.1643981933594,
      "logps/rejected": -400.71490478515625,
      "loss": 0.5864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.045196056365967,
      "rewards/margins": 0.9822682738304138,
      "rewards/rejected": -3.0274643898010254,
      "step": 12958
    },
    {
      "epoch": 1.6957716528993212,
      "grad_norm": 11.68837833404541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2664906084537506,
      "logits/rejected": -0.21363519132137299,
      "logps/chosen": -374.8069152832031,
      "logps/rejected": -462.5969543457031,
      "loss": 0.4544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6844284534454346,
      "rewards/margins": 1.1709470748901367,
      "rewards/rejected": -1.8553755283355713,
      "step": 12959
    },
    {
      "epoch": 1.695902510836673,
      "grad_norm": 11.638168334960938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2836819589138031,
      "logits/rejected": -0.49269700050354004,
      "logps/chosen": -460.8841552734375,
      "logps/rejected": -493.66290283203125,
      "loss": 0.6646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5485056042671204,
      "rewards/margins": 0.7720715999603271,
      "rewards/rejected": -1.3205772638320923,
      "step": 12960
    },
    {
      "epoch": 1.6960333687740246,
      "grad_norm": 9.292374610900879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24305427074432373,
      "logits/rejected": -0.37201547622680664,
      "logps/chosen": -359.435791015625,
      "logps/rejected": -334.53704833984375,
      "loss": 0.6647,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31605303287506104,
      "rewards/margins": 0.788770318031311,
      "rewards/rejected": -1.104823350906372,
      "step": 12961
    },
    {
      "epoch": 1.6961642267113763,
      "grad_norm": 8.591014862060547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37865039706230164,
      "logits/rejected": -0.3949551582336426,
      "logps/chosen": -416.12353515625,
      "logps/rejected": -431.0701599121094,
      "loss": 0.4086,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3767731785774231,
      "rewards/margins": 1.3595943450927734,
      "rewards/rejected": -1.7363678216934204,
      "step": 12962
    },
    {
      "epoch": 1.6962950846487281,
      "grad_norm": 6.7514872550964355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03789254277944565,
      "logits/rejected": -0.2690565586090088,
      "logps/chosen": -417.7994689941406,
      "logps/rejected": -344.41290283203125,
      "loss": 0.3624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13566435873508453,
      "rewards/margins": 1.5381799936294556,
      "rewards/rejected": -1.673844337463379,
      "step": 12963
    },
    {
      "epoch": 1.69642594258608,
      "grad_norm": 4.6760663986206055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17734138667583466,
      "logits/rejected": -0.4052734673023224,
      "logps/chosen": -364.11395263671875,
      "logps/rejected": -433.8706359863281,
      "loss": 0.2017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.09708571434021,
      "rewards/margins": 2.046659469604492,
      "rewards/rejected": -3.1437454223632812,
      "step": 12964
    },
    {
      "epoch": 1.6965568005234317,
      "grad_norm": 7.503209590911865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0739087462425232,
      "logits/rejected": -0.12569990754127502,
      "logps/chosen": -290.08331298828125,
      "logps/rejected": -315.9937744140625,
      "loss": 0.4964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17299816012382507,
      "rewards/margins": 0.9123803377151489,
      "rewards/rejected": -1.0853785276412964,
      "step": 12965
    },
    {
      "epoch": 1.6966876584607835,
      "grad_norm": 8.335604667663574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06939145177602768,
      "logits/rejected": 0.14506074786186218,
      "logps/chosen": -448.0140686035156,
      "logps/rejected": -399.9149169921875,
      "loss": 0.3973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.500370979309082,
      "rewards/margins": 0.9003146886825562,
      "rewards/rejected": -1.4006857872009277,
      "step": 12966
    },
    {
      "epoch": 1.6968185163981353,
      "grad_norm": 15.999459266662598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07104478031396866,
      "logits/rejected": -0.10786853730678558,
      "logps/chosen": -594.05126953125,
      "logps/rejected": -484.27325439453125,
      "loss": 1.0335,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5919045209884644,
      "rewards/margins": -0.08498537540435791,
      "rewards/rejected": -0.5069190859794617,
      "step": 12967
    },
    {
      "epoch": 1.696949374335487,
      "grad_norm": 10.38353157043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2026461660861969,
      "logits/rejected": -0.20747804641723633,
      "logps/chosen": -413.4867858886719,
      "logps/rejected": -355.34503173828125,
      "loss": 0.4725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5093460083007812,
      "rewards/margins": 1.1157011985778809,
      "rewards/rejected": -1.625047206878662,
      "step": 12968
    },
    {
      "epoch": 1.6970802322728389,
      "grad_norm": 7.607278823852539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10350304841995239,
      "logits/rejected": -0.23933660984039307,
      "logps/chosen": -404.7327880859375,
      "logps/rejected": -365.6794738769531,
      "loss": 0.3173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35643333196640015,
      "rewards/margins": 2.2307534217834473,
      "rewards/rejected": -2.587186813354492,
      "step": 12969
    },
    {
      "epoch": 1.6972110902101907,
      "grad_norm": 8.616212844848633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33080875873565674,
      "logits/rejected": -0.4499222934246063,
      "logps/chosen": -666.3221435546875,
      "logps/rejected": -544.32958984375,
      "loss": 0.454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2830566167831421,
      "rewards/margins": 1.3649466037750244,
      "rewards/rejected": -1.0818899869918823,
      "step": 12970
    },
    {
      "epoch": 1.6973419481475425,
      "grad_norm": 10.093949317932129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21900813281536102,
      "logits/rejected": -0.35023024678230286,
      "logps/chosen": -479.13958740234375,
      "logps/rejected": -449.1956481933594,
      "loss": 0.4927,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4983108937740326,
      "rewards/margins": 1.3923676013946533,
      "rewards/rejected": -1.8906785249710083,
      "step": 12971
    },
    {
      "epoch": 1.697472806084894,
      "grad_norm": 10.005091667175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.024531275033950806,
      "logits/rejected": -0.18842023611068726,
      "logps/chosen": -514.8634643554688,
      "logps/rejected": -687.4378051757812,
      "loss": 0.5184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35142838954925537,
      "rewards/margins": 0.8890547156333923,
      "rewards/rejected": -1.240483045578003,
      "step": 12972
    },
    {
      "epoch": 1.6976036640222458,
      "grad_norm": 10.899272918701172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1943844109773636,
      "logits/rejected": -0.40284281969070435,
      "logps/chosen": -388.31060791015625,
      "logps/rejected": -398.8824462890625,
      "loss": 0.5175,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38294750452041626,
      "rewards/margins": 0.9091335535049438,
      "rewards/rejected": -1.2920811176300049,
      "step": 12973
    },
    {
      "epoch": 1.6977345219595976,
      "grad_norm": 5.6234846115112305,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021633530035614967,
      "logits/rejected": 0.08370135724544525,
      "logps/chosen": -400.8890686035156,
      "logps/rejected": -393.5281982421875,
      "loss": 0.2899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35183465480804443,
      "rewards/margins": 1.8301502466201782,
      "rewards/rejected": -1.4783153533935547,
      "step": 12974
    },
    {
      "epoch": 1.6978653798969494,
      "grad_norm": 10.508967399597168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04066803678870201,
      "logits/rejected": -0.2812753915786743,
      "logps/chosen": -397.9896240234375,
      "logps/rejected": -461.05523681640625,
      "loss": 0.5041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0665249824523926,
      "rewards/margins": 1.7008328437805176,
      "rewards/rejected": -2.76735782623291,
      "step": 12975
    },
    {
      "epoch": 1.697996237834301,
      "grad_norm": 8.101550102233887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21635711193084717,
      "logits/rejected": -0.3507480025291443,
      "logps/chosen": -517.1000366210938,
      "logps/rejected": -641.0114135742188,
      "loss": 0.3495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6713300943374634,
      "rewards/margins": 1.5818639993667603,
      "rewards/rejected": -2.2531940937042236,
      "step": 12976
    },
    {
      "epoch": 1.6981270957716528,
      "grad_norm": 7.501298427581787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010716388002038002,
      "logits/rejected": -0.03724895790219307,
      "logps/chosen": -595.94970703125,
      "logps/rejected": -467.49273681640625,
      "loss": 0.3184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.421420693397522,
      "rewards/margins": 2.037091016769409,
      "rewards/rejected": -2.4585118293762207,
      "step": 12977
    },
    {
      "epoch": 1.6982579537090046,
      "grad_norm": 12.525240898132324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21544870734214783,
      "logits/rejected": -0.29312604665756226,
      "logps/chosen": -545.6505126953125,
      "logps/rejected": -533.24267578125,
      "loss": 0.7871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5548030734062195,
      "rewards/margins": 0.4464334547519684,
      "rewards/rejected": -1.0012365579605103,
      "step": 12978
    },
    {
      "epoch": 1.6983888116463564,
      "grad_norm": 9.447779655456543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014622718095779419,
      "logits/rejected": -0.33621037006378174,
      "logps/chosen": -570.4722900390625,
      "logps/rejected": -455.57470703125,
      "loss": 0.6996,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15396974980831146,
      "rewards/margins": 0.7218062877655029,
      "rewards/rejected": -0.8757761120796204,
      "step": 12979
    },
    {
      "epoch": 1.6985196695837081,
      "grad_norm": 10.888317108154297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31996282935142517,
      "logits/rejected": -0.16763530671596527,
      "logps/chosen": -510.3864440917969,
      "logps/rejected": -583.7985229492188,
      "loss": 0.5735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48111873865127563,
      "rewards/margins": 0.804700493812561,
      "rewards/rejected": -1.2858192920684814,
      "step": 12980
    },
    {
      "epoch": 1.69865052752106,
      "grad_norm": 8.534521102905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33171015977859497,
      "logits/rejected": -0.3804153800010681,
      "logps/chosen": -438.8790283203125,
      "logps/rejected": -451.2466125488281,
      "loss": 0.3323,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3319973349571228,
      "rewards/margins": 1.815791368484497,
      "rewards/rejected": -2.1477885246276855,
      "step": 12981
    },
    {
      "epoch": 1.6987813854584117,
      "grad_norm": 12.236469268798828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46441784501075745,
      "logits/rejected": -0.25308364629745483,
      "logps/chosen": -491.75518798828125,
      "logps/rejected": -500.3519287109375,
      "loss": 0.5167,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.248439908027649,
      "rewards/margins": 0.6372056603431702,
      "rewards/rejected": -1.8856456279754639,
      "step": 12982
    },
    {
      "epoch": 1.6989122433957635,
      "grad_norm": 10.210709571838379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6157230734825134,
      "logits/rejected": -0.7608325481414795,
      "logps/chosen": -468.2782287597656,
      "logps/rejected": -392.8603210449219,
      "loss": 0.5472,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6708516478538513,
      "rewards/margins": 0.9439995288848877,
      "rewards/rejected": -1.6148512363433838,
      "step": 12983
    },
    {
      "epoch": 1.6990431013331153,
      "grad_norm": 7.142552375793457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22093495726585388,
      "logits/rejected": -0.3195962905883789,
      "logps/chosen": -396.1738586425781,
      "logps/rejected": -389.22198486328125,
      "loss": 0.5078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2518994808197021,
      "rewards/margins": 1.1085573434829712,
      "rewards/rejected": -2.360456705093384,
      "step": 12984
    },
    {
      "epoch": 1.699173959270467,
      "grad_norm": 11.281036376953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1727416217327118,
      "logits/rejected": -0.03736213222146034,
      "logps/chosen": -326.70343017578125,
      "logps/rejected": -484.83148193359375,
      "loss": 0.6277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.829693615436554,
      "rewards/margins": 1.1235098838806152,
      "rewards/rejected": -1.9532033205032349,
      "step": 12985
    },
    {
      "epoch": 1.699304817207819,
      "grad_norm": 8.165084838867188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5018934011459351,
      "logits/rejected": -0.5233892202377319,
      "logps/chosen": -462.4154357910156,
      "logps/rejected": -528.5905151367188,
      "loss": 0.6232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0850855112075806,
      "rewards/margins": 0.7355440855026245,
      "rewards/rejected": -1.820629596710205,
      "step": 12986
    },
    {
      "epoch": 1.6994356751451707,
      "grad_norm": 6.586997032165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21201735734939575,
      "logits/rejected": -0.23892328143119812,
      "logps/chosen": -389.1194152832031,
      "logps/rejected": -560.4082641601562,
      "loss": 0.3623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31185382604599,
      "rewards/margins": 1.796633005142212,
      "rewards/rejected": -2.1084868907928467,
      "step": 12987
    },
    {
      "epoch": 1.6995665330825223,
      "grad_norm": 9.998041152954102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1225370466709137,
      "logits/rejected": -0.4567908048629761,
      "logps/chosen": -475.3341064453125,
      "logps/rejected": -510.4691162109375,
      "loss": 0.4245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7189567685127258,
      "rewards/margins": 2.2666032314300537,
      "rewards/rejected": -2.985560178756714,
      "step": 12988
    },
    {
      "epoch": 1.699697391019874,
      "grad_norm": 6.8647661209106445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08959968388080597,
      "logits/rejected": -0.1750139594078064,
      "logps/chosen": -447.9203186035156,
      "logps/rejected": -423.2023010253906,
      "loss": 0.2964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1478002369403839,
      "rewards/margins": 1.6469775438308716,
      "rewards/rejected": -1.4991774559020996,
      "step": 12989
    },
    {
      "epoch": 1.6998282489572258,
      "grad_norm": 6.9348955154418945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13823774456977844,
      "logits/rejected": -0.34313255548477173,
      "logps/chosen": -427.2491455078125,
      "logps/rejected": -465.2835388183594,
      "loss": 0.4323,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3272398114204407,
      "rewards/margins": 1.0057275295257568,
      "rewards/rejected": -1.3329672813415527,
      "step": 12990
    },
    {
      "epoch": 1.6999591068945774,
      "grad_norm": 7.57949686050415,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32502055168151855,
      "logits/rejected": -0.3387487232685089,
      "logps/chosen": -375.6009521484375,
      "logps/rejected": -324.50799560546875,
      "loss": 0.5778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0273261070251465,
      "rewards/margins": 0.5666061043739319,
      "rewards/rejected": -1.5939321517944336,
      "step": 12991
    },
    {
      "epoch": 1.7000899648319292,
      "grad_norm": 11.144153594970703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2965226471424103,
      "logits/rejected": -0.3273082971572876,
      "logps/chosen": -290.1155090332031,
      "logps/rejected": -423.91192626953125,
      "loss": 0.5033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.397020697593689,
      "rewards/margins": 0.864284336566925,
      "rewards/rejected": -2.261305093765259,
      "step": 12992
    },
    {
      "epoch": 1.700220822769281,
      "grad_norm": 10.380656242370605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09083063900470734,
      "logits/rejected": 0.047532349824905396,
      "logps/chosen": -556.2252197265625,
      "logps/rejected": -561.6506958007812,
      "loss": 0.6317,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5430253744125366,
      "rewards/margins": 1.0609102249145508,
      "rewards/rejected": -1.6039355993270874,
      "step": 12993
    },
    {
      "epoch": 1.7003516807066328,
      "grad_norm": 13.322340965270996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11702211201190948,
      "logits/rejected": -0.25073882937431335,
      "logps/chosen": -636.9060668945312,
      "logps/rejected": -501.8244323730469,
      "loss": 0.9493,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9829555153846741,
      "rewards/margins": -0.15094812214374542,
      "rewards/rejected": -0.8320074081420898,
      "step": 12994
    },
    {
      "epoch": 1.7004825386439846,
      "grad_norm": 7.9569411277771,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07583584636449814,
      "logits/rejected": -0.1616104394197464,
      "logps/chosen": -519.9722290039062,
      "logps/rejected": -388.9665832519531,
      "loss": 0.4434,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5976399183273315,
      "rewards/margins": 1.280790090560913,
      "rewards/rejected": -1.878429889678955,
      "step": 12995
    },
    {
      "epoch": 1.7006133965813364,
      "grad_norm": 8.837908744812012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06335572898387909,
      "logits/rejected": 0.05911778286099434,
      "logps/chosen": -673.61181640625,
      "logps/rejected": -703.887451171875,
      "loss": 0.4371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6346889138221741,
      "rewards/margins": 1.1962517499923706,
      "rewards/rejected": -1.830940842628479,
      "step": 12996
    },
    {
      "epoch": 1.7007442545186882,
      "grad_norm": 9.693175315856934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4113343060016632,
      "logits/rejected": -0.3510361313819885,
      "logps/chosen": -415.2001037597656,
      "logps/rejected": -403.5927734375,
      "loss": 0.4663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2075414657592773,
      "rewards/margins": 0.734389066696167,
      "rewards/rejected": -1.9419306516647339,
      "step": 12997
    },
    {
      "epoch": 1.70087511245604,
      "grad_norm": 8.268489837646484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19860494136810303,
      "logits/rejected": -0.07715114951133728,
      "logps/chosen": -605.4893798828125,
      "logps/rejected": -481.11297607421875,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4098684787750244,
      "rewards/margins": 0.9694399237632751,
      "rewards/rejected": -2.3793082237243652,
      "step": 12998
    },
    {
      "epoch": 1.7010059703933917,
      "grad_norm": 9.681012153625488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026760410517454147,
      "logits/rejected": -0.3123567998409271,
      "logps/chosen": -324.24420166015625,
      "logps/rejected": -580.2870483398438,
      "loss": 0.4871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.592115044593811,
      "rewards/margins": 1.1566431522369385,
      "rewards/rejected": -1.74875807762146,
      "step": 12999
    },
    {
      "epoch": 1.7011368283307435,
      "grad_norm": 4.6926045417785645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5709151029586792,
      "logits/rejected": -0.7193444967269897,
      "logps/chosen": -360.4474792480469,
      "logps/rejected": -409.5921630859375,
      "loss": 0.3216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2815096378326416,
      "rewards/margins": 1.477970004081726,
      "rewards/rejected": -2.7594797611236572,
      "step": 13000
    },
    {
      "epoch": 1.7012676862680953,
      "grad_norm": 10.38184642791748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17474761605262756,
      "logits/rejected": -0.5414652824401855,
      "logps/chosen": -361.64910888671875,
      "logps/rejected": -319.2139587402344,
      "loss": 0.6712,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3314931392669678,
      "rewards/margins": 0.8604261875152588,
      "rewards/rejected": -2.1919193267822266,
      "step": 13001
    },
    {
      "epoch": 1.7013985442054471,
      "grad_norm": 14.14680004119873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10813409835100174,
      "logits/rejected": -0.1925332397222519,
      "logps/chosen": -503.7602233886719,
      "logps/rejected": -361.0087890625,
      "loss": 0.611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8824021816253662,
      "rewards/margins": 1.2840964794158936,
      "rewards/rejected": -2.1664984226226807,
      "step": 13002
    },
    {
      "epoch": 1.7015294021427987,
      "grad_norm": 6.986914157867432,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1890566200017929,
      "logits/rejected": -0.2079308032989502,
      "logps/chosen": -383.0861511230469,
      "logps/rejected": -331.42401123046875,
      "loss": 0.3177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.264923095703125,
      "rewards/margins": 1.4201784133911133,
      "rewards/rejected": -2.6851015090942383,
      "step": 13003
    },
    {
      "epoch": 1.7016602600801505,
      "grad_norm": 4.941007137298584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06353799253702164,
      "logits/rejected": 0.09435783326625824,
      "logps/chosen": -664.5755615234375,
      "logps/rejected": -607.5579833984375,
      "loss": 0.1943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6729491949081421,
      "rewards/margins": 2.326002597808838,
      "rewards/rejected": -1.6530532836914062,
      "step": 13004
    },
    {
      "epoch": 1.7017911180175023,
      "grad_norm": 8.624250411987305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3179185092449188,
      "logits/rejected": -0.254847913980484,
      "logps/chosen": -291.47039794921875,
      "logps/rejected": -335.2611999511719,
      "loss": 0.5886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3631247282028198,
      "rewards/margins": 0.5157595276832581,
      "rewards/rejected": -1.8788844347000122,
      "step": 13005
    },
    {
      "epoch": 1.701921975954854,
      "grad_norm": 9.340312004089355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43381235003471375,
      "logits/rejected": -0.19293543696403503,
      "logps/chosen": -498.8939208984375,
      "logps/rejected": -385.69696044921875,
      "loss": 0.5756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2807332277297974,
      "rewards/margins": 0.8778207898139954,
      "rewards/rejected": -2.1585538387298584,
      "step": 13006
    },
    {
      "epoch": 1.7020528338922056,
      "grad_norm": 8.183913230895996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046246305108070374,
      "logits/rejected": 0.1456831991672516,
      "logps/chosen": -451.5880432128906,
      "logps/rejected": -416.6307373046875,
      "loss": 0.5252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1468565464019775,
      "rewards/margins": 0.7152979373931885,
      "rewards/rejected": -1.8621543645858765,
      "step": 13007
    },
    {
      "epoch": 1.7021836918295574,
      "grad_norm": 8.084223747253418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02104916051030159,
      "logits/rejected": 0.021228043362498283,
      "logps/chosen": -438.39874267578125,
      "logps/rejected": -433.29949951171875,
      "loss": 0.4047,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02836209535598755,
      "rewards/margins": 0.8996021151542664,
      "rewards/rejected": -0.9279642105102539,
      "step": 13008
    },
    {
      "epoch": 1.7023145497669092,
      "grad_norm": 6.598147869110107,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18156515061855316,
      "logits/rejected": -0.08718881756067276,
      "logps/chosen": -476.0602111816406,
      "logps/rejected": -472.10919189453125,
      "loss": 0.2406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07310158014297485,
      "rewards/margins": 1.989324927330017,
      "rewards/rejected": -2.0624265670776367,
      "step": 13009
    },
    {
      "epoch": 1.702445407704261,
      "grad_norm": 4.228578567504883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11085762083530426,
      "logits/rejected": -0.22416608035564423,
      "logps/chosen": -555.419677734375,
      "logps/rejected": -522.3408203125,
      "loss": 0.1729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3196600079536438,
      "rewards/margins": 2.1502339839935303,
      "rewards/rejected": -2.4698939323425293,
      "step": 13010
    },
    {
      "epoch": 1.7025762656416128,
      "grad_norm": 11.134570121765137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0984150841832161,
      "logits/rejected": -0.0031061479821801186,
      "logps/chosen": -498.2367248535156,
      "logps/rejected": -623.1038208007812,
      "loss": 0.7647,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0023956401273608208,
      "rewards/margins": 0.3533223867416382,
      "rewards/rejected": -0.3557180166244507,
      "step": 13011
    },
    {
      "epoch": 1.7027071235789646,
      "grad_norm": 10.984247207641602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13624262809753418,
      "logits/rejected": 0.0009286105632781982,
      "logps/chosen": -426.7792663574219,
      "logps/rejected": -346.2061462402344,
      "loss": 0.7745,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7174443006515503,
      "rewards/margins": 0.683603048324585,
      "rewards/rejected": -1.4010474681854248,
      "step": 13012
    },
    {
      "epoch": 1.7028379815163164,
      "grad_norm": 9.231694221496582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3057028353214264,
      "logits/rejected": -0.19734956324100494,
      "logps/chosen": -399.6865234375,
      "logps/rejected": -444.29541015625,
      "loss": 0.3767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.130073070526123,
      "rewards/margins": 1.5849926471710205,
      "rewards/rejected": -2.7150659561157227,
      "step": 13013
    },
    {
      "epoch": 1.7029688394536682,
      "grad_norm": 11.253401756286621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2040044367313385,
      "logits/rejected": -0.44062697887420654,
      "logps/chosen": -379.1360778808594,
      "logps/rejected": -415.1820068359375,
      "loss": 0.7166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.177156925201416,
      "rewards/margins": 0.4892765283584595,
      "rewards/rejected": -1.666433334350586,
      "step": 13014
    },
    {
      "epoch": 1.70309969739102,
      "grad_norm": 9.249348640441895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003514655865728855,
      "logits/rejected": -0.21017055213451385,
      "logps/chosen": -329.51336669921875,
      "logps/rejected": -354.5286865234375,
      "loss": 0.6473,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4077179431915283,
      "rewards/margins": 0.6586886644363403,
      "rewards/rejected": -2.066406726837158,
      "step": 13015
    },
    {
      "epoch": 1.7032305553283718,
      "grad_norm": 12.527961730957031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3496936857700348,
      "logits/rejected": -0.4316239655017853,
      "logps/chosen": -489.80255126953125,
      "logps/rejected": -418.99609375,
      "loss": 0.5883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.052358828485012054,
      "rewards/margins": 1.0934181213378906,
      "rewards/rejected": -1.1457769870758057,
      "step": 13016
    },
    {
      "epoch": 1.7033614132657235,
      "grad_norm": 7.227239608764648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09797827154397964,
      "logits/rejected": 0.06442593038082123,
      "logps/chosen": -352.8421936035156,
      "logps/rejected": -411.736572265625,
      "loss": 0.4529,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4602527618408203,
      "rewards/margins": 1.1685802936553955,
      "rewards/rejected": -1.6288331747055054,
      "step": 13017
    },
    {
      "epoch": 1.7034922712030751,
      "grad_norm": 15.911561965942383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28183290362358093,
      "logits/rejected": -0.3695181906223297,
      "logps/chosen": -371.5629577636719,
      "logps/rejected": -422.8470764160156,
      "loss": 1.042,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.9069229364395142,
      "rewards/margins": -0.04402293264865875,
      "rewards/rejected": -1.8629001379013062,
      "step": 13018
    },
    {
      "epoch": 1.703623129140427,
      "grad_norm": 14.203618049621582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14015232026576996,
      "logits/rejected": -0.25046268105506897,
      "logps/chosen": -483.8361511230469,
      "logps/rejected": -534.4825439453125,
      "loss": 0.8572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5573360919952393,
      "rewards/margins": 0.29638350009918213,
      "rewards/rejected": -0.8537195324897766,
      "step": 13019
    },
    {
      "epoch": 1.7037539870777787,
      "grad_norm": 6.5493645668029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.035392485558986664,
      "logits/rejected": -0.0020491909235715866,
      "logps/chosen": -550.43505859375,
      "logps/rejected": -488.15765380859375,
      "loss": 0.2477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0348106324672699,
      "rewards/margins": 2.10231876373291,
      "rewards/rejected": -2.137129306793213,
      "step": 13020
    },
    {
      "epoch": 1.7038848450151305,
      "grad_norm": 12.503560066223145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23442383110523224,
      "logits/rejected": -0.3275802433490753,
      "logps/chosen": -613.9493408203125,
      "logps/rejected": -429.1033935546875,
      "loss": 0.6117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9525842666625977,
      "rewards/margins": 0.4670608341693878,
      "rewards/rejected": -1.419645071029663,
      "step": 13021
    },
    {
      "epoch": 1.704015702952482,
      "grad_norm": 9.806534767150879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18958236277103424,
      "logits/rejected": -0.3875524699687958,
      "logps/chosen": -664.0256958007812,
      "logps/rejected": -777.8460693359375,
      "loss": 0.3467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2967514097690582,
      "rewards/margins": 1.0981736183166504,
      "rewards/rejected": -1.3949248790740967,
      "step": 13022
    },
    {
      "epoch": 1.7041465608898339,
      "grad_norm": 9.494457244873047,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10009095072746277,
      "logits/rejected": 0.1444454789161682,
      "logps/chosen": -433.86810302734375,
      "logps/rejected": -509.87982177734375,
      "loss": 0.45,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8617550134658813,
      "rewards/margins": 1.018541932106018,
      "rewards/rejected": -1.8802969455718994,
      "step": 13023
    },
    {
      "epoch": 1.7042774188271856,
      "grad_norm": 10.094330787658691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014849632978439331,
      "logits/rejected": -0.03484618663787842,
      "logps/chosen": -379.3257141113281,
      "logps/rejected": -489.244873046875,
      "loss": 0.5396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2622639238834381,
      "rewards/margins": 1.1831588745117188,
      "rewards/rejected": -1.445422887802124,
      "step": 13024
    },
    {
      "epoch": 1.7044082767645374,
      "grad_norm": 9.598990440368652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07575006783008575,
      "logits/rejected": -0.0994621217250824,
      "logps/chosen": -435.39776611328125,
      "logps/rejected": -444.513916015625,
      "loss": 0.5917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04998244717717171,
      "rewards/margins": 1.0309158563613892,
      "rewards/rejected": -1.0808982849121094,
      "step": 13025
    },
    {
      "epoch": 1.7045391347018892,
      "grad_norm": 11.140332221984863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13271301984786987,
      "logits/rejected": -0.06526586413383484,
      "logps/chosen": -443.0158996582031,
      "logps/rejected": -455.3612976074219,
      "loss": 0.5183,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.148951530456543,
      "rewards/margins": 0.5921825766563416,
      "rewards/rejected": -1.7411340475082397,
      "step": 13026
    },
    {
      "epoch": 1.704669992639241,
      "grad_norm": 7.555248260498047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012148329988121986,
      "logits/rejected": 0.0287428330630064,
      "logps/chosen": -446.5955810546875,
      "logps/rejected": -457.61761474609375,
      "loss": 0.4316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4978172183036804,
      "rewards/margins": 1.1704819202423096,
      "rewards/rejected": -1.6682993173599243,
      "step": 13027
    },
    {
      "epoch": 1.7048008505765928,
      "grad_norm": 11.845041275024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2534712553024292,
      "logits/rejected": -0.5179795622825623,
      "logps/chosen": -497.1925964355469,
      "logps/rejected": -444.7445373535156,
      "loss": 0.4925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.124926209449768,
      "rewards/margins": 1.0009915828704834,
      "rewards/rejected": -2.125917673110962,
      "step": 13028
    },
    {
      "epoch": 1.7049317085139446,
      "grad_norm": 10.033316612243652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08674846589565277,
      "logits/rejected": -0.25269997119903564,
      "logps/chosen": -661.2467041015625,
      "logps/rejected": -487.50726318359375,
      "loss": 0.4871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08194351196289062,
      "rewards/margins": 1.0886270999908447,
      "rewards/rejected": -1.1705706119537354,
      "step": 13029
    },
    {
      "epoch": 1.7050625664512964,
      "grad_norm": 6.385038375854492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3996858596801758,
      "logits/rejected": -0.47285011410713196,
      "logps/chosen": -529.6322021484375,
      "logps/rejected": -436.91339111328125,
      "loss": 0.2778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.410636842250824,
      "rewards/margins": 1.9624103307724,
      "rewards/rejected": -2.373047113418579,
      "step": 13030
    },
    {
      "epoch": 1.7051934243886482,
      "grad_norm": 10.852395057678223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18154466152191162,
      "logits/rejected": -0.14137160778045654,
      "logps/chosen": -623.9412841796875,
      "logps/rejected": -681.5223999023438,
      "loss": 0.4806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.788068413734436,
      "rewards/margins": 0.5626304745674133,
      "rewards/rejected": -1.3506988286972046,
      "step": 13031
    },
    {
      "epoch": 1.705324282326,
      "grad_norm": 9.717137336730957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26122623682022095,
      "logits/rejected": -0.48041149973869324,
      "logps/chosen": -412.0096435546875,
      "logps/rejected": -611.7594604492188,
      "loss": 0.4649,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3252720832824707,
      "rewards/margins": 1.525216817855835,
      "rewards/rejected": -2.8504886627197266,
      "step": 13032
    },
    {
      "epoch": 1.7054551402633518,
      "grad_norm": 7.118118762969971,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16526064276695251,
      "logits/rejected": -0.005951426923274994,
      "logps/chosen": -620.4765014648438,
      "logps/rejected": -610.9158325195312,
      "loss": 0.3945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17955592274665833,
      "rewards/margins": 1.9426429271697998,
      "rewards/rejected": -1.7630871534347534,
      "step": 13033
    },
    {
      "epoch": 1.7055859982007033,
      "grad_norm": 9.019465446472168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17587852478027344,
      "logits/rejected": -0.44311168789863586,
      "logps/chosen": -400.6383056640625,
      "logps/rejected": -469.61993408203125,
      "loss": 0.4343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6599195599555969,
      "rewards/margins": 1.809918761253357,
      "rewards/rejected": -2.4698383808135986,
      "step": 13034
    },
    {
      "epoch": 1.7057168561380551,
      "grad_norm": 14.893646240234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11743929982185364,
      "logits/rejected": -0.08581339567899704,
      "logps/chosen": -473.221923828125,
      "logps/rejected": -455.3599548339844,
      "loss": 0.9307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.62504243850708,
      "rewards/margins": 0.3360621929168701,
      "rewards/rejected": -1.9611047506332397,
      "step": 13035
    },
    {
      "epoch": 1.705847714075407,
      "grad_norm": 8.068014144897461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04034705460071564,
      "logits/rejected": -0.09135255962610245,
      "logps/chosen": -602.9039916992188,
      "logps/rejected": -533.4520263671875,
      "loss": 0.4639,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5776926279067993,
      "rewards/margins": 1.3404314517974854,
      "rewards/rejected": -1.9181240797042847,
      "step": 13036
    },
    {
      "epoch": 1.7059785720127585,
      "grad_norm": 8.577101707458496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33718323707580566,
      "logits/rejected": -0.5237166285514832,
      "logps/chosen": -305.8702392578125,
      "logps/rejected": -343.7618713378906,
      "loss": 0.3657,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2741293907165527,
      "rewards/margins": 1.2861310243606567,
      "rewards/rejected": -2.560260534286499,
      "step": 13037
    },
    {
      "epoch": 1.7061094299501103,
      "grad_norm": 11.494793891906738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4137057363986969,
      "logits/rejected": -0.45901089906692505,
      "logps/chosen": -388.5661926269531,
      "logps/rejected": -419.274658203125,
      "loss": 0.6426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0278141498565674,
      "rewards/margins": 0.813833475112915,
      "rewards/rejected": -1.8416473865509033,
      "step": 13038
    },
    {
      "epoch": 1.706240287887462,
      "grad_norm": 10.117136001586914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2585638463497162,
      "logits/rejected": -0.09875231981277466,
      "logps/chosen": -394.9581298828125,
      "logps/rejected": -509.0429382324219,
      "loss": 0.5532,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7584339380264282,
      "rewards/margins": 0.9251140356063843,
      "rewards/rejected": -1.6835479736328125,
      "step": 13039
    },
    {
      "epoch": 1.7063711458248139,
      "grad_norm": 8.969391822814941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3169195353984833,
      "logits/rejected": -0.6205809116363525,
      "logps/chosen": -460.307861328125,
      "logps/rejected": -438.3780212402344,
      "loss": 0.452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7320799231529236,
      "rewards/margins": 1.0091981887817383,
      "rewards/rejected": -1.7412781715393066,
      "step": 13040
    },
    {
      "epoch": 1.7065020037621657,
      "grad_norm": 10.309372901916504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4193906784057617,
      "logits/rejected": -0.40619778633117676,
      "logps/chosen": -481.9202575683594,
      "logps/rejected": -502.36932373046875,
      "loss": 0.6235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8853965997695923,
      "rewards/margins": 0.3051765561103821,
      "rewards/rejected": -1.1905732154846191,
      "step": 13041
    },
    {
      "epoch": 1.7066328616995174,
      "grad_norm": 9.256589889526367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22095654904842377,
      "logits/rejected": -0.25020745396614075,
      "logps/chosen": -463.60980224609375,
      "logps/rejected": -485.625,
      "loss": 0.638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2837883234024048,
      "rewards/margins": 0.44944608211517334,
      "rewards/rejected": -1.7332342863082886,
      "step": 13042
    },
    {
      "epoch": 1.7067637196368692,
      "grad_norm": 13.045904159545898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34000906348228455,
      "logits/rejected": -0.4837908446788788,
      "logps/chosen": -495.4554748535156,
      "logps/rejected": -446.066162109375,
      "loss": 0.749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.701303243637085,
      "rewards/margins": 0.4745674431324005,
      "rewards/rejected": -1.1758705377578735,
      "step": 13043
    },
    {
      "epoch": 1.706894577574221,
      "grad_norm": 10.92557430267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11080864816904068,
      "logits/rejected": -0.10024376213550568,
      "logps/chosen": -488.386962890625,
      "logps/rejected": -546.4857177734375,
      "loss": 0.6163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15397799015045166,
      "rewards/margins": 0.3375522494316101,
      "rewards/rejected": -0.4915302097797394,
      "step": 13044
    },
    {
      "epoch": 1.7070254355115728,
      "grad_norm": 10.097161293029785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12475433200597763,
      "logits/rejected": -0.018242085352540016,
      "logps/chosen": -450.3944091796875,
      "logps/rejected": -410.8033142089844,
      "loss": 0.5678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.780995786190033,
      "rewards/margins": 0.5933386087417603,
      "rewards/rejected": -1.3743343353271484,
      "step": 13045
    },
    {
      "epoch": 1.7071562934489246,
      "grad_norm": 6.462815284729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37730032205581665,
      "logits/rejected": -0.3591163754463196,
      "logps/chosen": -452.0455017089844,
      "logps/rejected": -336.0342102050781,
      "loss": 0.4491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8273053765296936,
      "rewards/margins": 1.1687328815460205,
      "rewards/rejected": -1.9960384368896484,
      "step": 13046
    },
    {
      "epoch": 1.7072871513862764,
      "grad_norm": 9.156866073608398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.012838134542107582,
      "logits/rejected": -0.1082192212343216,
      "logps/chosen": -561.152587890625,
      "logps/rejected": -513.2991943359375,
      "loss": 0.4963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9738538861274719,
      "rewards/margins": 1.1351110935211182,
      "rewards/rejected": -2.1089651584625244,
      "step": 13047
    },
    {
      "epoch": 1.7074180093236282,
      "grad_norm": 3.272120714187622,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27885904908180237,
      "logits/rejected": -0.214961975812912,
      "logps/chosen": -471.5499572753906,
      "logps/rejected": -492.4308776855469,
      "loss": 0.1199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6797380447387695,
      "rewards/margins": 2.309208631515503,
      "rewards/rejected": -2.9889469146728516,
      "step": 13048
    },
    {
      "epoch": 1.7075488672609798,
      "grad_norm": 11.098332405090332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33242976665496826,
      "logits/rejected": -0.5272336006164551,
      "logps/chosen": -437.7037048339844,
      "logps/rejected": -336.75933837890625,
      "loss": 0.6263,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7618923187255859,
      "rewards/margins": 0.6800311803817749,
      "rewards/rejected": -1.4419234991073608,
      "step": 13049
    },
    {
      "epoch": 1.7076797251983316,
      "grad_norm": 9.102592468261719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2242278903722763,
      "logits/rejected": -0.31514793634414673,
      "logps/chosen": -789.0072631835938,
      "logps/rejected": -534.2882690429688,
      "loss": 0.4419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7379152774810791,
      "rewards/margins": 0.9287551641464233,
      "rewards/rejected": -1.6666704416275024,
      "step": 13050
    },
    {
      "epoch": 1.7078105831356833,
      "grad_norm": 8.238030433654785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09227969497442245,
      "logits/rejected": -0.3430476188659668,
      "logps/chosen": -579.1904907226562,
      "logps/rejected": -561.290771484375,
      "loss": 0.2958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9002034068107605,
      "rewards/margins": 1.4582158327102661,
      "rewards/rejected": -2.3584189414978027,
      "step": 13051
    },
    {
      "epoch": 1.7079414410730351,
      "grad_norm": 11.290438652038574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26383286714553833,
      "logits/rejected": -0.3435082733631134,
      "logps/chosen": -348.7178955078125,
      "logps/rejected": -415.97265625,
      "loss": 0.6372,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8575489521026611,
      "rewards/margins": 0.3837270736694336,
      "rewards/rejected": -1.2412760257720947,
      "step": 13052
    },
    {
      "epoch": 1.7080722990103867,
      "grad_norm": 10.432562828063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13989010453224182,
      "logits/rejected": -0.2516460716724396,
      "logps/chosen": -687.9144897460938,
      "logps/rejected": -515.3089599609375,
      "loss": 0.5586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2978006601333618,
      "rewards/margins": 0.9413789510726929,
      "rewards/rejected": -2.2391796112060547,
      "step": 13053
    },
    {
      "epoch": 1.7082031569477385,
      "grad_norm": 6.599225044250488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3026455342769623,
      "logits/rejected": -0.4193999171257019,
      "logps/chosen": -511.6708984375,
      "logps/rejected": -406.1103515625,
      "loss": 0.3466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0101042985916138,
      "rewards/margins": 0.9860409498214722,
      "rewards/rejected": -1.9961451292037964,
      "step": 13054
    },
    {
      "epoch": 1.7083340148850903,
      "grad_norm": 10.13250732421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3550489842891693,
      "logits/rejected": -0.3519202768802643,
      "logps/chosen": -506.8507385253906,
      "logps/rejected": -437.68951416015625,
      "loss": 0.5952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.249433994293213,
      "rewards/margins": 0.31693100929260254,
      "rewards/rejected": -2.5663650035858154,
      "step": 13055
    },
    {
      "epoch": 1.708464872822442,
      "grad_norm": 5.451015472412109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46866172552108765,
      "logits/rejected": -0.711797297000885,
      "logps/chosen": -534.2049560546875,
      "logps/rejected": -682.5999145507812,
      "loss": 0.2253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3576136827468872,
      "rewards/margins": 1.93284010887146,
      "rewards/rejected": -3.2904539108276367,
      "step": 13056
    },
    {
      "epoch": 1.7085957307597939,
      "grad_norm": 13.577335357666016,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19233369827270508,
      "logits/rejected": -0.05266650766134262,
      "logps/chosen": -572.3828735351562,
      "logps/rejected": -478.68914794921875,
      "loss": 0.4163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41100555658340454,
      "rewards/margins": 1.6415841579437256,
      "rewards/rejected": -2.0525896549224854,
      "step": 13057
    },
    {
      "epoch": 1.7087265886971457,
      "grad_norm": 7.521006107330322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1800081431865692,
      "logits/rejected": -0.4492565095424652,
      "logps/chosen": -453.232666015625,
      "logps/rejected": -457.52197265625,
      "loss": 0.2703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3753744065761566,
      "rewards/margins": 2.535148859024048,
      "rewards/rejected": -2.9105234146118164,
      "step": 13058
    },
    {
      "epoch": 1.7088574466344975,
      "grad_norm": 8.459759712219238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46335160732269287,
      "logits/rejected": -0.5811046361923218,
      "logps/chosen": -312.44842529296875,
      "logps/rejected": -291.1619873046875,
      "loss": 0.5627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9871042966842651,
      "rewards/margins": 0.8965921401977539,
      "rewards/rejected": -1.8836963176727295,
      "step": 13059
    },
    {
      "epoch": 1.7089883045718492,
      "grad_norm": 7.643637657165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11057057231664658,
      "logits/rejected": -0.09555188566446304,
      "logps/chosen": -322.4720153808594,
      "logps/rejected": -429.99169921875,
      "loss": 0.3627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1575864851474762,
      "rewards/margins": 1.6233075857162476,
      "rewards/rejected": -1.7808940410614014,
      "step": 13060
    },
    {
      "epoch": 1.709119162509201,
      "grad_norm": 5.284961700439453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1760803461074829,
      "logits/rejected": -0.3222382068634033,
      "logps/chosen": -215.64495849609375,
      "logps/rejected": -359.5496826171875,
      "loss": 0.235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8212959170341492,
      "rewards/margins": 1.9310221672058105,
      "rewards/rejected": -2.7523183822631836,
      "step": 13061
    },
    {
      "epoch": 1.7092500204465528,
      "grad_norm": 9.13707447052002,
      "learning_rate": 5e-06,
      "logits/chosen": 0.30809083580970764,
      "logits/rejected": 0.18697777390480042,
      "logps/chosen": -722.2481689453125,
      "logps/rejected": -605.8299560546875,
      "loss": 0.4987,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10600605607032776,
      "rewards/margins": 0.9280100464820862,
      "rewards/rejected": -0.822003960609436,
      "step": 13062
    },
    {
      "epoch": 1.7093808783839046,
      "grad_norm": 4.20411491394043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47118377685546875,
      "logits/rejected": -0.8228671550750732,
      "logps/chosen": -506.62646484375,
      "logps/rejected": -351.1352233886719,
      "loss": 0.1961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1223911046981812,
      "rewards/margins": 1.7518296241760254,
      "rewards/rejected": -2.874220609664917,
      "step": 13063
    },
    {
      "epoch": 1.7095117363212562,
      "grad_norm": 7.488992214202881,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3622126281261444,
      "logits/rejected": -0.1702839583158493,
      "logps/chosen": -334.1970520019531,
      "logps/rejected": -474.47540283203125,
      "loss": 0.4582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1914970874786377,
      "rewards/margins": 1.377846121788025,
      "rewards/rejected": -2.569343090057373,
      "step": 13064
    },
    {
      "epoch": 1.709642594258608,
      "grad_norm": 7.851831436157227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12123733758926392,
      "logits/rejected": -0.33861878514289856,
      "logps/chosen": -397.4976806640625,
      "logps/rejected": -313.5102844238281,
      "loss": 0.4892,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9075119495391846,
      "rewards/margins": 0.7210185527801514,
      "rewards/rejected": -1.628530502319336,
      "step": 13065
    },
    {
      "epoch": 1.7097734521959598,
      "grad_norm": 9.691855430603027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052962999790906906,
      "logits/rejected": -0.37246501445770264,
      "logps/chosen": -478.5264892578125,
      "logps/rejected": -445.7057800292969,
      "loss": 0.58,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5194549560546875,
      "rewards/margins": 1.0771158933639526,
      "rewards/rejected": -2.596571207046509,
      "step": 13066
    },
    {
      "epoch": 1.7099043101333116,
      "grad_norm": 8.385993003845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3007659316062927,
      "logits/rejected": -0.18977981805801392,
      "logps/chosen": -330.59320068359375,
      "logps/rejected": -580.0095825195312,
      "loss": 0.3747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.846723198890686,
      "rewards/margins": 1.6572163105010986,
      "rewards/rejected": -2.503939390182495,
      "step": 13067
    },
    {
      "epoch": 1.7100351680706631,
      "grad_norm": 13.369810104370117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48648518323898315,
      "logits/rejected": -0.515784740447998,
      "logps/chosen": -443.9350891113281,
      "logps/rejected": -395.8728942871094,
      "loss": 0.837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.467125654220581,
      "rewards/margins": 0.37607574462890625,
      "rewards/rejected": -1.8432013988494873,
      "step": 13068
    },
    {
      "epoch": 1.710166026008015,
      "grad_norm": 10.922621726989746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1286146193742752,
      "logits/rejected": -0.1814575344324112,
      "logps/chosen": -686.5185546875,
      "logps/rejected": -513.3859252929688,
      "loss": 0.4929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2425949573516846,
      "rewards/margins": 1.5556472539901733,
      "rewards/rejected": -2.7982425689697266,
      "step": 13069
    },
    {
      "epoch": 1.7102968839453667,
      "grad_norm": 4.065851211547852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14113155007362366,
      "logits/rejected": -0.335316002368927,
      "logps/chosen": -764.7120361328125,
      "logps/rejected": -573.1319580078125,
      "loss": 0.1693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42766380310058594,
      "rewards/margins": 2.3481462001800537,
      "rewards/rejected": -1.9204823970794678,
      "step": 13070
    },
    {
      "epoch": 1.7104277418827185,
      "grad_norm": 9.73132038116455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02361585944890976,
      "logits/rejected": -0.1376953274011612,
      "logps/chosen": -497.6526184082031,
      "logps/rejected": -545.3975830078125,
      "loss": 0.4576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.690354824066162,
      "rewards/margins": 1.1480872631072998,
      "rewards/rejected": -2.838442087173462,
      "step": 13071
    },
    {
      "epoch": 1.7105585998200703,
      "grad_norm": 10.158912658691406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24294349551200867,
      "logits/rejected": -0.4151950180530548,
      "logps/chosen": -379.53167724609375,
      "logps/rejected": -349.4846496582031,
      "loss": 0.5319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.412867546081543,
      "rewards/margins": 1.1027032136917114,
      "rewards/rejected": -2.515570878982544,
      "step": 13072
    },
    {
      "epoch": 1.710689457757422,
      "grad_norm": 6.827920436859131,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13275665044784546,
      "logits/rejected": -0.05252072960138321,
      "logps/chosen": -605.8468627929688,
      "logps/rejected": -567.834716796875,
      "loss": 0.3494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.566341757774353,
      "rewards/margins": 1.4613454341888428,
      "rewards/rejected": -3.0276873111724854,
      "step": 13073
    },
    {
      "epoch": 1.7108203156947739,
      "grad_norm": 6.35915470123291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3913000822067261,
      "logits/rejected": -0.4448978304862976,
      "logps/chosen": -522.7453002929688,
      "logps/rejected": -609.6473388671875,
      "loss": 0.2692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8512817621231079,
      "rewards/margins": 1.755607008934021,
      "rewards/rejected": -2.606889009475708,
      "step": 13074
    },
    {
      "epoch": 1.7109511736321257,
      "grad_norm": 6.644299507141113,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12869293987751007,
      "logits/rejected": -0.13962313532829285,
      "logps/chosen": -435.6452331542969,
      "logps/rejected": -377.5293884277344,
      "loss": 0.3352,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7579647302627563,
      "rewards/margins": 1.1017894744873047,
      "rewards/rejected": -1.859754204750061,
      "step": 13075
    },
    {
      "epoch": 1.7110820315694775,
      "grad_norm": 7.202716827392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2623659372329712,
      "logits/rejected": -0.35176587104797363,
      "logps/chosen": -490.3758239746094,
      "logps/rejected": -462.8226013183594,
      "loss": 0.2534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5541534423828125,
      "rewards/margins": 2.0945684909820557,
      "rewards/rejected": -2.648721694946289,
      "step": 13076
    },
    {
      "epoch": 1.7112128895068293,
      "grad_norm": 11.5521240234375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19390803575515747,
      "logits/rejected": -0.24808278679847717,
      "logps/chosen": -469.07525634765625,
      "logps/rejected": -555.5358276367188,
      "loss": 0.4338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.883304476737976,
      "rewards/margins": 1.4172614812850952,
      "rewards/rejected": -3.3005661964416504,
      "step": 13077
    },
    {
      "epoch": 1.711343747444181,
      "grad_norm": 6.643913269042969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28283849358558655,
      "logits/rejected": -0.6487268805503845,
      "logps/chosen": -348.4131774902344,
      "logps/rejected": -334.5884094238281,
      "loss": 0.3912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7640030980110168,
      "rewards/margins": 1.5491071939468384,
      "rewards/rejected": -2.3131103515625,
      "step": 13078
    },
    {
      "epoch": 1.7114746053815326,
      "grad_norm": 17.27617835998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44072425365448,
      "logits/rejected": -0.5259609222412109,
      "logps/chosen": -412.5389099121094,
      "logps/rejected": -492.27313232421875,
      "loss": 0.8216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6106548309326172,
      "rewards/margins": 0.8900948166847229,
      "rewards/rejected": -2.5007495880126953,
      "step": 13079
    },
    {
      "epoch": 1.7116054633188844,
      "grad_norm": 7.170241832733154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5466386675834656,
      "logits/rejected": -0.4280974268913269,
      "logps/chosen": -353.6339111328125,
      "logps/rejected": -475.52001953125,
      "loss": 0.3408,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7666293978691101,
      "rewards/margins": 1.4644802808761597,
      "rewards/rejected": -2.231109619140625,
      "step": 13080
    },
    {
      "epoch": 1.7117363212562362,
      "grad_norm": 7.22778844833374,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22763022780418396,
      "logits/rejected": -0.045621804893016815,
      "logps/chosen": -355.2951965332031,
      "logps/rejected": -327.2565612792969,
      "loss": 0.3867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2583168745040894,
      "rewards/margins": 1.0840747356414795,
      "rewards/rejected": -2.3423914909362793,
      "step": 13081
    },
    {
      "epoch": 1.711867179193588,
      "grad_norm": 9.3095703125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5969657897949219,
      "logits/rejected": -0.7152764201164246,
      "logps/chosen": -481.2790832519531,
      "logps/rejected": -475.447509765625,
      "loss": 0.57,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8889778852462769,
      "rewards/margins": 0.8852180242538452,
      "rewards/rejected": -2.7741963863372803,
      "step": 13082
    },
    {
      "epoch": 1.7119980371309396,
      "grad_norm": 8.092824935913086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.413404256105423,
      "logits/rejected": -0.1092083752155304,
      "logps/chosen": -463.7454833984375,
      "logps/rejected": -574.6771240234375,
      "loss": 0.3953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8375099301338196,
      "rewards/margins": 1.170523762702942,
      "rewards/rejected": -2.0080337524414062,
      "step": 13083
    },
    {
      "epoch": 1.7121288950682914,
      "grad_norm": 7.809622764587402,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04938018321990967,
      "logits/rejected": -0.21241608262062073,
      "logps/chosen": -480.40155029296875,
      "logps/rejected": -411.368896484375,
      "loss": 0.3473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3691377341747284,
      "rewards/margins": 1.5393028259277344,
      "rewards/rejected": -1.9084405899047852,
      "step": 13084
    },
    {
      "epoch": 1.7122597530056431,
      "grad_norm": 5.9007344245910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5816848874092102,
      "logits/rejected": -0.34118521213531494,
      "logps/chosen": -366.72247314453125,
      "logps/rejected": -444.4350280761719,
      "loss": 0.2559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.933047354221344,
      "rewards/margins": 1.816835880279541,
      "rewards/rejected": -2.749882936477661,
      "step": 13085
    },
    {
      "epoch": 1.712390610942995,
      "grad_norm": 5.6493072509765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18821918964385986,
      "logits/rejected": -0.37355124950408936,
      "logps/chosen": -298.7047119140625,
      "logps/rejected": -289.70501708984375,
      "loss": 0.4312,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1774566173553467,
      "rewards/margins": 1.3142404556274414,
      "rewards/rejected": -2.491697311401367,
      "step": 13086
    },
    {
      "epoch": 1.7125214688803467,
      "grad_norm": 9.908045768737793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0861053615808487,
      "logits/rejected": -0.09986186772584915,
      "logps/chosen": -518.2410278320312,
      "logps/rejected": -359.9358215332031,
      "loss": 0.5106,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2904107570648193,
      "rewards/margins": 1.0043827295303345,
      "rewards/rejected": -2.2947933673858643,
      "step": 13087
    },
    {
      "epoch": 1.7126523268176985,
      "grad_norm": 10.372093200683594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6722562313079834,
      "logits/rejected": -0.5813416242599487,
      "logps/chosen": -450.65570068359375,
      "logps/rejected": -520.1530151367188,
      "loss": 0.5158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.094808340072632,
      "rewards/margins": 1.0140116214752197,
      "rewards/rejected": -3.1088197231292725,
      "step": 13088
    },
    {
      "epoch": 1.7127831847550503,
      "grad_norm": 9.210861206054688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18191039562225342,
      "logits/rejected": -0.08266925811767578,
      "logps/chosen": -548.602294921875,
      "logps/rejected": -635.0458984375,
      "loss": 0.4194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6683075428009033,
      "rewards/margins": 1.460937738418579,
      "rewards/rejected": -3.1292452812194824,
      "step": 13089
    },
    {
      "epoch": 1.712914042692402,
      "grad_norm": 12.389540672302246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18054775893688202,
      "logits/rejected": -0.3401753008365631,
      "logps/chosen": -634.0614013671875,
      "logps/rejected": -460.7308654785156,
      "loss": 0.7208,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.7423099279403687,
      "rewards/margins": 0.21629315614700317,
      "rewards/rejected": -0.9586030840873718,
      "step": 13090
    },
    {
      "epoch": 1.713044900629754,
      "grad_norm": 7.983173370361328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1301535665988922,
      "logits/rejected": -0.10467574000358582,
      "logps/chosen": -440.49078369140625,
      "logps/rejected": -393.58642578125,
      "loss": 0.3573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5777008533477783,
      "rewards/margins": 1.141619086265564,
      "rewards/rejected": -2.7193198204040527,
      "step": 13091
    },
    {
      "epoch": 1.7131757585671057,
      "grad_norm": 14.05327033996582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12145726382732391,
      "logits/rejected": -0.1107093021273613,
      "logps/chosen": -405.9113464355469,
      "logps/rejected": -436.8420715332031,
      "loss": 0.88,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4584567546844482,
      "rewards/margins": 0.9319068789482117,
      "rewards/rejected": -2.3903636932373047,
      "step": 13092
    },
    {
      "epoch": 1.7133066165044575,
      "grad_norm": 8.341508865356445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2942606210708618,
      "logits/rejected": -0.3079114556312561,
      "logps/chosen": -363.42535400390625,
      "logps/rejected": -457.0257873535156,
      "loss": 0.4239,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4596754312515259,
      "rewards/margins": 1.267593502998352,
      "rewards/rejected": -2.727268934249878,
      "step": 13093
    },
    {
      "epoch": 1.7134374744418093,
      "grad_norm": 6.529974937438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4460866451263428,
      "logits/rejected": -0.20338600873947144,
      "logps/chosen": -419.15557861328125,
      "logps/rejected": -450.55865478515625,
      "loss": 0.3728,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41669994592666626,
      "rewards/margins": 2.0812060832977295,
      "rewards/rejected": -2.497905969619751,
      "step": 13094
    },
    {
      "epoch": 1.7135683323791608,
      "grad_norm": 18.126453399658203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34641480445861816,
      "logits/rejected": -0.3883627951145172,
      "logps/chosen": -459.19036865234375,
      "logps/rejected": -415.6026611328125,
      "loss": 1.1965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.444790244102478,
      "rewards/margins": -0.2856212556362152,
      "rewards/rejected": -1.15916907787323,
      "step": 13095
    },
    {
      "epoch": 1.7136991903165126,
      "grad_norm": 4.9263811111450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15514011681079865,
      "logits/rejected": -0.20324817299842834,
      "logps/chosen": -601.6452026367188,
      "logps/rejected": -483.56048583984375,
      "loss": 0.2207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0204343795776367,
      "rewards/margins": 1.9666798114776611,
      "rewards/rejected": -2.987114429473877,
      "step": 13096
    },
    {
      "epoch": 1.7138300482538644,
      "grad_norm": 13.769888877868652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19086650013923645,
      "logits/rejected": -0.2446315735578537,
      "logps/chosen": -454.3780517578125,
      "logps/rejected": -493.9273681640625,
      "loss": 0.8989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4441226720809937,
      "rewards/margins": 0.31286293268203735,
      "rewards/rejected": -1.7569856643676758,
      "step": 13097
    },
    {
      "epoch": 1.7139609061912162,
      "grad_norm": 5.460052490234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11052563041448593,
      "logits/rejected": -0.13412347435951233,
      "logps/chosen": -476.94354248046875,
      "logps/rejected": -430.5887756347656,
      "loss": 0.194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6594087481498718,
      "rewards/margins": 2.149838447570801,
      "rewards/rejected": -2.8092472553253174,
      "step": 13098
    },
    {
      "epoch": 1.7140917641285678,
      "grad_norm": 8.211542129516602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0706786960363388,
      "logits/rejected": -0.0732102319598198,
      "logps/chosen": -419.3111572265625,
      "logps/rejected": -373.745849609375,
      "loss": 0.5344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.694216787815094,
      "rewards/margins": 0.9336845278739929,
      "rewards/rejected": -1.627901315689087,
      "step": 13099
    },
    {
      "epoch": 1.7142226220659196,
      "grad_norm": 8.928818702697754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11084476113319397,
      "logits/rejected": -0.23992671072483063,
      "logps/chosen": -716.2247314453125,
      "logps/rejected": -526.1984252929688,
      "loss": 0.4432,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.282818078994751,
      "rewards/margins": 0.9956772327423096,
      "rewards/rejected": -2.2784953117370605,
      "step": 13100
    },
    {
      "epoch": 1.7143534800032714,
      "grad_norm": 7.962910175323486,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12278394401073456,
      "logits/rejected": -0.09324675798416138,
      "logps/chosen": -398.001708984375,
      "logps/rejected": -397.56103515625,
      "loss": 0.4208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6416035890579224,
      "rewards/margins": 1.2419133186340332,
      "rewards/rejected": -2.883516550064087,
      "step": 13101
    },
    {
      "epoch": 1.7144843379406232,
      "grad_norm": 7.294461250305176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3804420232772827,
      "logits/rejected": -0.58043372631073,
      "logps/chosen": -491.7433776855469,
      "logps/rejected": -380.0570983886719,
      "loss": 0.5117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8766844272613525,
      "rewards/margins": 1.0020779371261597,
      "rewards/rejected": -1.8787622451782227,
      "step": 13102
    },
    {
      "epoch": 1.714615195877975,
      "grad_norm": 10.321905136108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2636585831642151,
      "logits/rejected": -0.3646871745586395,
      "logps/chosen": -582.3325805664062,
      "logps/rejected": -423.63421630859375,
      "loss": 0.6256,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2494014501571655,
      "rewards/margins": 0.6784263849258423,
      "rewards/rejected": -1.9278278350830078,
      "step": 13103
    },
    {
      "epoch": 1.7147460538153267,
      "grad_norm": 13.525321006774902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03580019623041153,
      "logits/rejected": -0.08858295530080795,
      "logps/chosen": -530.7250366210938,
      "logps/rejected": -569.2327270507812,
      "loss": 0.8358,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.855605959892273,
      "rewards/margins": -0.029567837715148926,
      "rewards/rejected": -0.8260380029678345,
      "step": 13104
    },
    {
      "epoch": 1.7148769117526785,
      "grad_norm": 11.684659004211426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20894525945186615,
      "logits/rejected": -0.1699172854423523,
      "logps/chosen": -395.1915283203125,
      "logps/rejected": -410.1950378417969,
      "loss": 0.6774,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2190883159637451,
      "rewards/margins": 0.5777389407157898,
      "rewards/rejected": -1.7968274354934692,
      "step": 13105
    },
    {
      "epoch": 1.7150077696900303,
      "grad_norm": 12.488884925842285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4164436161518097,
      "logits/rejected": -0.21818974614143372,
      "logps/chosen": -373.5211181640625,
      "logps/rejected": -506.1744384765625,
      "loss": 0.9051,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.219022512435913,
      "rewards/margins": 0.12678222358226776,
      "rewards/rejected": -2.3458046913146973,
      "step": 13106
    },
    {
      "epoch": 1.7151386276273821,
      "grad_norm": 10.286335945129395,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1892392486333847,
      "logits/rejected": -0.0604747049510479,
      "logps/chosen": -508.6558837890625,
      "logps/rejected": -360.3720703125,
      "loss": 0.5077,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4812181293964386,
      "rewards/margins": 1.1436281204223633,
      "rewards/rejected": -1.6248462200164795,
      "step": 13107
    },
    {
      "epoch": 1.715269485564734,
      "grad_norm": 10.34337043762207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16455690562725067,
      "logits/rejected": -0.4215940237045288,
      "logps/chosen": -521.955322265625,
      "logps/rejected": -473.5312805175781,
      "loss": 0.5349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1369513273239136,
      "rewards/margins": 1.1069138050079346,
      "rewards/rejected": -2.2438652515411377,
      "step": 13108
    },
    {
      "epoch": 1.7154003435020857,
      "grad_norm": 14.352118492126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4256380796432495,
      "logits/rejected": -0.2934929430484772,
      "logps/chosen": -426.1264953613281,
      "logps/rejected": -520.6238403320312,
      "loss": 0.7623,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47799432277679443,
      "rewards/margins": 0.36532366275787354,
      "rewards/rejected": -0.843317985534668,
      "step": 13109
    },
    {
      "epoch": 1.7155312014394373,
      "grad_norm": 8.625753402709961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.058326274156570435,
      "logits/rejected": -0.16708356142044067,
      "logps/chosen": -366.69732666015625,
      "logps/rejected": -395.95947265625,
      "loss": 0.5371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9452831149101257,
      "rewards/margins": 0.8219829797744751,
      "rewards/rejected": -1.767266035079956,
      "step": 13110
    },
    {
      "epoch": 1.715662059376789,
      "grad_norm": 8.769018173217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27446267008781433,
      "logits/rejected": -0.3648194670677185,
      "logps/chosen": -401.1690979003906,
      "logps/rejected": -488.71917724609375,
      "loss": 0.4241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45680999755859375,
      "rewards/margins": 1.2271838188171387,
      "rewards/rejected": -1.6839938163757324,
      "step": 13111
    },
    {
      "epoch": 1.7157929173141409,
      "grad_norm": 7.334950923919678,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36086076498031616,
      "logits/rejected": -0.2196253389120102,
      "logps/chosen": -353.16046142578125,
      "logps/rejected": -283.77532958984375,
      "loss": 0.4485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4280836582183838,
      "rewards/margins": 1.0004057884216309,
      "rewards/rejected": -2.4284894466400146,
      "step": 13112
    },
    {
      "epoch": 1.7159237752514926,
      "grad_norm": 7.696539878845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5714256167411804,
      "logits/rejected": -0.4638034701347351,
      "logps/chosen": -303.5396728515625,
      "logps/rejected": -290.3350524902344,
      "loss": 0.4722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.385200023651123,
      "rewards/margins": 1.03389573097229,
      "rewards/rejected": -2.419095754623413,
      "step": 13113
    },
    {
      "epoch": 1.7160546331888442,
      "grad_norm": 4.112638473510742,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08707976341247559,
      "logits/rejected": 0.02555951103568077,
      "logps/chosen": -476.86181640625,
      "logps/rejected": -427.7125549316406,
      "loss": 0.2112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2703189551830292,
      "rewards/margins": 2.437413454055786,
      "rewards/rejected": -2.707732677459717,
      "step": 13114
    },
    {
      "epoch": 1.716185491126196,
      "grad_norm": 12.94322681427002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2930927872657776,
      "logits/rejected": -0.010728225111961365,
      "logps/chosen": -446.25579833984375,
      "logps/rejected": -583.0032348632812,
      "loss": 0.7829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.931788444519043,
      "rewards/margins": 0.4818395972251892,
      "rewards/rejected": -1.413628101348877,
      "step": 13115
    },
    {
      "epoch": 1.7163163490635478,
      "grad_norm": 8.09354305267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2258158028125763,
      "logits/rejected": -0.09767880290746689,
      "logps/chosen": -471.5823974609375,
      "logps/rejected": -569.2103271484375,
      "loss": 0.3502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6873272657394409,
      "rewards/margins": 1.244940161705017,
      "rewards/rejected": -1.9322673082351685,
      "step": 13116
    },
    {
      "epoch": 1.7164472070008996,
      "grad_norm": 7.353014945983887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02251625806093216,
      "logits/rejected": -0.1145634800195694,
      "logps/chosen": -581.8617553710938,
      "logps/rejected": -588.7781372070312,
      "loss": 0.357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7425800561904907,
      "rewards/margins": 1.2910126447677612,
      "rewards/rejected": -2.033592700958252,
      "step": 13117
    },
    {
      "epoch": 1.7165780649382514,
      "grad_norm": 7.03633451461792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03719130530953407,
      "logits/rejected": -0.28139907121658325,
      "logps/chosen": -350.93621826171875,
      "logps/rejected": -444.5165710449219,
      "loss": 0.4449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3606891632080078,
      "rewards/margins": 1.039880394935608,
      "rewards/rejected": -2.400569438934326,
      "step": 13118
    },
    {
      "epoch": 1.7167089228756032,
      "grad_norm": 7.047537326812744,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2239498794078827,
      "logits/rejected": -0.26882869005203247,
      "logps/chosen": -393.96136474609375,
      "logps/rejected": -449.5047912597656,
      "loss": 0.2846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10405561327934265,
      "rewards/margins": 2.100123643875122,
      "rewards/rejected": -2.204179286956787,
      "step": 13119
    },
    {
      "epoch": 1.716839780812955,
      "grad_norm": 12.072134971618652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30818912386894226,
      "logits/rejected": -0.4134250283241272,
      "logps/chosen": -332.5597229003906,
      "logps/rejected": -434.74407958984375,
      "loss": 0.564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5727901458740234,
      "rewards/margins": 0.936863899230957,
      "rewards/rejected": -2.5096540451049805,
      "step": 13120
    },
    {
      "epoch": 1.7169706387503068,
      "grad_norm": 12.823139190673828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09360413998365402,
      "logits/rejected": 0.04927884787321091,
      "logps/chosen": -542.1290893554688,
      "logps/rejected": -568.63916015625,
      "loss": 0.877,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0311204195022583,
      "rewards/margins": 0.29067015647888184,
      "rewards/rejected": -1.3217906951904297,
      "step": 13121
    },
    {
      "epoch": 1.7171014966876585,
      "grad_norm": 10.076583862304688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14988377690315247,
      "logits/rejected": -0.3695383071899414,
      "logps/chosen": -426.3609619140625,
      "logps/rejected": -258.0221862792969,
      "loss": 0.3858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3748148679733276,
      "rewards/margins": 1.3460826873779297,
      "rewards/rejected": -2.7208974361419678,
      "step": 13122
    },
    {
      "epoch": 1.7172323546250103,
      "grad_norm": 10.120185852050781,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01578901708126068,
      "logits/rejected": -0.08885776251554489,
      "logps/chosen": -476.10980224609375,
      "logps/rejected": -478.8942565917969,
      "loss": 0.394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6893947124481201,
      "rewards/margins": 1.355454683303833,
      "rewards/rejected": -2.044849395751953,
      "step": 13123
    },
    {
      "epoch": 1.7173632125623621,
      "grad_norm": 15.888388633728027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20813068747520447,
      "logits/rejected": -0.08739128708839417,
      "logps/chosen": -442.92034912109375,
      "logps/rejected": -529.3721313476562,
      "loss": 1.2306,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.2358148097991943,
      "rewards/margins": -0.5265949964523315,
      "rewards/rejected": -1.7092196941375732,
      "step": 13124
    },
    {
      "epoch": 1.7174940704997137,
      "grad_norm": 9.569833755493164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08696362376213074,
      "logits/rejected": -0.006591496057808399,
      "logps/chosen": -427.0340270996094,
      "logps/rejected": -392.9397277832031,
      "loss": 0.594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7570280432701111,
      "rewards/margins": 0.9444072246551514,
      "rewards/rejected": -1.7014352083206177,
      "step": 13125
    },
    {
      "epoch": 1.7176249284370655,
      "grad_norm": 9.7215576171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23714807629585266,
      "logits/rejected": -0.32512784004211426,
      "logps/chosen": -462.6894836425781,
      "logps/rejected": -499.9254455566406,
      "loss": 0.4305,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6373462677001953,
      "rewards/margins": 1.1165218353271484,
      "rewards/rejected": -1.7538681030273438,
      "step": 13126
    },
    {
      "epoch": 1.7177557863744173,
      "grad_norm": 13.18654727935791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08868918567895889,
      "logits/rejected": -0.2668663263320923,
      "logps/chosen": -600.561767578125,
      "logps/rejected": -390.26641845703125,
      "loss": 0.735,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9572422504425049,
      "rewards/margins": 0.2780786454677582,
      "rewards/rejected": -2.235321044921875,
      "step": 13127
    },
    {
      "epoch": 1.717886644311769,
      "grad_norm": 7.949611186981201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1222531646490097,
      "logits/rejected": -0.28667983412742615,
      "logps/chosen": -379.4163818359375,
      "logps/rejected": -385.53204345703125,
      "loss": 0.3716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1778396368026733,
      "rewards/margins": 1.4917361736297607,
      "rewards/rejected": -2.6695761680603027,
      "step": 13128
    },
    {
      "epoch": 1.7180175022491206,
      "grad_norm": 9.008841514587402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06365297734737396,
      "logits/rejected": 0.034467317163944244,
      "logps/chosen": -464.2857360839844,
      "logps/rejected": -437.3216247558594,
      "loss": 0.5197,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.735795259475708,
      "rewards/margins": 0.4718500077724457,
      "rewards/rejected": -1.207645297050476,
      "step": 13129
    },
    {
      "epoch": 1.7181483601864724,
      "grad_norm": 6.986966133117676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022053267806768417,
      "logits/rejected": -0.2074752151966095,
      "logps/chosen": -349.48089599609375,
      "logps/rejected": -307.0492248535156,
      "loss": 0.2636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2122443914413452,
      "rewards/margins": 1.9478062391281128,
      "rewards/rejected": -3.160050392150879,
      "step": 13130
    },
    {
      "epoch": 1.7182792181238242,
      "grad_norm": 11.161978721618652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20009389519691467,
      "logits/rejected": 0.06498678028583527,
      "logps/chosen": -423.1462097167969,
      "logps/rejected": -468.99493408203125,
      "loss": 0.4702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3273922204971313,
      "rewards/margins": 0.7032617926597595,
      "rewards/rejected": -2.030653953552246,
      "step": 13131
    },
    {
      "epoch": 1.718410076061176,
      "grad_norm": 11.15266227722168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5411100387573242,
      "logits/rejected": -0.41308268904685974,
      "logps/chosen": -589.5242919921875,
      "logps/rejected": -536.97900390625,
      "loss": 0.6362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1319284439086914,
      "rewards/margins": 0.4634687304496765,
      "rewards/rejected": -1.5953971147537231,
      "step": 13132
    },
    {
      "epoch": 1.7185409339985278,
      "grad_norm": 4.1096577644348145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13535186648368835,
      "logits/rejected": -0.24175500869750977,
      "logps/chosen": -657.5635986328125,
      "logps/rejected": -536.1480102539062,
      "loss": 0.1462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24295252561569214,
      "rewards/margins": 2.6669094562530518,
      "rewards/rejected": -2.9098620414733887,
      "step": 13133
    },
    {
      "epoch": 1.7186717919358796,
      "grad_norm": 11.301602363586426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05760737136006355,
      "logits/rejected": -0.26049521565437317,
      "logps/chosen": -530.1305541992188,
      "logps/rejected": -432.9930725097656,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8984375,
      "rewards/margins": 0.28706592321395874,
      "rewards/rejected": -1.185503363609314,
      "step": 13134
    },
    {
      "epoch": 1.7188026498732314,
      "grad_norm": 8.49205493927002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23979604244232178,
      "logits/rejected": -0.4711465835571289,
      "logps/chosen": -464.0150146484375,
      "logps/rejected": -371.3393249511719,
      "loss": 0.4235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.618313193321228,
      "rewards/margins": 1.3016228675842285,
      "rewards/rejected": -1.919935941696167,
      "step": 13135
    },
    {
      "epoch": 1.7189335078105832,
      "grad_norm": 7.796741485595703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21673861145973206,
      "logits/rejected": -0.0986344963312149,
      "logps/chosen": -363.17578125,
      "logps/rejected": -440.0113525390625,
      "loss": 0.4136,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3975500166416168,
      "rewards/margins": 1.5159847736358643,
      "rewards/rejected": -1.9135347604751587,
      "step": 13136
    },
    {
      "epoch": 1.719064365747935,
      "grad_norm": 12.340516090393066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35311317443847656,
      "logits/rejected": -0.23993641138076782,
      "logps/chosen": -402.9627685546875,
      "logps/rejected": -407.5073547363281,
      "loss": 0.7333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0005178451538086,
      "rewards/margins": 0.35220566391944885,
      "rewards/rejected": -2.3527233600616455,
      "step": 13137
    },
    {
      "epoch": 1.7191952236852868,
      "grad_norm": 14.362066268920898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.198600172996521,
      "logits/rejected": -0.27142664790153503,
      "logps/chosen": -544.4544067382812,
      "logps/rejected": -571.4012451171875,
      "loss": 0.707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4791580438613892,
      "rewards/margins": 0.7485402822494507,
      "rewards/rejected": -2.22769832611084,
      "step": 13138
    },
    {
      "epoch": 1.7193260816226386,
      "grad_norm": 6.638121128082275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002262171357870102,
      "logits/rejected": -0.08252622187137604,
      "logps/chosen": -549.4583129882812,
      "logps/rejected": -552.2738037109375,
      "loss": 0.296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9563441872596741,
      "rewards/margins": 1.7590850591659546,
      "rewards/rejected": -2.7154293060302734,
      "step": 13139
    },
    {
      "epoch": 1.7194569395599903,
      "grad_norm": 8.49116325378418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.061745744198560715,
      "logits/rejected": -0.08228403329849243,
      "logps/chosen": -454.76080322265625,
      "logps/rejected": -465.20050048828125,
      "loss": 0.4908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22338221967220306,
      "rewards/margins": 0.9924516081809998,
      "rewards/rejected": -1.2158339023590088,
      "step": 13140
    },
    {
      "epoch": 1.719587797497342,
      "grad_norm": 5.272050857543945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48400723934173584,
      "logits/rejected": -0.32236817479133606,
      "logps/chosen": -451.1695556640625,
      "logps/rejected": -540.9737548828125,
      "loss": 0.2444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4402141571044922,
      "rewards/margins": 1.5003008842468262,
      "rewards/rejected": -1.9405150413513184,
      "step": 13141
    },
    {
      "epoch": 1.7197186554346937,
      "grad_norm": 6.38944673538208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18923626840114594,
      "logits/rejected": -0.09032928198575974,
      "logps/chosen": -299.021240234375,
      "logps/rejected": -368.432373046875,
      "loss": 0.4259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8167724609375,
      "rewards/margins": 1.7467738389968872,
      "rewards/rejected": -2.5635461807250977,
      "step": 13142
    },
    {
      "epoch": 1.7198495133720455,
      "grad_norm": 7.6158952713012695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14166900515556335,
      "logits/rejected": -0.01859450154006481,
      "logps/chosen": -421.31829833984375,
      "logps/rejected": -422.5377197265625,
      "loss": 0.3883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2897157669067383,
      "rewards/margins": 1.3552422523498535,
      "rewards/rejected": -1.6449581384658813,
      "step": 13143
    },
    {
      "epoch": 1.719980371309397,
      "grad_norm": 10.999672889709473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11099602282047272,
      "logits/rejected": -0.2040131837129593,
      "logps/chosen": -540.3744506835938,
      "logps/rejected": -428.5931701660156,
      "loss": 0.5494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0048080682754517,
      "rewards/margins": 0.837552011013031,
      "rewards/rejected": -1.8423601388931274,
      "step": 13144
    },
    {
      "epoch": 1.7201112292467489,
      "grad_norm": 12.039266586303711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11129193753004074,
      "logits/rejected": -0.37205901741981506,
      "logps/chosen": -515.38037109375,
      "logps/rejected": -367.4983215332031,
      "loss": 0.7596,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1013174057006836,
      "rewards/margins": 0.16927069425582886,
      "rewards/rejected": -1.2705880403518677,
      "step": 13145
    },
    {
      "epoch": 1.7202420871841007,
      "grad_norm": 10.345171928405762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.002251170575618744,
      "logits/rejected": -0.0020459704101085663,
      "logps/chosen": -375.3356018066406,
      "logps/rejected": -375.2190856933594,
      "loss": 0.5703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.133603811264038,
      "rewards/margins": 1.2235978841781616,
      "rewards/rejected": -2.3572018146514893,
      "step": 13146
    },
    {
      "epoch": 1.7203729451214524,
      "grad_norm": 6.923626899719238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17515113949775696,
      "logits/rejected": -0.07673707604408264,
      "logps/chosen": -387.5841064453125,
      "logps/rejected": -428.67535400390625,
      "loss": 0.3921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39207959175109863,
      "rewards/margins": 1.1177386045455933,
      "rewards/rejected": -1.5098180770874023,
      "step": 13147
    },
    {
      "epoch": 1.7205038030588042,
      "grad_norm": 12.17393970489502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2923366129398346,
      "logits/rejected": -0.19033020734786987,
      "logps/chosen": -549.9750366210938,
      "logps/rejected": -529.4185180664062,
      "loss": 0.8141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4949191808700562,
      "rewards/margins": 0.15156477689743042,
      "rewards/rejected": -1.646484136581421,
      "step": 13148
    },
    {
      "epoch": 1.720634660996156,
      "grad_norm": 10.506256103515625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0019820332527160645,
      "logits/rejected": -0.07788311690092087,
      "logps/chosen": -481.3057861328125,
      "logps/rejected": -459.1197509765625,
      "loss": 0.5135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42847558856010437,
      "rewards/margins": 1.0634024143218994,
      "rewards/rejected": -1.4918780326843262,
      "step": 13149
    },
    {
      "epoch": 1.7207655189335078,
      "grad_norm": 11.848230361938477,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12458939105272293,
      "logits/rejected": -0.06877483427524567,
      "logps/chosen": -498.60693359375,
      "logps/rejected": -480.15435791015625,
      "loss": 0.4479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.663242757320404,
      "rewards/margins": 0.8677979111671448,
      "rewards/rejected": -1.5310405492782593,
      "step": 13150
    },
    {
      "epoch": 1.7208963768708596,
      "grad_norm": 9.76269245147705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.33148640394210815,
      "logits/rejected": 0.10785410553216934,
      "logps/chosen": -798.639404296875,
      "logps/rejected": -455.45782470703125,
      "loss": 0.3461,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08015173673629761,
      "rewards/margins": 1.5477344989776611,
      "rewards/rejected": -1.627886176109314,
      "step": 13151
    },
    {
      "epoch": 1.7210272348082114,
      "grad_norm": 7.2443318367004395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35208091139793396,
      "logits/rejected": -0.4095860421657562,
      "logps/chosen": -369.08135986328125,
      "logps/rejected": -316.7012939453125,
      "loss": 0.4766,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2989660501480103,
      "rewards/margins": 1.2326799631118774,
      "rewards/rejected": -2.5316460132598877,
      "step": 13152
    },
    {
      "epoch": 1.7211580927455632,
      "grad_norm": 5.566082000732422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2525584399700165,
      "logits/rejected": -0.3224171996116638,
      "logps/chosen": -416.0157470703125,
      "logps/rejected": -463.4610290527344,
      "loss": 0.192,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25263193249702454,
      "rewards/margins": 2.420236587524414,
      "rewards/rejected": -2.672868490219116,
      "step": 13153
    },
    {
      "epoch": 1.721288950682915,
      "grad_norm": 5.53842306137085,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3903430104255676,
      "logits/rejected": -0.28732791543006897,
      "logps/chosen": -326.8426513671875,
      "logps/rejected": -448.38824462890625,
      "loss": 0.2341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3415238857269287,
      "rewards/margins": 1.8006987571716309,
      "rewards/rejected": -3.1422226428985596,
      "step": 13154
    },
    {
      "epoch": 1.7214198086202668,
      "grad_norm": 8.869694709777832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07723764330148697,
      "logits/rejected": -0.24196577072143555,
      "logps/chosen": -526.7708740234375,
      "logps/rejected": -474.64105224609375,
      "loss": 0.4052,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6747661828994751,
      "rewards/margins": 1.2558354139328003,
      "rewards/rejected": -1.9306014776229858,
      "step": 13155
    },
    {
      "epoch": 1.7215506665576183,
      "grad_norm": 10.069478034973145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31522679328918457,
      "logits/rejected": -0.40463143587112427,
      "logps/chosen": -592.2340698242188,
      "logps/rejected": -489.5386657714844,
      "loss": 0.5483,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7655385732650757,
      "rewards/margins": 0.6478646993637085,
      "rewards/rejected": -1.4134032726287842,
      "step": 13156
    },
    {
      "epoch": 1.7216815244949701,
      "grad_norm": 8.003122329711914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09821340441703796,
      "logits/rejected": -0.21243169903755188,
      "logps/chosen": -330.1581115722656,
      "logps/rejected": -399.1442565917969,
      "loss": 0.4934,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5856494903564453,
      "rewards/margins": 1.3766310214996338,
      "rewards/rejected": -2.962280511856079,
      "step": 13157
    },
    {
      "epoch": 1.721812382432322,
      "grad_norm": 5.760653972625732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2877683639526367,
      "logits/rejected": -0.1844894289970398,
      "logps/chosen": -444.32696533203125,
      "logps/rejected": -427.2832336425781,
      "loss": 0.3396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7447851300239563,
      "rewards/margins": 1.585856556892395,
      "rewards/rejected": -2.330641746520996,
      "step": 13158
    },
    {
      "epoch": 1.7219432403696737,
      "grad_norm": 7.244626998901367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1536826193332672,
      "logits/rejected": -0.12607818841934204,
      "logps/chosen": -347.5413818359375,
      "logps/rejected": -381.0408020019531,
      "loss": 0.4811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9835443496704102,
      "rewards/margins": 1.0706634521484375,
      "rewards/rejected": -2.0542078018188477,
      "step": 13159
    },
    {
      "epoch": 1.7220740983070253,
      "grad_norm": 11.2144775390625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10269278287887573,
      "logits/rejected": -0.17574557662010193,
      "logps/chosen": -588.4425048828125,
      "logps/rejected": -507.433837890625,
      "loss": 0.3294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7922104597091675,
      "rewards/margins": 1.785455346107483,
      "rewards/rejected": -2.5776658058166504,
      "step": 13160
    },
    {
      "epoch": 1.722204956244377,
      "grad_norm": 11.978409767150879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18896529078483582,
      "logits/rejected": -0.2043685019016266,
      "logps/chosen": -433.83575439453125,
      "logps/rejected": -438.2660217285156,
      "loss": 0.6156,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.705447793006897,
      "rewards/margins": 1.122839331626892,
      "rewards/rejected": -1.8282872438430786,
      "step": 13161
    },
    {
      "epoch": 1.7223358141817289,
      "grad_norm": 1.9894226789474487,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04210006073117256,
      "logits/rejected": -0.19764119386672974,
      "logps/chosen": -471.33416748046875,
      "logps/rejected": -535.7266845703125,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7126508355140686,
      "rewards/margins": 3.0411744117736816,
      "rewards/rejected": -3.7538254261016846,
      "step": 13162
    },
    {
      "epoch": 1.7224666721190807,
      "grad_norm": 5.445197105407715,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08180686831474304,
      "logits/rejected": -0.044367700815200806,
      "logps/chosen": -283.2729797363281,
      "logps/rejected": -414.62811279296875,
      "loss": 0.2728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.30550387501716614,
      "rewards/margins": 1.804068922996521,
      "rewards/rejected": -2.1095728874206543,
      "step": 13163
    },
    {
      "epoch": 1.7225975300564325,
      "grad_norm": 12.0480318069458,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06513921171426773,
      "logits/rejected": -0.18646034598350525,
      "logps/chosen": -422.3831481933594,
      "logps/rejected": -604.2333374023438,
      "loss": 0.6125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2379926443099976,
      "rewards/margins": 0.7499949932098389,
      "rewards/rejected": -1.9879875183105469,
      "step": 13164
    },
    {
      "epoch": 1.7227283879937842,
      "grad_norm": 9.283296585083008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0617256835103035,
      "logits/rejected": 0.08853185176849365,
      "logps/chosen": -438.0528564453125,
      "logps/rejected": -417.16131591796875,
      "loss": 0.507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7570233345031738,
      "rewards/margins": 1.1306564807891846,
      "rewards/rejected": -1.8876798152923584,
      "step": 13165
    },
    {
      "epoch": 1.722859245931136,
      "grad_norm": 12.211005210876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13613885641098022,
      "logits/rejected": -0.21597349643707275,
      "logps/chosen": -575.9091796875,
      "logps/rejected": -584.8316040039062,
      "loss": 0.7956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6291652917861938,
      "rewards/margins": 0.6926388740539551,
      "rewards/rejected": -2.3218040466308594,
      "step": 13166
    },
    {
      "epoch": 1.7229901038684878,
      "grad_norm": 11.314459800720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6928924322128296,
      "logits/rejected": -0.5299263596534729,
      "logps/chosen": -385.83709716796875,
      "logps/rejected": -411.6556396484375,
      "loss": 0.6137,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2067992687225342,
      "rewards/margins": 0.6112525463104248,
      "rewards/rejected": -1.8180519342422485,
      "step": 13167
    },
    {
      "epoch": 1.7231209618058396,
      "grad_norm": 12.203463554382324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08415591716766357,
      "logits/rejected": -0.10842277109622955,
      "logps/chosen": -594.6142578125,
      "logps/rejected": -449.3987731933594,
      "loss": 0.5981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0316647291183472,
      "rewards/margins": 1.2247722148895264,
      "rewards/rejected": -2.256437063217163,
      "step": 13168
    },
    {
      "epoch": 1.7232518197431914,
      "grad_norm": 13.302685737609863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21972249448299408,
      "logits/rejected": -0.5629613995552063,
      "logps/chosen": -443.13385009765625,
      "logps/rejected": -433.7419738769531,
      "loss": 0.816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.79606032371521,
      "rewards/margins": 0.07968635857105255,
      "rewards/rejected": -1.8757466077804565,
      "step": 13169
    },
    {
      "epoch": 1.7233826776805432,
      "grad_norm": 9.965692520141602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4329812526702881,
      "logits/rejected": -0.5896212458610535,
      "logps/chosen": -357.86456298828125,
      "logps/rejected": -289.5089111328125,
      "loss": 0.5212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.648667335510254,
      "rewards/margins": 1.1877355575561523,
      "rewards/rejected": -2.8364028930664062,
      "step": 13170
    },
    {
      "epoch": 1.7235135356178948,
      "grad_norm": 4.719830513000488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.028555624186992645,
      "logits/rejected": -0.014698661863803864,
      "logps/chosen": -520.6163940429688,
      "logps/rejected": -654.0064086914062,
      "loss": 0.1924,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.036301612854003906,
      "rewards/margins": 2.410079002380371,
      "rewards/rejected": -2.373777151107788,
      "step": 13171
    },
    {
      "epoch": 1.7236443935552466,
      "grad_norm": 7.636299133300781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2949204742908478,
      "logits/rejected": -0.39578741788864136,
      "logps/chosen": -528.6084594726562,
      "logps/rejected": -533.8270874023438,
      "loss": 0.3173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6375975012779236,
      "rewards/margins": 1.5730197429656982,
      "rewards/rejected": -2.2106175422668457,
      "step": 13172
    },
    {
      "epoch": 1.7237752514925984,
      "grad_norm": 6.600832462310791,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10121825337409973,
      "logits/rejected": 0.006795637309551239,
      "logps/chosen": -416.34930419921875,
      "logps/rejected": -473.057861328125,
      "loss": 0.315,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9259004592895508,
      "rewards/margins": 1.3820545673370361,
      "rewards/rejected": -2.307955026626587,
      "step": 13173
    },
    {
      "epoch": 1.7239061094299502,
      "grad_norm": 6.526741027832031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09877780079841614,
      "logits/rejected": -0.09266134351491928,
      "logps/chosen": -489.1815185546875,
      "logps/rejected": -453.20538330078125,
      "loss": 0.3318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5546762943267822,
      "rewards/margins": 1.4506924152374268,
      "rewards/rejected": -2.005368709564209,
      "step": 13174
    },
    {
      "epoch": 1.7240369673673017,
      "grad_norm": 9.272705078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1411246359348297,
      "logits/rejected": -0.07106894254684448,
      "logps/chosen": -323.2426452636719,
      "logps/rejected": -301.6675720214844,
      "loss": 0.632,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7679949998855591,
      "rewards/margins": 0.3360935151576996,
      "rewards/rejected": -1.1040886640548706,
      "step": 13175
    },
    {
      "epoch": 1.7241678253046535,
      "grad_norm": 8.442968368530273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06518399715423584,
      "logits/rejected": -0.27019551396369934,
      "logps/chosen": -548.7111206054688,
      "logps/rejected": -379.2034912109375,
      "loss": 0.5298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04038812592625618,
      "rewards/margins": 1.431122899055481,
      "rewards/rejected": -1.4715111255645752,
      "step": 13176
    },
    {
      "epoch": 1.7242986832420053,
      "grad_norm": 7.700691223144531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07987241446971893,
      "logits/rejected": -0.060714371502399445,
      "logps/chosen": -501.9272155761719,
      "logps/rejected": -398.4583740234375,
      "loss": 0.3887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0826561450958252,
      "rewards/margins": 2.0007760524749756,
      "rewards/rejected": -3.0834319591522217,
      "step": 13177
    },
    {
      "epoch": 1.724429541179357,
      "grad_norm": 9.362264633178711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3014455735683441,
      "logits/rejected": -0.42330580949783325,
      "logps/chosen": -310.7264404296875,
      "logps/rejected": -358.431884765625,
      "loss": 0.7218,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7272821664810181,
      "rewards/margins": 0.7798255681991577,
      "rewards/rejected": -1.5071077346801758,
      "step": 13178
    },
    {
      "epoch": 1.7245603991167089,
      "grad_norm": 9.655473709106445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.438479483127594,
      "logits/rejected": 0.2820807099342346,
      "logps/chosen": -468.2494812011719,
      "logps/rejected": -521.9469604492188,
      "loss": 0.4975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20036564767360687,
      "rewards/margins": 1.043032169342041,
      "rewards/rejected": -1.2433977127075195,
      "step": 13179
    },
    {
      "epoch": 1.7246912570540607,
      "grad_norm": 9.222707748413086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1410655975341797,
      "logits/rejected": -0.07172147929668427,
      "logps/chosen": -545.5427856445312,
      "logps/rejected": -560.5776977539062,
      "loss": 0.4802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39128273725509644,
      "rewards/margins": 1.1399104595184326,
      "rewards/rejected": -1.5311932563781738,
      "step": 13180
    },
    {
      "epoch": 1.7248221149914125,
      "grad_norm": 7.578327655792236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10209636390209198,
      "logits/rejected": -0.3487653136253357,
      "logps/chosen": -390.86077880859375,
      "logps/rejected": -430.2995300292969,
      "loss": 0.2923,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8685407042503357,
      "rewards/margins": 2.8396527767181396,
      "rewards/rejected": -3.70819354057312,
      "step": 13181
    },
    {
      "epoch": 1.7249529729287643,
      "grad_norm": 7.247304916381836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25254449248313904,
      "logits/rejected": -0.4780387878417969,
      "logps/chosen": -271.86700439453125,
      "logps/rejected": -405.6970520019531,
      "loss": 0.3786,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44634076952934265,
      "rewards/margins": 1.3729995489120483,
      "rewards/rejected": -1.8193403482437134,
      "step": 13182
    },
    {
      "epoch": 1.725083830866116,
      "grad_norm": 7.531274795532227,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22757075726985931,
      "logits/rejected": 0.03645695373415947,
      "logps/chosen": -601.388916015625,
      "logps/rejected": -523.6592407226562,
      "loss": 0.41,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5685968399047852,
      "rewards/margins": 1.3035460710525513,
      "rewards/rejected": -1.8721429109573364,
      "step": 13183
    },
    {
      "epoch": 1.7252146888034678,
      "grad_norm": 12.795157432556152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20388945937156677,
      "logits/rejected": -0.019108973443508148,
      "logps/chosen": -527.4696655273438,
      "logps/rejected": -483.87066650390625,
      "loss": 0.4415,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40093404054641724,
      "rewards/margins": 1.4702008962631226,
      "rewards/rejected": -1.8711349964141846,
      "step": 13184
    },
    {
      "epoch": 1.7253455467408196,
      "grad_norm": 9.429850578308105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027656467631459236,
      "logits/rejected": -0.1437373012304306,
      "logps/chosen": -347.7212219238281,
      "logps/rejected": -461.5290832519531,
      "loss": 0.7416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4621928930282593,
      "rewards/margins": 0.5798872113227844,
      "rewards/rejected": -2.0420799255371094,
      "step": 13185
    },
    {
      "epoch": 1.7254764046781712,
      "grad_norm": 8.826876640319824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09470308572053909,
      "logits/rejected": -0.24239152669906616,
      "logps/chosen": -574.514892578125,
      "logps/rejected": -524.9296875,
      "loss": 0.4917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04417818784713745,
      "rewards/margins": 0.5868403911590576,
      "rewards/rejected": -0.6310186386108398,
      "step": 13186
    },
    {
      "epoch": 1.725607262615523,
      "grad_norm": 8.107756614685059,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07550951093435287,
      "logits/rejected": 0.28641703724861145,
      "logps/chosen": -842.780517578125,
      "logps/rejected": -643.3518676757812,
      "loss": 0.325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.652360200881958,
      "rewards/margins": 1.9391405582427979,
      "rewards/rejected": -2.5915005207061768,
      "step": 13187
    },
    {
      "epoch": 1.7257381205528748,
      "grad_norm": 8.848358154296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21058997511863708,
      "logits/rejected": -0.220331609249115,
      "logps/chosen": -327.40374755859375,
      "logps/rejected": -476.0101318359375,
      "loss": 0.4658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.996124267578125,
      "rewards/margins": 1.223346471786499,
      "rewards/rejected": -2.219470977783203,
      "step": 13188
    },
    {
      "epoch": 1.7258689784902266,
      "grad_norm": 9.641463279724121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23304139077663422,
      "logits/rejected": -0.21958684921264648,
      "logps/chosen": -523.8690185546875,
      "logps/rejected": -409.6960754394531,
      "loss": 0.4764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23434831202030182,
      "rewards/margins": 1.2203526496887207,
      "rewards/rejected": -1.4547009468078613,
      "step": 13189
    },
    {
      "epoch": 1.7259998364275781,
      "grad_norm": 8.367465019226074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18347017467021942,
      "logits/rejected": 0.002375233918428421,
      "logps/chosen": -546.760498046875,
      "logps/rejected": -516.9676513671875,
      "loss": 0.4686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015305757522583008,
      "rewards/margins": 0.7753536701202393,
      "rewards/rejected": -0.7600479125976562,
      "step": 13190
    },
    {
      "epoch": 1.72613069436493,
      "grad_norm": 4.831787109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2739899158477783,
      "logits/rejected": -0.1574750691652298,
      "logps/chosen": -404.2970886230469,
      "logps/rejected": -378.9925537109375,
      "loss": 0.2846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5625883340835571,
      "rewards/margins": 1.8377728462219238,
      "rewards/rejected": -2.4003610610961914,
      "step": 13191
    },
    {
      "epoch": 1.7262615523022817,
      "grad_norm": 7.037599086761475,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0569131076335907,
      "logits/rejected": -0.0016981065273284912,
      "logps/chosen": -394.89801025390625,
      "logps/rejected": -383.66943359375,
      "loss": 0.3872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.74129319190979,
      "rewards/margins": 1.3622124195098877,
      "rewards/rejected": -2.1035056114196777,
      "step": 13192
    },
    {
      "epoch": 1.7263924102396335,
      "grad_norm": 10.233318328857422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1853322833776474,
      "logits/rejected": -0.3380160927772522,
      "logps/chosen": -414.5040588378906,
      "logps/rejected": -489.7334289550781,
      "loss": 0.6726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3572423756122589,
      "rewards/margins": 1.7361958026885986,
      "rewards/rejected": -2.093438148498535,
      "step": 13193
    },
    {
      "epoch": 1.7265232681769853,
      "grad_norm": 8.297607421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04594466835260391,
      "logits/rejected": -0.18285861611366272,
      "logps/chosen": -430.0466003417969,
      "logps/rejected": -383.5457763671875,
      "loss": 0.4627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.158603549003601,
      "rewards/margins": 1.0918394327163696,
      "rewards/rejected": -2.25044322013855,
      "step": 13194
    },
    {
      "epoch": 1.726654126114337,
      "grad_norm": 5.709969997406006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31547942757606506,
      "logits/rejected": -0.5111879110336304,
      "logps/chosen": -469.4992370605469,
      "logps/rejected": -395.0711364746094,
      "loss": 0.3576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03244621306657791,
      "rewards/margins": 1.5295920372009277,
      "rewards/rejected": -1.5620383024215698,
      "step": 13195
    },
    {
      "epoch": 1.726784984051689,
      "grad_norm": 11.080240249633789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08459680527448654,
      "logits/rejected": 0.008074171841144562,
      "logps/chosen": -823.2395629882812,
      "logps/rejected": -898.6788330078125,
      "loss": 0.6589,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5726367235183716,
      "rewards/margins": 0.6692072153091431,
      "rewards/rejected": -1.2418440580368042,
      "step": 13196
    },
    {
      "epoch": 1.7269158419890407,
      "grad_norm": 13.781928062438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025255843997001648,
      "logits/rejected": -0.40107282996177673,
      "logps/chosen": -548.13525390625,
      "logps/rejected": -533.2449340820312,
      "loss": 0.6485,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.460390567779541,
      "rewards/margins": 1.524239182472229,
      "rewards/rejected": -2.9846298694610596,
      "step": 13197
    },
    {
      "epoch": 1.7270466999263925,
      "grad_norm": 10.485570907592773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4549659788608551,
      "logits/rejected": -0.29141247272491455,
      "logps/chosen": -353.8543701171875,
      "logps/rejected": -432.9270935058594,
      "loss": 0.5419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.423269659280777,
      "rewards/margins": 1.6517691612243652,
      "rewards/rejected": -2.0750386714935303,
      "step": 13198
    },
    {
      "epoch": 1.7271775578637443,
      "grad_norm": 8.999615669250488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10666263103485107,
      "logits/rejected": -0.39247792959213257,
      "logps/chosen": -522.6075439453125,
      "logps/rejected": -398.266357421875,
      "loss": 0.5243,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3644142150878906,
      "rewards/margins": 1.3147510290145874,
      "rewards/rejected": -1.6791653633117676,
      "step": 13199
    },
    {
      "epoch": 1.727308415801096,
      "grad_norm": 7.971725940704346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09804881364107132,
      "logits/rejected": -0.376314640045166,
      "logps/chosen": -435.0177307128906,
      "logps/rejected": -412.1301574707031,
      "loss": 0.4476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1170477867126465,
      "rewards/margins": 0.9440674781799316,
      "rewards/rejected": -2.061115264892578,
      "step": 13200
    },
    {
      "epoch": 1.7274392737384479,
      "grad_norm": 13.10025405883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07123473286628723,
      "logits/rejected": -0.18544131517410278,
      "logps/chosen": -430.9555969238281,
      "logps/rejected": -427.4921875,
      "loss": 0.7281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1697874069213867,
      "rewards/margins": 0.48200106620788574,
      "rewards/rejected": -1.651788592338562,
      "step": 13201
    },
    {
      "epoch": 1.7275701316757994,
      "grad_norm": 8.03188705444336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1479063481092453,
      "logits/rejected": -0.10101215541362762,
      "logps/chosen": -314.7986755371094,
      "logps/rejected": -337.4404296875,
      "loss": 0.4817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33641624450683594,
      "rewards/margins": 0.703399658203125,
      "rewards/rejected": -1.039815902709961,
      "step": 13202
    },
    {
      "epoch": 1.7277009896131512,
      "grad_norm": 8.55241584777832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42299962043762207,
      "logits/rejected": -0.29822829365730286,
      "logps/chosen": -387.4476318359375,
      "logps/rejected": -383.4116516113281,
      "loss": 0.4415,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2549903988838196,
      "rewards/margins": 1.1710805892944336,
      "rewards/rejected": -1.426071047782898,
      "step": 13203
    },
    {
      "epoch": 1.727831847550503,
      "grad_norm": 9.724089622497559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17621701955795288,
      "logits/rejected": -0.385538786649704,
      "logps/chosen": -586.8994140625,
      "logps/rejected": -590.8154907226562,
      "loss": 0.5504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7549005746841431,
      "rewards/margins": 1.1881513595581055,
      "rewards/rejected": -1.9430519342422485,
      "step": 13204
    },
    {
      "epoch": 1.7279627054878548,
      "grad_norm": 8.442069053649902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.051070891320705414,
      "logits/rejected": -0.315321683883667,
      "logps/chosen": -506.79205322265625,
      "logps/rejected": -389.6578369140625,
      "loss": 0.4714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5078911781311035,
      "rewards/margins": 0.8484064936637878,
      "rewards/rejected": -2.356297731399536,
      "step": 13205
    },
    {
      "epoch": 1.7280935634252064,
      "grad_norm": 6.522049903869629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.296297550201416,
      "logits/rejected": -0.31016409397125244,
      "logps/chosen": -472.4815368652344,
      "logps/rejected": -581.8048706054688,
      "loss": 0.2689,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29023057222366333,
      "rewards/margins": 1.7890996932983398,
      "rewards/rejected": -1.4988691806793213,
      "step": 13206
    },
    {
      "epoch": 1.7282244213625582,
      "grad_norm": 8.433263778686523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21692289412021637,
      "logits/rejected": -0.23284661769866943,
      "logps/chosen": -338.09564208984375,
      "logps/rejected": -287.9970703125,
      "loss": 0.4113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2302476167678833,
      "rewards/margins": 0.9423983097076416,
      "rewards/rejected": -2.1726460456848145,
      "step": 13207
    },
    {
      "epoch": 1.72835527929991,
      "grad_norm": 7.749892711639404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1097329705953598,
      "logits/rejected": -0.25625288486480713,
      "logps/chosen": -573.4226684570312,
      "logps/rejected": -491.8392639160156,
      "loss": 0.3526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.024893373250961304,
      "rewards/margins": 1.5823661088943481,
      "rewards/rejected": -1.557472586631775,
      "step": 13208
    },
    {
      "epoch": 1.7284861372372617,
      "grad_norm": 7.557244300842285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.018947601318359375,
      "logits/rejected": -0.3188122510910034,
      "logps/chosen": -626.4041137695312,
      "logps/rejected": -413.873779296875,
      "loss": 0.3711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19109687209129333,
      "rewards/margins": 1.3420289754867554,
      "rewards/rejected": -1.1509321928024292,
      "step": 13209
    },
    {
      "epoch": 1.7286169951746135,
      "grad_norm": 10.184941291809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16110114753246307,
      "logits/rejected": -0.19064149260520935,
      "logps/chosen": -417.2890319824219,
      "logps/rejected": -480.1708068847656,
      "loss": 0.4703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7353839874267578,
      "rewards/margins": 1.1960678100585938,
      "rewards/rejected": -2.9314517974853516,
      "step": 13210
    },
    {
      "epoch": 1.7287478531119653,
      "grad_norm": 6.52760648727417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.134173184633255,
      "logits/rejected": -0.23954541981220245,
      "logps/chosen": -411.7266845703125,
      "logps/rejected": -372.2766418457031,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2516613006591797,
      "rewards/margins": 1.3411524295806885,
      "rewards/rejected": -1.0894911289215088,
      "step": 13211
    },
    {
      "epoch": 1.7288787110493171,
      "grad_norm": 6.2702484130859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06349073350429535,
      "logits/rejected": -0.17930813133716583,
      "logps/chosen": -408.66778564453125,
      "logps/rejected": -342.5248718261719,
      "loss": 0.339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08808115124702454,
      "rewards/margins": 1.05025315284729,
      "rewards/rejected": -1.1383342742919922,
      "step": 13212
    },
    {
      "epoch": 1.729009568986669,
      "grad_norm": 11.840225219726562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1198853999376297,
      "logits/rejected": -0.22085687518119812,
      "logps/chosen": -697.4284057617188,
      "logps/rejected": -595.6296997070312,
      "loss": 0.7057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7610169649124146,
      "rewards/margins": 0.5632467269897461,
      "rewards/rejected": -1.3242636919021606,
      "step": 13213
    },
    {
      "epoch": 1.7291404269240207,
      "grad_norm": 7.832902908325195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2858867645263672,
      "logits/rejected": -0.2802261412143707,
      "logps/chosen": -273.59014892578125,
      "logps/rejected": -375.06475830078125,
      "loss": 0.3014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2441085577011108,
      "rewards/margins": 1.1134672164916992,
      "rewards/rejected": -2.3575758934020996,
      "step": 13214
    },
    {
      "epoch": 1.7292712848613725,
      "grad_norm": 10.10386848449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013250625692307949,
      "logits/rejected": 0.13378897309303284,
      "logps/chosen": -382.795654296875,
      "logps/rejected": -544.2720336914062,
      "loss": 0.6296,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1225252151489258,
      "rewards/margins": 0.2776516079902649,
      "rewards/rejected": -1.400176763534546,
      "step": 13215
    },
    {
      "epoch": 1.7294021427987243,
      "grad_norm": 5.896993637084961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28873729705810547,
      "logits/rejected": -0.2771309018135071,
      "logps/chosen": -413.9336242675781,
      "logps/rejected": -392.4311218261719,
      "loss": 0.2536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7487331628799438,
      "rewards/margins": 1.4353504180908203,
      "rewards/rejected": -2.1840837001800537,
      "step": 13216
    },
    {
      "epoch": 1.7295330007360759,
      "grad_norm": 4.73737096786499,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12118268758058548,
      "logits/rejected": -0.5470829606056213,
      "logps/chosen": -339.2764892578125,
      "logps/rejected": -289.3057861328125,
      "loss": 0.2585,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6846869587898254,
      "rewards/margins": 2.2082931995391846,
      "rewards/rejected": -2.8929800987243652,
      "step": 13217
    },
    {
      "epoch": 1.7296638586734276,
      "grad_norm": 8.005364418029785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0983569473028183,
      "logits/rejected": -0.15371325612068176,
      "logps/chosen": -624.5001220703125,
      "logps/rejected": -549.175537109375,
      "loss": 0.325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13689842820167542,
      "rewards/margins": 1.5222641229629517,
      "rewards/rejected": -1.6591625213623047,
      "step": 13218
    },
    {
      "epoch": 1.7297947166107794,
      "grad_norm": 11.132436752319336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15540310740470886,
      "logits/rejected": 0.11120326817035675,
      "logps/chosen": -461.5752868652344,
      "logps/rejected": -296.23834228515625,
      "loss": 0.8151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0123708248138428,
      "rewards/margins": 0.48148733377456665,
      "rewards/rejected": -2.4938583374023438,
      "step": 13219
    },
    {
      "epoch": 1.7299255745481312,
      "grad_norm": 7.99149227142334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11552499234676361,
      "logits/rejected": -0.15750153362751007,
      "logps/chosen": -561.458740234375,
      "logps/rejected": -537.7106323242188,
      "loss": 0.4304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6698862314224243,
      "rewards/margins": 1.102522850036621,
      "rewards/rejected": -1.7724089622497559,
      "step": 13220
    },
    {
      "epoch": 1.7300564324854828,
      "grad_norm": 8.027816772460938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23816373944282532,
      "logits/rejected": -0.2952823042869568,
      "logps/chosen": -515.7420043945312,
      "logps/rejected": -430.60919189453125,
      "loss": 0.4024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5863046646118164,
      "rewards/margins": 1.2899647951126099,
      "rewards/rejected": -1.8762695789337158,
      "step": 13221
    },
    {
      "epoch": 1.7301872904228346,
      "grad_norm": 14.477375984191895,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10946126282215118,
      "logits/rejected": 0.15245594084262848,
      "logps/chosen": -527.9942016601562,
      "logps/rejected": -494.73919677734375,
      "loss": 1.026,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8989010453224182,
      "rewards/margins": 0.12040943652391434,
      "rewards/rejected": -1.0193103551864624,
      "step": 13222
    },
    {
      "epoch": 1.7303181483601864,
      "grad_norm": 4.950510501861572,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32696229219436646,
      "logits/rejected": -0.3036006987094879,
      "logps/chosen": -446.66021728515625,
      "logps/rejected": -387.23809814453125,
      "loss": 0.2529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3074454069137573,
      "rewards/margins": 2.0914998054504395,
      "rewards/rejected": -2.3989450931549072,
      "step": 13223
    },
    {
      "epoch": 1.7304490062975382,
      "grad_norm": 7.043903350830078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047256506979465485,
      "logits/rejected": -0.13246841728687286,
      "logps/chosen": -408.18035888671875,
      "logps/rejected": -467.8944091796875,
      "loss": 0.4109,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8214647769927979,
      "rewards/margins": 0.802176296710968,
      "rewards/rejected": -1.623641014099121,
      "step": 13224
    },
    {
      "epoch": 1.73057986423489,
      "grad_norm": 7.849635601043701,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20183056592941284,
      "logits/rejected": -0.10144772380590439,
      "logps/chosen": -492.5063781738281,
      "logps/rejected": -387.81005859375,
      "loss": 0.4566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2015310376882553,
      "rewards/margins": 1.2881035804748535,
      "rewards/rejected": -1.0865724086761475,
      "step": 13225
    },
    {
      "epoch": 1.7307107221722418,
      "grad_norm": 11.006940841674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013498404994606972,
      "logits/rejected": -0.026779042556881905,
      "logps/chosen": -455.27294921875,
      "logps/rejected": -455.7225341796875,
      "loss": 0.6083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11441479623317719,
      "rewards/margins": 0.42739200592041016,
      "rewards/rejected": -0.5418068170547485,
      "step": 13226
    },
    {
      "epoch": 1.7308415801095935,
      "grad_norm": 6.0175909996032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07137702405452728,
      "logits/rejected": -0.12226622551679611,
      "logps/chosen": -453.4239501953125,
      "logps/rejected": -565.7391357421875,
      "loss": 0.3287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1719204187393188,
      "rewards/margins": 1.7665408849716187,
      "rewards/rejected": -2.9384615421295166,
      "step": 13227
    },
    {
      "epoch": 1.7309724380469453,
      "grad_norm": 9.999556541442871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23251917958259583,
      "logits/rejected": -0.33498281240463257,
      "logps/chosen": -809.4522705078125,
      "logps/rejected": -531.69091796875,
      "loss": 0.354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15700075030326843,
      "rewards/margins": 1.8214728832244873,
      "rewards/rejected": -1.9784736633300781,
      "step": 13228
    },
    {
      "epoch": 1.7311032959842971,
      "grad_norm": 8.066388130187988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.009340576827526093,
      "logits/rejected": -0.252860963344574,
      "logps/chosen": -461.70098876953125,
      "logps/rejected": -504.57110595703125,
      "loss": 0.4879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6579193472862244,
      "rewards/margins": 1.504570484161377,
      "rewards/rejected": -2.162489891052246,
      "step": 13229
    },
    {
      "epoch": 1.731234153921649,
      "grad_norm": 5.45612096786499,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40353554487228394,
      "logits/rejected": -0.4229091703891754,
      "logps/chosen": -365.72869873046875,
      "logps/rejected": -471.9326171875,
      "loss": 0.2792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24501244723796844,
      "rewards/margins": 2.0448269844055176,
      "rewards/rejected": -2.289839506149292,
      "step": 13230
    },
    {
      "epoch": 1.7313650118590007,
      "grad_norm": 6.93917179107666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20904441177845,
      "logits/rejected": -0.10514350235462189,
      "logps/chosen": -452.4765625,
      "logps/rejected": -476.7305908203125,
      "loss": 0.3514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5448957681655884,
      "rewards/margins": 1.0583374500274658,
      "rewards/rejected": -1.6032332181930542,
      "step": 13231
    },
    {
      "epoch": 1.7314958697963523,
      "grad_norm": 11.022106170654297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04851057380437851,
      "logits/rejected": -0.12439079582691193,
      "logps/chosen": -313.7471923828125,
      "logps/rejected": -348.08349609375,
      "loss": 0.8132,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1618173122406006,
      "rewards/margins": -0.05043037235736847,
      "rewards/rejected": -1.111387014389038,
      "step": 13232
    },
    {
      "epoch": 1.731626727733704,
      "grad_norm": 7.045170307159424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33504775166511536,
      "logits/rejected": -0.4546124339103699,
      "logps/chosen": -402.8273010253906,
      "logps/rejected": -412.9233703613281,
      "loss": 0.3714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21792030334472656,
      "rewards/margins": 1.3961200714111328,
      "rewards/rejected": -1.6140403747558594,
      "step": 13233
    },
    {
      "epoch": 1.7317575856710559,
      "grad_norm": 5.840219974517822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.443523645401001,
      "logits/rejected": -0.36101070046424866,
      "logps/chosen": -488.60870361328125,
      "logps/rejected": -349.60491943359375,
      "loss": 0.2565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40836814045906067,
      "rewards/margins": 1.5703740119934082,
      "rewards/rejected": -1.978742003440857,
      "step": 13234
    },
    {
      "epoch": 1.7318884436084077,
      "grad_norm": 6.3781962394714355,
      "learning_rate": 5e-06,
      "logits/chosen": 0.320903480052948,
      "logits/rejected": 0.15078559517860413,
      "logps/chosen": -382.93780517578125,
      "logps/rejected": -414.61029052734375,
      "loss": 0.3597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22144222259521484,
      "rewards/margins": 0.9568511247634888,
      "rewards/rejected": -0.7354090213775635,
      "step": 13235
    },
    {
      "epoch": 1.7320193015457592,
      "grad_norm": 7.222658157348633,
      "learning_rate": 5e-06,
      "logits/chosen": 0.040406279265880585,
      "logits/rejected": 0.10258868336677551,
      "logps/chosen": -494.7499084472656,
      "logps/rejected": -370.2301330566406,
      "loss": 0.4572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14055541157722473,
      "rewards/margins": 1.5034043788909912,
      "rewards/rejected": -1.643959879875183,
      "step": 13236
    },
    {
      "epoch": 1.732150159483111,
      "grad_norm": 7.020036220550537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026497110724449158,
      "logits/rejected": -0.14569689333438873,
      "logps/chosen": -721.3758544921875,
      "logps/rejected": -598.6564331054688,
      "loss": 0.2858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2765072286128998,
      "rewards/margins": 1.4215800762176514,
      "rewards/rejected": -1.6980873346328735,
      "step": 13237
    },
    {
      "epoch": 1.7322810174204628,
      "grad_norm": 9.338290214538574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08536218106746674,
      "logits/rejected": -0.3237394690513611,
      "logps/chosen": -454.0830078125,
      "logps/rejected": -364.556396484375,
      "loss": 0.4888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1126136779785156,
      "rewards/margins": 1.1075865030288696,
      "rewards/rejected": -2.220200300216675,
      "step": 13238
    },
    {
      "epoch": 1.7324118753578146,
      "grad_norm": 6.90742826461792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08775337785482407,
      "logits/rejected": -0.3951633870601654,
      "logps/chosen": -394.4169921875,
      "logps/rejected": -415.6262512207031,
      "loss": 0.4011,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3546549081802368,
      "rewards/margins": 1.2816696166992188,
      "rewards/rejected": -2.636324644088745,
      "step": 13239
    },
    {
      "epoch": 1.7325427332951664,
      "grad_norm": 13.352723121643066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14383189380168915,
      "logits/rejected": -0.2084973007440567,
      "logps/chosen": -394.22698974609375,
      "logps/rejected": -443.6151123046875,
      "loss": 0.579,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8413373827934265,
      "rewards/margins": 1.0337413549423218,
      "rewards/rejected": -1.8750786781311035,
      "step": 13240
    },
    {
      "epoch": 1.7326735912325182,
      "grad_norm": 14.043680191040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.514661431312561,
      "logits/rejected": -0.4377441704273224,
      "logps/chosen": -456.1836853027344,
      "logps/rejected": -571.9832763671875,
      "loss": 0.6705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7332777976989746,
      "rewards/margins": 0.6808806657791138,
      "rewards/rejected": -2.414158582687378,
      "step": 13241
    },
    {
      "epoch": 1.73280444916987,
      "grad_norm": 11.889296531677246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36873409152030945,
      "logits/rejected": -0.37783536314964294,
      "logps/chosen": -408.912841796875,
      "logps/rejected": -599.7244873046875,
      "loss": 0.5169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5638176202774048,
      "rewards/margins": 2.036245822906494,
      "rewards/rejected": -2.6000633239746094,
      "step": 13242
    },
    {
      "epoch": 1.7329353071072218,
      "grad_norm": 3.043877363204956,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06718021631240845,
      "logits/rejected": -0.3188493847846985,
      "logps/chosen": -434.4619445800781,
      "logps/rejected": -478.174560546875,
      "loss": 0.1529,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9884439706802368,
      "rewards/margins": 2.14434552192688,
      "rewards/rejected": -3.132789373397827,
      "step": 13243
    },
    {
      "epoch": 1.7330661650445736,
      "grad_norm": 9.126020431518555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.137674018740654,
      "logits/rejected": -0.04140911623835564,
      "logps/chosen": -490.9937744140625,
      "logps/rejected": -522.6195678710938,
      "loss": 0.494,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5622607469558716,
      "rewards/margins": 0.7631019353866577,
      "rewards/rejected": -2.3253626823425293,
      "step": 13244
    },
    {
      "epoch": 1.7331970229819254,
      "grad_norm": 12.187859535217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04212615638971329,
      "logits/rejected": -0.1717580109834671,
      "logps/chosen": -408.18902587890625,
      "logps/rejected": -380.4780578613281,
      "loss": 0.8399,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.1443514823913574,
      "rewards/margins": 0.055888205766677856,
      "rewards/rejected": -1.2002397775650024,
      "step": 13245
    },
    {
      "epoch": 1.7333278809192771,
      "grad_norm": 9.107163429260254,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18512044847011566,
      "logits/rejected": -0.02276238426566124,
      "logps/chosen": -627.1304931640625,
      "logps/rejected": -405.4759826660156,
      "loss": 0.4038,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.922524631023407,
      "rewards/margins": 1.3292701244354248,
      "rewards/rejected": -2.2517948150634766,
      "step": 13246
    },
    {
      "epoch": 1.733458738856629,
      "grad_norm": 3.674258232116699,
      "learning_rate": 5e-06,
      "logits/chosen": 0.331379771232605,
      "logits/rejected": 0.3157660663127899,
      "logps/chosen": -436.4414978027344,
      "logps/rejected": -593.7440795898438,
      "loss": 0.2258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4120500683784485,
      "rewards/margins": 2.114330291748047,
      "rewards/rejected": -2.5263803005218506,
      "step": 13247
    },
    {
      "epoch": 1.7335895967939805,
      "grad_norm": 8.516797065734863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0024833595380187035,
      "logits/rejected": 0.004639375954866409,
      "logps/chosen": -464.05621337890625,
      "logps/rejected": -532.4253540039062,
      "loss": 0.3989,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2607044577598572,
      "rewards/margins": 1.626590371131897,
      "rewards/rejected": -1.887294888496399,
      "step": 13248
    },
    {
      "epoch": 1.7337204547313323,
      "grad_norm": 8.7578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15878325700759888,
      "logits/rejected": -0.07277882099151611,
      "logps/chosen": -637.7706298828125,
      "logps/rejected": -487.653076171875,
      "loss": 0.4736,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09892311692237854,
      "rewards/margins": 1.0137771368026733,
      "rewards/rejected": -1.1127002239227295,
      "step": 13249
    },
    {
      "epoch": 1.733851312668684,
      "grad_norm": 7.227773666381836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17706608772277832,
      "logits/rejected": -0.2146577686071396,
      "logps/chosen": -484.64422607421875,
      "logps/rejected": -529.0488891601562,
      "loss": 0.3581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2415164709091187,
      "rewards/margins": 1.4228746891021729,
      "rewards/rejected": -2.664391040802002,
      "step": 13250
    },
    {
      "epoch": 1.7339821706060359,
      "grad_norm": 6.782544136047363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06561805307865143,
      "logits/rejected": -0.23548737168312073,
      "logps/chosen": -558.9013061523438,
      "logps/rejected": -441.2601623535156,
      "loss": 0.3189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5657612085342407,
      "rewards/margins": 1.1740367412567139,
      "rewards/rejected": -1.7397979497909546,
      "step": 13251
    },
    {
      "epoch": 1.7341130285433874,
      "grad_norm": 8.165495872497559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13452062010765076,
      "logits/rejected": -0.41765519976615906,
      "logps/chosen": -309.38848876953125,
      "logps/rejected": -267.9407958984375,
      "loss": 0.3714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7852510213851929,
      "rewards/margins": 1.3953018188476562,
      "rewards/rejected": -2.1805529594421387,
      "step": 13252
    },
    {
      "epoch": 1.7342438864807392,
      "grad_norm": 11.89030933380127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19875949621200562,
      "logits/rejected": -0.36597737669944763,
      "logps/chosen": -351.62615966796875,
      "logps/rejected": -428.67724609375,
      "loss": 0.7389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.920201301574707,
      "rewards/margins": 0.5938469171524048,
      "rewards/rejected": -1.5140480995178223,
      "step": 13253
    },
    {
      "epoch": 1.734374744418091,
      "grad_norm": 7.7412261962890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2288706749677658,
      "logits/rejected": -0.19242896139621735,
      "logps/chosen": -362.50164794921875,
      "logps/rejected": -358.56195068359375,
      "loss": 0.5198,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7981147766113281,
      "rewards/margins": 0.895978569984436,
      "rewards/rejected": -1.6940933465957642,
      "step": 13254
    },
    {
      "epoch": 1.7345056023554428,
      "grad_norm": 11.75638198852539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06769590079784393,
      "logits/rejected": 0.22866083681583405,
      "logps/chosen": -472.75604248046875,
      "logps/rejected": -511.1639099121094,
      "loss": 0.6144,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8259891271591187,
      "rewards/margins": 0.6649450063705444,
      "rewards/rejected": -1.4909342527389526,
      "step": 13255
    },
    {
      "epoch": 1.7346364602927946,
      "grad_norm": 9.909443855285645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2215948849916458,
      "logits/rejected": -0.21589523553848267,
      "logps/chosen": -455.71734619140625,
      "logps/rejected": -530.9581298828125,
      "loss": 0.5432,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5162872076034546,
      "rewards/margins": 1.204117774963379,
      "rewards/rejected": -1.720405101776123,
      "step": 13256
    },
    {
      "epoch": 1.7347673182301464,
      "grad_norm": 13.800073623657227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17847883701324463,
      "logits/rejected": 0.003946119919419289,
      "logps/chosen": -386.84490966796875,
      "logps/rejected": -525.7644653320312,
      "loss": 0.7603,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9445555210113525,
      "rewards/margins": 0.46310102939605713,
      "rewards/rejected": -2.407656669616699,
      "step": 13257
    },
    {
      "epoch": 1.7348981761674982,
      "grad_norm": 5.230325222015381,
      "learning_rate": 5e-06,
      "logits/chosen": -0.517520546913147,
      "logits/rejected": -0.6258508563041687,
      "logps/chosen": -248.37258911132812,
      "logps/rejected": -449.93731689453125,
      "loss": 0.2771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8841437697410583,
      "rewards/margins": 1.9379539489746094,
      "rewards/rejected": -2.8220975399017334,
      "step": 13258
    },
    {
      "epoch": 1.73502903410485,
      "grad_norm": 9.624168395996094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07996335625648499,
      "logits/rejected": -0.09743388742208481,
      "logps/chosen": -507.8768310546875,
      "logps/rejected": -491.7331848144531,
      "loss": 0.5299,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1587711572647095,
      "rewards/margins": 1.0086381435394287,
      "rewards/rejected": -2.1674091815948486,
      "step": 13259
    },
    {
      "epoch": 1.7351598920422018,
      "grad_norm": 6.089775085449219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11873068660497665,
      "logits/rejected": -0.37256163358688354,
      "logps/chosen": -440.9696960449219,
      "logps/rejected": -499.5390319824219,
      "loss": 0.2721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7042208313941956,
      "rewards/margins": 1.4798097610473633,
      "rewards/rejected": -2.184030532836914,
      "step": 13260
    },
    {
      "epoch": 1.7352907499795536,
      "grad_norm": 6.753054618835449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18683844804763794,
      "logits/rejected": -0.2840142548084259,
      "logps/chosen": -432.3731384277344,
      "logps/rejected": -359.865234375,
      "loss": 0.3543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7553943395614624,
      "rewards/margins": 1.7131856679916382,
      "rewards/rejected": -2.4685800075531006,
      "step": 13261
    },
    {
      "epoch": 1.7354216079169054,
      "grad_norm": 8.732967376708984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.308517187833786,
      "logits/rejected": -0.5641880631446838,
      "logps/chosen": -355.7022399902344,
      "logps/rejected": -393.827392578125,
      "loss": 0.5778,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.021570086479187,
      "rewards/margins": 0.5061517953872681,
      "rewards/rejected": -1.527721881866455,
      "step": 13262
    },
    {
      "epoch": 1.735552465854257,
      "grad_norm": 7.525784015655518,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16717875003814697,
      "logits/rejected": -0.09704143553972244,
      "logps/chosen": -744.2037353515625,
      "logps/rejected": -481.373779296875,
      "loss": 0.3173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0417274236679077,
      "rewards/margins": 1.8869071006774902,
      "rewards/rejected": -2.9286346435546875,
      "step": 13263
    },
    {
      "epoch": 1.7356833237916087,
      "grad_norm": 8.499151229858398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16957145929336548,
      "logits/rejected": -0.046405766159296036,
      "logps/chosen": -567.3594970703125,
      "logps/rejected": -349.29669189453125,
      "loss": 0.3601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3263048529624939,
      "rewards/margins": 1.4983716011047363,
      "rewards/rejected": -1.824676513671875,
      "step": 13264
    },
    {
      "epoch": 1.7358141817289605,
      "grad_norm": 9.013971328735352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19764795899391174,
      "logits/rejected": -0.07946436107158661,
      "logps/chosen": -494.48675537109375,
      "logps/rejected": -442.9689025878906,
      "loss": 0.3962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4199194312095642,
      "rewards/margins": 1.5076640844345093,
      "rewards/rejected": -1.9275835752487183,
      "step": 13265
    },
    {
      "epoch": 1.7359450396663123,
      "grad_norm": 6.617651462554932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09726686775684357,
      "logits/rejected": -0.23466284573078156,
      "logps/chosen": -398.69744873046875,
      "logps/rejected": -346.7914123535156,
      "loss": 0.4565,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8238436579704285,
      "rewards/margins": 1.6094021797180176,
      "rewards/rejected": -2.4332454204559326,
      "step": 13266
    },
    {
      "epoch": 1.7360758976036639,
      "grad_norm": 11.765585899353027,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0745924860239029,
      "logits/rejected": -0.19498367607593536,
      "logps/chosen": -522.3829345703125,
      "logps/rejected": -366.2218933105469,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.756039023399353,
      "rewards/margins": 0.5081433653831482,
      "rewards/rejected": -2.2641825675964355,
      "step": 13267
    },
    {
      "epoch": 1.7362067555410157,
      "grad_norm": 5.917540550231934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2488032579421997,
      "logits/rejected": -0.4560757875442505,
      "logps/chosen": -393.4103698730469,
      "logps/rejected": -414.3746643066406,
      "loss": 0.2279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0336698293685913,
      "rewards/margins": 2.1816272735595703,
      "rewards/rejected": -3.215296983718872,
      "step": 13268
    },
    {
      "epoch": 1.7363376134783675,
      "grad_norm": 8.081734657287598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2788558602333069,
      "logits/rejected": -0.5514531135559082,
      "logps/chosen": -477.0008544921875,
      "logps/rejected": -445.1891174316406,
      "loss": 0.4704,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7645527124404907,
      "rewards/margins": 1.164686679840088,
      "rewards/rejected": -1.9292393922805786,
      "step": 13269
    },
    {
      "epoch": 1.7364684714157193,
      "grad_norm": 8.476337432861328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07690687477588654,
      "logits/rejected": -0.31695762276649475,
      "logps/chosen": -614.232666015625,
      "logps/rejected": -527.3895874023438,
      "loss": 0.3269,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43959715962409973,
      "rewards/margins": 1.9910391569137573,
      "rewards/rejected": -2.430636167526245,
      "step": 13270
    },
    {
      "epoch": 1.736599329353071,
      "grad_norm": 5.362140655517578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2394530028104782,
      "logits/rejected": -0.20103174448013306,
      "logps/chosen": -400.8688659667969,
      "logps/rejected": -356.27410888671875,
      "loss": 0.2474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.844833493232727,
      "rewards/margins": 1.5284775495529175,
      "rewards/rejected": -2.3733110427856445,
      "step": 13271
    },
    {
      "epoch": 1.7367301872904228,
      "grad_norm": 19.170907974243164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4419857859611511,
      "logits/rejected": -0.3888276517391205,
      "logps/chosen": -394.48590087890625,
      "logps/rejected": -351.42742919921875,
      "loss": 1.055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0091278553009033,
      "rewards/margins": 0.4455419182777405,
      "rewards/rejected": -2.454669952392578,
      "step": 13272
    },
    {
      "epoch": 1.7368610452277746,
      "grad_norm": 5.39447021484375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03557540848851204,
      "logits/rejected": 0.017220869660377502,
      "logps/chosen": -366.4192810058594,
      "logps/rejected": -348.9349365234375,
      "loss": 0.2322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35104334354400635,
      "rewards/margins": 1.6455029249191284,
      "rewards/rejected": -1.9965462684631348,
      "step": 13273
    },
    {
      "epoch": 1.7369919031651264,
      "grad_norm": 6.743006706237793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.37274885177612305,
      "logits/rejected": -0.1426072120666504,
      "logps/chosen": -576.3881225585938,
      "logps/rejected": -589.8302001953125,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0230170488357544,
      "rewards/margins": 1.443536639213562,
      "rewards/rejected": -2.4665536880493164,
      "step": 13274
    },
    {
      "epoch": 1.7371227611024782,
      "grad_norm": 14.207036972045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19807595014572144,
      "logits/rejected": -0.10430900752544403,
      "logps/chosen": -472.3413391113281,
      "logps/rejected": -474.083251953125,
      "loss": 0.8026,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1015050411224365,
      "rewards/margins": 0.7613637447357178,
      "rewards/rejected": -2.8628687858581543,
      "step": 13275
    },
    {
      "epoch": 1.73725361903983,
      "grad_norm": 10.141834259033203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15886756777763367,
      "logits/rejected": -0.38131383061408997,
      "logps/chosen": -395.88671875,
      "logps/rejected": -420.9807434082031,
      "loss": 0.4948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0445867776870728,
      "rewards/margins": 0.9254387021064758,
      "rewards/rejected": -1.9700255393981934,
      "step": 13276
    },
    {
      "epoch": 1.7373844769771818,
      "grad_norm": 7.195507526397705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18255957961082458,
      "logits/rejected": -0.17057380080223083,
      "logps/chosen": -672.4822998046875,
      "logps/rejected": -411.6568603515625,
      "loss": 0.2914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9360410571098328,
      "rewards/margins": 1.7765918970108032,
      "rewards/rejected": -2.712632894515991,
      "step": 13277
    },
    {
      "epoch": 1.7375153349145334,
      "grad_norm": 5.11907434463501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22998562455177307,
      "logits/rejected": -0.23204222321510315,
      "logps/chosen": -529.0894165039062,
      "logps/rejected": -372.72589111328125,
      "loss": 0.2354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9975183606147766,
      "rewards/margins": 1.7799197435379028,
      "rewards/rejected": -2.777438163757324,
      "step": 13278
    },
    {
      "epoch": 1.7376461928518852,
      "grad_norm": 6.240407466888428,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05791136249899864,
      "logits/rejected": -0.09975790232419968,
      "logps/chosen": -478.2114562988281,
      "logps/rejected": -528.0270385742188,
      "loss": 0.2915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06662904471158981,
      "rewards/margins": 1.9988254308700562,
      "rewards/rejected": -1.9321963787078857,
      "step": 13279
    },
    {
      "epoch": 1.737777050789237,
      "grad_norm": 10.767643928527832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2527126669883728,
      "logits/rejected": -0.24693995714187622,
      "logps/chosen": -454.73919677734375,
      "logps/rejected": -404.84613037109375,
      "loss": 0.5412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23870542645454407,
      "rewards/margins": 0.9561547040939331,
      "rewards/rejected": -1.1948601007461548,
      "step": 13280
    },
    {
      "epoch": 1.7379079087265887,
      "grad_norm": 13.736431121826172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3570792078971863,
      "logits/rejected": -0.12115901708602905,
      "logps/chosen": -545.8704223632812,
      "logps/rejected": -583.9837646484375,
      "loss": 0.6643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0149439573287964,
      "rewards/margins": 0.8703412413597107,
      "rewards/rejected": -1.8852849006652832,
      "step": 13281
    },
    {
      "epoch": 1.7380387666639403,
      "grad_norm": 18.29998779296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2700340151786804,
      "logits/rejected": -0.12095601856708527,
      "logps/chosen": -428.86395263671875,
      "logps/rejected": -539.897705078125,
      "loss": 1.2728,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.8776819705963135,
      "rewards/margins": -0.14180028438568115,
      "rewards/rejected": -1.7358815670013428,
      "step": 13282
    },
    {
      "epoch": 1.738169624601292,
      "grad_norm": 6.92246675491333,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23508834838867188,
      "logits/rejected": -0.3539535403251648,
      "logps/chosen": -418.478759765625,
      "logps/rejected": -381.67413330078125,
      "loss": 0.4356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5863885879516602,
      "rewards/margins": 0.965981662273407,
      "rewards/rejected": -1.552370309829712,
      "step": 13283
    },
    {
      "epoch": 1.7383004825386439,
      "grad_norm": 9.927399635314941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32069131731987,
      "logits/rejected": -0.2248966544866562,
      "logps/chosen": -559.1430053710938,
      "logps/rejected": -447.47149658203125,
      "loss": 0.4494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.498095154762268,
      "rewards/margins": 1.4186193943023682,
      "rewards/rejected": -2.9167141914367676,
      "step": 13284
    },
    {
      "epoch": 1.7384313404759957,
      "grad_norm": 6.332818984985352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1473105102777481,
      "logits/rejected": -0.24724362790584564,
      "logps/chosen": -479.7292785644531,
      "logps/rejected": -393.6482849121094,
      "loss": 0.3441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1402580738067627,
      "rewards/margins": 1.1250501871109009,
      "rewards/rejected": -2.265308380126953,
      "step": 13285
    },
    {
      "epoch": 1.7385621984133475,
      "grad_norm": 5.957540512084961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2575957775115967,
      "logits/rejected": -0.23894299566745758,
      "logps/chosen": -471.5851135253906,
      "logps/rejected": -437.05950927734375,
      "loss": 0.3516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.37656861543655396,
      "rewards/margins": 1.2789520025253296,
      "rewards/rejected": -1.6555205583572388,
      "step": 13286
    },
    {
      "epoch": 1.7386930563506993,
      "grad_norm": 4.242106914520264,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1849365234375,
      "logits/rejected": -0.11995720863342285,
      "logps/chosen": -331.60443115234375,
      "logps/rejected": -376.25054931640625,
      "loss": 0.2359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7731363773345947,
      "rewards/margins": 1.9251365661621094,
      "rewards/rejected": -2.698272943496704,
      "step": 13287
    },
    {
      "epoch": 1.738823914288051,
      "grad_norm": 9.926899909973145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3627789616584778,
      "logits/rejected": -0.38632822036743164,
      "logps/chosen": -466.0943298339844,
      "logps/rejected": -540.0322265625,
      "loss": 0.4273,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6090407967567444,
      "rewards/margins": 2.420933723449707,
      "rewards/rejected": -3.0299746990203857,
      "step": 13288
    },
    {
      "epoch": 1.7389547722254028,
      "grad_norm": 10.989285469055176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2705237865447998,
      "logits/rejected": -0.48592010140419006,
      "logps/chosen": -723.1261596679688,
      "logps/rejected": -514.3982543945312,
      "loss": 0.407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.332161545753479,
      "rewards/margins": 1.6217873096466064,
      "rewards/rejected": -1.953948736190796,
      "step": 13289
    },
    {
      "epoch": 1.7390856301627546,
      "grad_norm": 10.645130157470703,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0008253934793174267,
      "logits/rejected": -0.05435531586408615,
      "logps/chosen": -583.0279541015625,
      "logps/rejected": -473.80633544921875,
      "loss": 0.6184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6768035888671875,
      "rewards/margins": 0.6829351782798767,
      "rewards/rejected": -1.3597387075424194,
      "step": 13290
    },
    {
      "epoch": 1.7392164881001064,
      "grad_norm": 4.8949055671691895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10982169955968857,
      "logits/rejected": -0.10199576616287231,
      "logps/chosen": -462.0095520019531,
      "logps/rejected": -476.81732177734375,
      "loss": 0.1929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2563358545303345,
      "rewards/margins": 2.1573486328125,
      "rewards/rejected": -2.413684368133545,
      "step": 13291
    },
    {
      "epoch": 1.7393473460374582,
      "grad_norm": 8.968151092529297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12828956544399261,
      "logits/rejected": -0.09493879973888397,
      "logps/chosen": -527.6419067382812,
      "logps/rejected": -535.3886108398438,
      "loss": 0.5064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0424226522445679,
      "rewards/margins": 1.5960948467254639,
      "rewards/rejected": -2.638517379760742,
      "step": 13292
    },
    {
      "epoch": 1.73947820397481,
      "grad_norm": 11.699845314025879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18395207822322845,
      "logits/rejected": -0.45674610137939453,
      "logps/chosen": -369.6780090332031,
      "logps/rejected": -496.2768249511719,
      "loss": 0.5287,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.314558506011963,
      "rewards/margins": 0.9178882837295532,
      "rewards/rejected": -2.2324469089508057,
      "step": 13293
    },
    {
      "epoch": 1.7396090619121616,
      "grad_norm": 11.808816909790039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1609306037425995,
      "logits/rejected": -0.07751664519309998,
      "logps/chosen": -397.27471923828125,
      "logps/rejected": -363.6297302246094,
      "loss": 0.7451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6308714151382446,
      "rewards/margins": 0.01698552817106247,
      "rewards/rejected": -0.6478569507598877,
      "step": 13294
    },
    {
      "epoch": 1.7397399198495134,
      "grad_norm": 7.178737640380859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17193259298801422,
      "logits/rejected": -0.25284212827682495,
      "logps/chosen": -645.2005615234375,
      "logps/rejected": -553.999267578125,
      "loss": 0.2297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5488691329956055,
      "rewards/margins": 1.9115346670150757,
      "rewards/rejected": -2.4604039192199707,
      "step": 13295
    },
    {
      "epoch": 1.7398707777868652,
      "grad_norm": 7.675490856170654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22480884194374084,
      "logits/rejected": -0.27081504464149475,
      "logps/chosen": -530.373779296875,
      "logps/rejected": -469.26214599609375,
      "loss": 0.3028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5567688345909119,
      "rewards/margins": 1.7205474376678467,
      "rewards/rejected": -2.2773163318634033,
      "step": 13296
    },
    {
      "epoch": 1.7400016357242167,
      "grad_norm": 12.139424324035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0849686861038208,
      "logits/rejected": -0.16800905764102936,
      "logps/chosen": -619.8272705078125,
      "logps/rejected": -717.5489501953125,
      "loss": 0.7578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1623066663742065,
      "rewards/margins": 1.0789070129394531,
      "rewards/rejected": -2.24121356010437,
      "step": 13297
    },
    {
      "epoch": 1.7401324936615685,
      "grad_norm": 12.11913013458252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24198874831199646,
      "logits/rejected": -0.0022216886281967163,
      "logps/chosen": -458.93121337890625,
      "logps/rejected": -471.5384826660156,
      "loss": 0.6416,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6580058336257935,
      "rewards/margins": 0.9072895050048828,
      "rewards/rejected": -2.565295457839966,
      "step": 13298
    },
    {
      "epoch": 1.7402633515989203,
      "grad_norm": 7.0114264488220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24630188941955566,
      "logits/rejected": -0.09027185291051865,
      "logps/chosen": -293.4790344238281,
      "logps/rejected": -283.5355224609375,
      "loss": 0.342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6314963102340698,
      "rewards/margins": 1.4525465965270996,
      "rewards/rejected": -3.08404278755188,
      "step": 13299
    },
    {
      "epoch": 1.740394209536272,
      "grad_norm": 7.3057861328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03510044142603874,
      "logits/rejected": -0.04714294523000717,
      "logps/chosen": -494.9982604980469,
      "logps/rejected": -468.8066711425781,
      "loss": 0.3552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.410960078239441,
      "rewards/margins": 1.1060380935668945,
      "rewards/rejected": -2.516998291015625,
      "step": 13300
    },
    {
      "epoch": 1.740525067473624,
      "grad_norm": 8.165985107421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46429145336151123,
      "logits/rejected": -0.28603434562683105,
      "logps/chosen": -461.42401123046875,
      "logps/rejected": -594.0955810546875,
      "loss": 0.3637,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0199248790740967,
      "rewards/margins": 1.1292692422866821,
      "rewards/rejected": -3.1491942405700684,
      "step": 13301
    },
    {
      "epoch": 1.7406559254109757,
      "grad_norm": 14.157163619995117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20569029450416565,
      "logits/rejected": 0.058711208403110504,
      "logps/chosen": -582.7227172851562,
      "logps/rejected": -574.205078125,
      "loss": 0.7701,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3506416082382202,
      "rewards/margins": 0.4266652762889862,
      "rewards/rejected": -1.7773070335388184,
      "step": 13302
    },
    {
      "epoch": 1.7407867833483275,
      "grad_norm": 5.727795600891113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6641702651977539,
      "logits/rejected": -0.630638062953949,
      "logps/chosen": -371.65435791015625,
      "logps/rejected": -331.4658203125,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.514132022857666,
      "rewards/margins": 1.9080106019973755,
      "rewards/rejected": -4.422142505645752,
      "step": 13303
    },
    {
      "epoch": 1.7409176412856793,
      "grad_norm": 62.37437438964844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02055281400680542,
      "logits/rejected": -0.21106919646263123,
      "logps/chosen": -523.4017333984375,
      "logps/rejected": -318.9220886230469,
      "loss": 1.1811,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.620137929916382,
      "rewards/margins": -0.06225365400314331,
      "rewards/rejected": -2.557884454727173,
      "step": 13304
    },
    {
      "epoch": 1.741048499223031,
      "grad_norm": 7.834239482879639,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10943785309791565,
      "logits/rejected": 0.09886690974235535,
      "logps/chosen": -497.2692565917969,
      "logps/rejected": -564.4697265625,
      "loss": 0.3667,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.791426420211792,
      "rewards/margins": 1.7502965927124023,
      "rewards/rejected": -2.5417230129241943,
      "step": 13305
    },
    {
      "epoch": 1.7411793571603829,
      "grad_norm": 6.944091796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004393685609102249,
      "logits/rejected": 0.00971376895904541,
      "logps/chosen": -230.35939025878906,
      "logps/rejected": -341.7908630371094,
      "loss": 0.3162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8495575785636902,
      "rewards/margins": 1.9685697555541992,
      "rewards/rejected": -2.818126916885376,
      "step": 13306
    },
    {
      "epoch": 1.7413102150977346,
      "grad_norm": 10.280699729919434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3792366683483124,
      "logits/rejected": -0.3566761016845703,
      "logps/chosen": -387.4884948730469,
      "logps/rejected": -396.0315856933594,
      "loss": 0.7339,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9442839622497559,
      "rewards/margins": 0.30757391452789307,
      "rewards/rejected": -2.2518579959869385,
      "step": 13307
    },
    {
      "epoch": 1.7414410730350864,
      "grad_norm": 6.053982257843018,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03734026104211807,
      "logits/rejected": 0.04832056909799576,
      "logps/chosen": -682.0228271484375,
      "logps/rejected": -646.1532592773438,
      "loss": 0.1983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5969192981719971,
      "rewards/margins": 1.851150631904602,
      "rewards/rejected": -2.4480700492858887,
      "step": 13308
    },
    {
      "epoch": 1.741571930972438,
      "grad_norm": 8.143239974975586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41616034507751465,
      "logits/rejected": -0.3223457336425781,
      "logps/chosen": -393.75274658203125,
      "logps/rejected": -563.9093017578125,
      "loss": 0.2441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7929993271827698,
      "rewards/margins": 2.3893983364105225,
      "rewards/rejected": -3.1823976039886475,
      "step": 13309
    },
    {
      "epoch": 1.7417027889097898,
      "grad_norm": 10.838836669921875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2652522921562195,
      "logits/rejected": -0.3738487958908081,
      "logps/chosen": -322.9091796875,
      "logps/rejected": -518.2777099609375,
      "loss": 0.5531,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2155853509902954,
      "rewards/margins": 1.4777958393096924,
      "rewards/rejected": -2.6933813095092773,
      "step": 13310
    },
    {
      "epoch": 1.7418336468471416,
      "grad_norm": 9.791933059692383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06383511424064636,
      "logits/rejected": -0.22989130020141602,
      "logps/chosen": -355.2323913574219,
      "logps/rejected": -394.9428405761719,
      "loss": 0.6232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6419158577919006,
      "rewards/margins": 0.5936078429222107,
      "rewards/rejected": -1.2355237007141113,
      "step": 13311
    },
    {
      "epoch": 1.7419645047844934,
      "grad_norm": 4.620604991912842,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11358270794153214,
      "logits/rejected": 0.05719359219074249,
      "logps/chosen": -387.12347412109375,
      "logps/rejected": -350.4409484863281,
      "loss": 0.1668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.007273465394973755,
      "rewards/margins": 2.075727939605713,
      "rewards/rejected": -2.0830016136169434,
      "step": 13312
    },
    {
      "epoch": 1.742095362721845,
      "grad_norm": 5.086094856262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0538920983672142,
      "logits/rejected": -0.17254815995693207,
      "logps/chosen": -619.7420654296875,
      "logps/rejected": -585.1052856445312,
      "loss": 0.2176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2607746720314026,
      "rewards/margins": 2.1006908416748047,
      "rewards/rejected": -2.3614654541015625,
      "step": 13313
    },
    {
      "epoch": 1.7422262206591967,
      "grad_norm": 6.617692470550537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011605799198150635,
      "logits/rejected": -0.11345730721950531,
      "logps/chosen": -449.1385498046875,
      "logps/rejected": -402.79144287109375,
      "loss": 0.3316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1522212028503418,
      "rewards/margins": 1.8160254955291748,
      "rewards/rejected": -2.9682466983795166,
      "step": 13314
    },
    {
      "epoch": 1.7423570785965485,
      "grad_norm": 13.801741600036621,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05537968501448631,
      "logits/rejected": 0.07819949090480804,
      "logps/chosen": -405.8824462890625,
      "logps/rejected": -323.7763671875,
      "loss": 0.6681,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9346208572387695,
      "rewards/margins": 0.4889255166053772,
      "rewards/rejected": -1.4235464334487915,
      "step": 13315
    },
    {
      "epoch": 1.7424879365339003,
      "grad_norm": 12.005871772766113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2652243971824646,
      "logits/rejected": -0.13300731778144836,
      "logps/chosen": -302.3516845703125,
      "logps/rejected": -379.0044860839844,
      "loss": 0.425,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3580574989318848,
      "rewards/margins": 1.7479679584503174,
      "rewards/rejected": -3.106025457382202,
      "step": 13316
    },
    {
      "epoch": 1.7426187944712521,
      "grad_norm": 8.925596237182617,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25173282623291016,
      "logits/rejected": -0.1890583336353302,
      "logps/chosen": -613.648681640625,
      "logps/rejected": -571.043701171875,
      "loss": 0.4313,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2826076745986938,
      "rewards/margins": 1.2555551528930664,
      "rewards/rejected": -2.5381627082824707,
      "step": 13317
    },
    {
      "epoch": 1.742749652408604,
      "grad_norm": 9.365096092224121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19645194709300995,
      "logits/rejected": 0.012445047497749329,
      "logps/chosen": -353.5606689453125,
      "logps/rejected": -510.4058837890625,
      "loss": 0.5036,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0234565734863281,
      "rewards/margins": 0.8869229555130005,
      "rewards/rejected": -1.9103796482086182,
      "step": 13318
    },
    {
      "epoch": 1.7428805103459557,
      "grad_norm": 5.158214092254639,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10218201577663422,
      "logits/rejected": -0.3924608528614044,
      "logps/chosen": -481.8586730957031,
      "logps/rejected": -412.42724609375,
      "loss": 0.3643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7740672826766968,
      "rewards/margins": 1.4192566871643066,
      "rewards/rejected": -2.193324089050293,
      "step": 13319
    },
    {
      "epoch": 1.7430113682833075,
      "grad_norm": 7.005129337310791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11918486654758453,
      "logits/rejected": -0.22574298083782196,
      "logps/chosen": -433.38140869140625,
      "logps/rejected": -469.5305480957031,
      "loss": 0.3274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40457838773727417,
      "rewards/margins": 2.4547433853149414,
      "rewards/rejected": -2.8593220710754395,
      "step": 13320
    },
    {
      "epoch": 1.7431422262206593,
      "grad_norm": 8.442987442016602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20384006202220917,
      "logits/rejected": -0.5119531154632568,
      "logps/chosen": -531.949462890625,
      "logps/rejected": -641.4332275390625,
      "loss": 0.405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9525318741798401,
      "rewards/margins": 1.6244131326675415,
      "rewards/rejected": -2.5769453048706055,
      "step": 13321
    },
    {
      "epoch": 1.743273084158011,
      "grad_norm": 8.490914344787598,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00945257768034935,
      "logits/rejected": 0.018879488110542297,
      "logps/chosen": -550.9612426757812,
      "logps/rejected": -410.69500732421875,
      "loss": 0.3234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5766729116439819,
      "rewards/margins": 2.3606972694396973,
      "rewards/rejected": -2.9373700618743896,
      "step": 13322
    },
    {
      "epoch": 1.7434039420953629,
      "grad_norm": 11.44587516784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2882985770702362,
      "logits/rejected": -0.20670010149478912,
      "logps/chosen": -428.0667724609375,
      "logps/rejected": -480.4204406738281,
      "loss": 0.556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4913538694381714,
      "rewards/margins": 0.7694655060768127,
      "rewards/rejected": -2.260819435119629,
      "step": 13323
    },
    {
      "epoch": 1.7435348000327144,
      "grad_norm": 6.301374435424805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13121667504310608,
      "logits/rejected": -0.44608721137046814,
      "logps/chosen": -462.6291809082031,
      "logps/rejected": -413.55047607421875,
      "loss": 0.2051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5251772403717041,
      "rewards/margins": 2.492438793182373,
      "rewards/rejected": -3.017615795135498,
      "step": 13324
    },
    {
      "epoch": 1.7436656579700662,
      "grad_norm": 7.693636894226074,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015097059309482574,
      "logits/rejected": -0.14545997977256775,
      "logps/chosen": -331.9734802246094,
      "logps/rejected": -469.08355712890625,
      "loss": 0.3517,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0482969284057617,
      "rewards/margins": 2.270627975463867,
      "rewards/rejected": -3.318925142288208,
      "step": 13325
    },
    {
      "epoch": 1.743796515907418,
      "grad_norm": 5.17844820022583,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0792839527130127,
      "logits/rejected": -0.06564764678478241,
      "logps/chosen": -439.8318176269531,
      "logps/rejected": -346.00048828125,
      "loss": 0.3016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9456621408462524,
      "rewards/margins": 2.4019219875335693,
      "rewards/rejected": -3.3475842475891113,
      "step": 13326
    },
    {
      "epoch": 1.7439273738447698,
      "grad_norm": 14.24495792388916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1814424693584442,
      "logits/rejected": -0.08316701650619507,
      "logps/chosen": -549.9786376953125,
      "logps/rejected": -416.1272277832031,
      "loss": 0.6593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2863800525665283,
      "rewards/margins": 0.8212136030197144,
      "rewards/rejected": -2.107593536376953,
      "step": 13327
    },
    {
      "epoch": 1.7440582317821214,
      "grad_norm": 12.972118377685547,
      "learning_rate": 5e-06,
      "logits/chosen": 0.026903199031949043,
      "logits/rejected": -0.0825684517621994,
      "logps/chosen": -442.3636474609375,
      "logps/rejected": -455.5461730957031,
      "loss": 0.5519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2460417747497559,
      "rewards/margins": 0.9347731471061707,
      "rewards/rejected": -2.1808149814605713,
      "step": 13328
    },
    {
      "epoch": 1.7441890897194732,
      "grad_norm": 7.368887901306152,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02304750122129917,
      "logits/rejected": -0.0976511687040329,
      "logps/chosen": -495.4813537597656,
      "logps/rejected": -491.369873046875,
      "loss": 0.4376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3984616994857788,
      "rewards/margins": 1.2517009973526,
      "rewards/rejected": -2.6501624584198,
      "step": 13329
    },
    {
      "epoch": 1.744319947656825,
      "grad_norm": 4.695301055908203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3329150080680847,
      "logits/rejected": -0.5309914350509644,
      "logps/chosen": -340.68292236328125,
      "logps/rejected": -497.46893310546875,
      "loss": 0.2193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9302982091903687,
      "rewards/margins": 2.1017684936523438,
      "rewards/rejected": -3.032066822052002,
      "step": 13330
    },
    {
      "epoch": 1.7444508055941768,
      "grad_norm": 6.83836555480957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23613430559635162,
      "logits/rejected": -0.2870958745479584,
      "logps/chosen": -273.7398986816406,
      "logps/rejected": -310.3575134277344,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6877667903900146,
      "rewards/margins": 1.4918328523635864,
      "rewards/rejected": -3.1795992851257324,
      "step": 13331
    },
    {
      "epoch": 1.7445816635315285,
      "grad_norm": 7.870023727416992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08358947932720184,
      "logits/rejected": -0.06787410378456116,
      "logps/chosen": -351.5362548828125,
      "logps/rejected": -398.0732116699219,
      "loss": 0.2831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9734860062599182,
      "rewards/margins": 1.9394726753234863,
      "rewards/rejected": -2.9129586219787598,
      "step": 13332
    },
    {
      "epoch": 1.7447125214688803,
      "grad_norm": 4.0693840980529785,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060192130506038666,
      "logits/rejected": -0.012123513966798782,
      "logps/chosen": -472.142333984375,
      "logps/rejected": -470.21063232421875,
      "loss": 0.1364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6346806287765503,
      "rewards/margins": 2.358757972717285,
      "rewards/rejected": -2.993438243865967,
      "step": 13333
    },
    {
      "epoch": 1.7448433794062321,
      "grad_norm": 7.714026927947998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41896212100982666,
      "logits/rejected": -0.34251272678375244,
      "logps/chosen": -384.720947265625,
      "logps/rejected": -453.12261962890625,
      "loss": 0.3601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.230594515800476,
      "rewards/margins": 1.8081187009811401,
      "rewards/rejected": -3.038713216781616,
      "step": 13334
    },
    {
      "epoch": 1.744974237343584,
      "grad_norm": 8.749361038208008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.519111156463623,
      "logits/rejected": -0.21692806482315063,
      "logps/chosen": -365.74908447265625,
      "logps/rejected": -546.0127563476562,
      "loss": 0.4433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8747485876083374,
      "rewards/margins": 1.3601230382919312,
      "rewards/rejected": -3.2348716259002686,
      "step": 13335
    },
    {
      "epoch": 1.7451050952809357,
      "grad_norm": 9.680877685546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06542675197124481,
      "logits/rejected": 0.221706360578537,
      "logps/chosen": -496.873046875,
      "logps/rejected": -544.5111694335938,
      "loss": 0.4629,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9191771745681763,
      "rewards/margins": 1.375012993812561,
      "rewards/rejected": -2.2941901683807373,
      "step": 13336
    },
    {
      "epoch": 1.7452359532182875,
      "grad_norm": 10.935613632202148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21233949065208435,
      "logits/rejected": -0.20328356325626373,
      "logps/chosen": -400.0183410644531,
      "logps/rejected": -334.9975280761719,
      "loss": 0.687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.7277891635894775,
      "rewards/margins": 0.3560991585254669,
      "rewards/rejected": -2.083888292312622,
      "step": 13337
    },
    {
      "epoch": 1.7453668111556393,
      "grad_norm": 8.08395004272461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2311955839395523,
      "logits/rejected": -0.26798340678215027,
      "logps/chosen": -608.6279296875,
      "logps/rejected": -489.44775390625,
      "loss": 0.3347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.136614441871643,
      "rewards/margins": 1.1603038311004639,
      "rewards/rejected": -2.2969183921813965,
      "step": 13338
    },
    {
      "epoch": 1.7454976690929909,
      "grad_norm": 9.137825012207031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04470576345920563,
      "logits/rejected": -0.12591196596622467,
      "logps/chosen": -511.0273132324219,
      "logps/rejected": -461.04473876953125,
      "loss": 0.3455,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6583036780357361,
      "rewards/margins": 1.3423230648040771,
      "rewards/rejected": -2.000626802444458,
      "step": 13339
    },
    {
      "epoch": 1.7456285270303427,
      "grad_norm": 10.077652931213379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13009241223335266,
      "logits/rejected": 0.052513763308525085,
      "logps/chosen": -469.21630859375,
      "logps/rejected": -493.9425354003906,
      "loss": 0.5222,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7536557912826538,
      "rewards/margins": 1.3159619569778442,
      "rewards/rejected": -2.069617509841919,
      "step": 13340
    },
    {
      "epoch": 1.7457593849676944,
      "grad_norm": 11.327933311462402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12167239189147949,
      "logits/rejected": -0.1418229043483734,
      "logps/chosen": -720.6478881835938,
      "logps/rejected": -578.8328247070312,
      "loss": 0.6117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7946136593818665,
      "rewards/margins": 1.484820008277893,
      "rewards/rejected": -2.2794337272644043,
      "step": 13341
    },
    {
      "epoch": 1.7458902429050462,
      "grad_norm": 7.345179557800293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13645169138908386,
      "logits/rejected": -0.2055482417345047,
      "logps/chosen": -490.0360107421875,
      "logps/rejected": -359.7866516113281,
      "loss": 0.3679,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3270758390426636,
      "rewards/margins": 1.7150672674179077,
      "rewards/rejected": -3.042142868041992,
      "step": 13342
    },
    {
      "epoch": 1.7460211008423978,
      "grad_norm": 9.66668701171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3287401795387268,
      "logits/rejected": -0.37788838148117065,
      "logps/chosen": -559.5830688476562,
      "logps/rejected": -498.664306640625,
      "loss": 0.3173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.150946855545044,
      "rewards/margins": 1.9503382444381714,
      "rewards/rejected": -3.101284980773926,
      "step": 13343
    },
    {
      "epoch": 1.7461519587797496,
      "grad_norm": 10.18819522857666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11484558880329132,
      "logits/rejected": -0.12239702045917511,
      "logps/chosen": -345.3881530761719,
      "logps/rejected": -333.1734924316406,
      "loss": 0.6709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.275948166847229,
      "rewards/margins": 0.5604183077812195,
      "rewards/rejected": -1.8363665342330933,
      "step": 13344
    },
    {
      "epoch": 1.7462828167171014,
      "grad_norm": 4.371682167053223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27989670634269714,
      "logits/rejected": -0.2490311563014984,
      "logps/chosen": -473.4658508300781,
      "logps/rejected": -607.3091430664062,
      "loss": 0.1968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3309866189956665,
      "rewards/margins": 1.9764554500579834,
      "rewards/rejected": -3.3074421882629395,
      "step": 13345
    },
    {
      "epoch": 1.7464136746544532,
      "grad_norm": 8.898713111877441,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00243426114320755,
      "logits/rejected": -0.08311394602060318,
      "logps/chosen": -463.59149169921875,
      "logps/rejected": -439.02813720703125,
      "loss": 0.5211,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14892198145389557,
      "rewards/margins": 0.7931432723999023,
      "rewards/rejected": -0.9420652985572815,
      "step": 13346
    },
    {
      "epoch": 1.746544532591805,
      "grad_norm": 5.12658166885376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07903921604156494,
      "logits/rejected": -0.04260352998971939,
      "logps/chosen": -423.2154846191406,
      "logps/rejected": -425.7554626464844,
      "loss": 0.2692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0302766561508179,
      "rewards/margins": 1.4071452617645264,
      "rewards/rejected": -2.437422037124634,
      "step": 13347
    },
    {
      "epoch": 1.7466753905291568,
      "grad_norm": 9.727927207946777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23386996984481812,
      "logits/rejected": -0.18455341458320618,
      "logps/chosen": -338.861572265625,
      "logps/rejected": -478.5398864746094,
      "loss": 0.6796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0437686443328857,
      "rewards/margins": 1.282446026802063,
      "rewards/rejected": -2.3262147903442383,
      "step": 13348
    },
    {
      "epoch": 1.7468062484665086,
      "grad_norm": 12.493976593017578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08936136215925217,
      "logits/rejected": -0.26002347469329834,
      "logps/chosen": -605.084228515625,
      "logps/rejected": -568.0361328125,
      "loss": 0.7162,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4541531801223755,
      "rewards/margins": 0.9183641672134399,
      "rewards/rejected": -2.3725171089172363,
      "step": 13349
    },
    {
      "epoch": 1.7469371064038604,
      "grad_norm": 4.073526382446289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11239978671073914,
      "logits/rejected": -0.020820677280426025,
      "logps/chosen": -426.10418701171875,
      "logps/rejected": -440.4710998535156,
      "loss": 0.1689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7091485857963562,
      "rewards/margins": 2.1962270736694336,
      "rewards/rejected": -2.9053757190704346,
      "step": 13350
    },
    {
      "epoch": 1.7470679643412121,
      "grad_norm": 7.276001930236816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11315512657165527,
      "logits/rejected": -0.18342836201190948,
      "logps/chosen": -459.8197326660156,
      "logps/rejected": -476.38897705078125,
      "loss": 0.4476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.166077971458435,
      "rewards/margins": 1.774232029914856,
      "rewards/rejected": -2.94031023979187,
      "step": 13351
    },
    {
      "epoch": 1.747198822278564,
      "grad_norm": 6.694302082061768,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2206311821937561,
      "logits/rejected": -0.1822613924741745,
      "logps/chosen": -440.16534423828125,
      "logps/rejected": -343.9354248046875,
      "loss": 0.2096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2761037349700928,
      "rewards/margins": 2.3133962154388428,
      "rewards/rejected": -3.5894999504089355,
      "step": 13352
    },
    {
      "epoch": 1.7473296802159157,
      "grad_norm": 8.144876480102539,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0686858668923378,
      "logits/rejected": -0.000790763646364212,
      "logps/chosen": -389.9914855957031,
      "logps/rejected": -408.04888916015625,
      "loss": 0.4813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49492010474205017,
      "rewards/margins": 1.1742243766784668,
      "rewards/rejected": -1.669144630432129,
      "step": 13353
    },
    {
      "epoch": 1.7474605381532675,
      "grad_norm": 11.073882102966309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22966012358665466,
      "logits/rejected": -0.08292949199676514,
      "logps/chosen": -548.0761108398438,
      "logps/rejected": -573.7599487304688,
      "loss": 0.6756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6186835765838623,
      "rewards/margins": 1.2499351501464844,
      "rewards/rejected": -2.8686187267303467,
      "step": 13354
    },
    {
      "epoch": 1.747591396090619,
      "grad_norm": 6.292712211608887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11947539448738098,
      "logits/rejected": -0.13096223771572113,
      "logps/chosen": -483.6721496582031,
      "logps/rejected": -449.028076171875,
      "loss": 0.3796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2951973080635071,
      "rewards/margins": 1.2341986894607544,
      "rewards/rejected": -1.5293959379196167,
      "step": 13355
    },
    {
      "epoch": 1.7477222540279709,
      "grad_norm": 9.15105152130127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04822675883769989,
      "logits/rejected": -0.05804252624511719,
      "logps/chosen": -462.1518249511719,
      "logps/rejected": -403.2947998046875,
      "loss": 0.389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6281400322914124,
      "rewards/margins": 1.6529818773269653,
      "rewards/rejected": -2.2811222076416016,
      "step": 13356
    },
    {
      "epoch": 1.7478531119653227,
      "grad_norm": 13.192276000976562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04845185577869415,
      "logits/rejected": -0.05519714951515198,
      "logps/chosen": -439.8607177734375,
      "logps/rejected": -655.5299682617188,
      "loss": 0.7997,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6643345355987549,
      "rewards/margins": 0.6943858861923218,
      "rewards/rejected": -2.358720541000366,
      "step": 13357
    },
    {
      "epoch": 1.7479839699026745,
      "grad_norm": 4.78753662109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34107106924057007,
      "logits/rejected": -0.24460364878177643,
      "logps/chosen": -404.4834289550781,
      "logps/rejected": -459.25189208984375,
      "loss": 0.2182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4806780815124512,
      "rewards/margins": 2.1196584701538086,
      "rewards/rejected": -3.600336790084839,
      "step": 13358
    },
    {
      "epoch": 1.748114827840026,
      "grad_norm": 9.024247169494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04498754069209099,
      "logits/rejected": -0.20453818142414093,
      "logps/chosen": -260.51715087890625,
      "logps/rejected": -334.99822998046875,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2070468664169312,
      "rewards/margins": 1.3353427648544312,
      "rewards/rejected": -2.542389154434204,
      "step": 13359
    },
    {
      "epoch": 1.7482456857773778,
      "grad_norm": 14.064024925231934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06890406459569931,
      "logits/rejected": -0.023356661200523376,
      "logps/chosen": -625.6572875976562,
      "logps/rejected": -555.96240234375,
      "loss": 0.4526,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5944479703903198,
      "rewards/margins": 1.938705325126648,
      "rewards/rejected": -3.533153533935547,
      "step": 13360
    },
    {
      "epoch": 1.7483765437147296,
      "grad_norm": 8.575721740722656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3675553500652313,
      "logits/rejected": -0.21481283009052277,
      "logps/chosen": -631.3256225585938,
      "logps/rejected": -510.93524169921875,
      "loss": 0.5157,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0742751806974411,
      "rewards/margins": 1.6083626747131348,
      "rewards/rejected": -1.6826380491256714,
      "step": 13361
    },
    {
      "epoch": 1.7485074016520814,
      "grad_norm": 18.127830505371094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3467179536819458,
      "logits/rejected": -0.22284208238124847,
      "logps/chosen": -392.8062438964844,
      "logps/rejected": -543.9495849609375,
      "loss": 0.9485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.336463689804077,
      "rewards/margins": 1.6146653890609741,
      "rewards/rejected": -4.951128959655762,
      "step": 13362
    },
    {
      "epoch": 1.7486382595894332,
      "grad_norm": 9.938796997070312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0858728364109993,
      "logits/rejected": -0.45565396547317505,
      "logps/chosen": -493.1567077636719,
      "logps/rejected": -427.3097229003906,
      "loss": 0.4535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6452919244766235,
      "rewards/margins": 1.536378026008606,
      "rewards/rejected": -3.1816697120666504,
      "step": 13363
    },
    {
      "epoch": 1.748769117526785,
      "grad_norm": 7.983409881591797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1315574198961258,
      "logits/rejected": 0.14496329426765442,
      "logps/chosen": -418.0209655761719,
      "logps/rejected": -313.7317199707031,
      "loss": 0.3746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8522465229034424,
      "rewards/margins": 1.8239867687225342,
      "rewards/rejected": -2.6762332916259766,
      "step": 13364
    },
    {
      "epoch": 1.7488999754641368,
      "grad_norm": 14.28640365600586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12599875032901764,
      "logits/rejected": -0.39137181639671326,
      "logps/chosen": -548.7525024414062,
      "logps/rejected": -517.3065185546875,
      "loss": 0.7617,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8427042961120605,
      "rewards/margins": 0.4099366366863251,
      "rewards/rejected": -2.252641201019287,
      "step": 13365
    },
    {
      "epoch": 1.7490308334014886,
      "grad_norm": 11.545574188232422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4353865087032318,
      "logits/rejected": -0.11905603855848312,
      "logps/chosen": -450.0052795410156,
      "logps/rejected": -643.227783203125,
      "loss": 0.6468,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.159574031829834,
      "rewards/margins": 0.8367020487785339,
      "rewards/rejected": -2.9962761402130127,
      "step": 13366
    },
    {
      "epoch": 1.7491616913388404,
      "grad_norm": 15.71814250946045,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13729584217071533,
      "logits/rejected": -0.007320184260606766,
      "logps/chosen": -507.23321533203125,
      "logps/rejected": -305.2915954589844,
      "loss": 0.794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9008619785308838,
      "rewards/margins": 0.11948871612548828,
      "rewards/rejected": -2.020350694656372,
      "step": 13367
    },
    {
      "epoch": 1.7492925492761922,
      "grad_norm": 5.502717971801758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13088491559028625,
      "logits/rejected": -0.40735507011413574,
      "logps/chosen": -479.6595153808594,
      "logps/rejected": -398.0226745605469,
      "loss": 0.1975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1292446851730347,
      "rewards/margins": 2.0570068359375,
      "rewards/rejected": -3.186251640319824,
      "step": 13368
    },
    {
      "epoch": 1.749423407213544,
      "grad_norm": 5.000790119171143,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21656781435012817,
      "logits/rejected": -0.35585370659828186,
      "logps/chosen": -346.552734375,
      "logps/rejected": -337.720458984375,
      "loss": 0.2519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.377621054649353,
      "rewards/margins": 2.1885006427764893,
      "rewards/rejected": -2.566121816635132,
      "step": 13369
    },
    {
      "epoch": 1.7495542651508955,
      "grad_norm": 11.902403831481934,
      "learning_rate": 5e-06,
      "logits/chosen": 0.034731872379779816,
      "logits/rejected": -0.35090163350105286,
      "logps/chosen": -539.4368896484375,
      "logps/rejected": -526.8995361328125,
      "loss": 0.4807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1914823055267334,
      "rewards/margins": 0.9601854681968689,
      "rewards/rejected": -2.151667833328247,
      "step": 13370
    },
    {
      "epoch": 1.7496851230882473,
      "grad_norm": 10.701314926147461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2111234962940216,
      "logits/rejected": -0.27845388650894165,
      "logps/chosen": -383.169189453125,
      "logps/rejected": -315.4958190917969,
      "loss": 0.5385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.659827709197998,
      "rewards/margins": 1.9147980213165283,
      "rewards/rejected": -3.5746257305145264,
      "step": 13371
    },
    {
      "epoch": 1.749815981025599,
      "grad_norm": 7.5978546142578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3097091615200043,
      "logits/rejected": -0.18547794222831726,
      "logps/chosen": -430.7868347167969,
      "logps/rejected": -444.24298095703125,
      "loss": 0.402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.3230040073394775,
      "rewards/margins": 1.5258738994598389,
      "rewards/rejected": -3.8488779067993164,
      "step": 13372
    },
    {
      "epoch": 1.749946838962951,
      "grad_norm": 8.123370170593262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09885257482528687,
      "logits/rejected": -0.1905594766139984,
      "logps/chosen": -350.3687438964844,
      "logps/rejected": -429.2927551269531,
      "loss": 0.5432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40225735306739807,
      "rewards/margins": 1.5302705764770508,
      "rewards/rejected": -1.932528018951416,
      "step": 13373
    },
    {
      "epoch": 1.7500776969003025,
      "grad_norm": 12.09167766571045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11739195883274078,
      "logits/rejected": -0.17702510952949524,
      "logps/chosen": -545.2008056640625,
      "logps/rejected": -463.0645751953125,
      "loss": 0.4862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.983042597770691,
      "rewards/margins": 1.0630239248275757,
      "rewards/rejected": -3.0460665225982666,
      "step": 13374
    },
    {
      "epoch": 1.7502085548376543,
      "grad_norm": 5.257709980010986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24367688596248627,
      "logits/rejected": -0.48788052797317505,
      "logps/chosen": -623.3208618164062,
      "logps/rejected": -454.68072509765625,
      "loss": 0.189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14320656657218933,
      "rewards/margins": 2.233386754989624,
      "rewards/rejected": -2.3765933513641357,
      "step": 13375
    },
    {
      "epoch": 1.750339412775006,
      "grad_norm": 16.299909591674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.046336498111486435,
      "logits/rejected": -0.0718768909573555,
      "logps/chosen": -338.70843505859375,
      "logps/rejected": -343.7081604003906,
      "loss": 0.6991,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.12986421585083,
      "rewards/margins": 0.5271574258804321,
      "rewards/rejected": -2.6570217609405518,
      "step": 13376
    },
    {
      "epoch": 1.7504702707123578,
      "grad_norm": 6.019967079162598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3798271119594574,
      "logits/rejected": -0.4079415202140808,
      "logps/chosen": -221.76451110839844,
      "logps/rejected": -215.34088134765625,
      "loss": 0.535,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.380751132965088,
      "rewards/margins": 1.010818362236023,
      "rewards/rejected": -2.3915693759918213,
      "step": 13377
    },
    {
      "epoch": 1.7506011286497096,
      "grad_norm": 8.235106468200684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1914205700159073,
      "logits/rejected": -0.10094217956066132,
      "logps/chosen": -423.66180419921875,
      "logps/rejected": -401.94171142578125,
      "loss": 0.6622,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1834053993225098,
      "rewards/margins": 0.39595532417297363,
      "rewards/rejected": -1.5793606042861938,
      "step": 13378
    },
    {
      "epoch": 1.7507319865870614,
      "grad_norm": 13.410972595214844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.377358078956604,
      "logits/rejected": -0.49252864718437195,
      "logps/chosen": -457.0063781738281,
      "logps/rejected": -448.19891357421875,
      "loss": 0.7459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3720163106918335,
      "rewards/margins": 0.7716345191001892,
      "rewards/rejected": -2.143650770187378,
      "step": 13379
    },
    {
      "epoch": 1.7508628445244132,
      "grad_norm": 9.570714950561523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023464791476726532,
      "logits/rejected": -0.07057982683181763,
      "logps/chosen": -500.3918762207031,
      "logps/rejected": -438.91497802734375,
      "loss": 0.4826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2060950994491577,
      "rewards/margins": 0.9523630142211914,
      "rewards/rejected": -2.1584579944610596,
      "step": 13380
    },
    {
      "epoch": 1.750993702461765,
      "grad_norm": 8.244292259216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011464908719062805,
      "logits/rejected": -0.04350142553448677,
      "logps/chosen": -526.388427734375,
      "logps/rejected": -539.00927734375,
      "loss": 0.3205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7962820529937744,
      "rewards/margins": 1.3799909353256226,
      "rewards/rejected": -2.1762731075286865,
      "step": 13381
    },
    {
      "epoch": 1.7511245603991168,
      "grad_norm": 11.79341983795166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.354314923286438,
      "logits/rejected": -0.16002270579338074,
      "logps/chosen": -400.00872802734375,
      "logps/rejected": -442.0074768066406,
      "loss": 0.601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4520716667175293,
      "rewards/margins": 0.7789803743362427,
      "rewards/rejected": -2.2310519218444824,
      "step": 13382
    },
    {
      "epoch": 1.7512554183364686,
      "grad_norm": 8.995136260986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010689154267311096,
      "logits/rejected": -0.3145487904548645,
      "logps/chosen": -536.930419921875,
      "logps/rejected": -405.0851135253906,
      "loss": 0.4747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.313661813735962,
      "rewards/margins": 1.204134225845337,
      "rewards/rejected": -2.5177958011627197,
      "step": 13383
    },
    {
      "epoch": 1.7513862762738204,
      "grad_norm": 13.96022891998291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32923805713653564,
      "logits/rejected": -0.4003141522407532,
      "logps/chosen": -609.622802734375,
      "logps/rejected": -560.1990966796875,
      "loss": 1.0859,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4813849925994873,
      "rewards/margins": 0.005554556846618652,
      "rewards/rejected": -1.4869396686553955,
      "step": 13384
    },
    {
      "epoch": 1.751517134211172,
      "grad_norm": 11.315999031066895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17705228924751282,
      "logits/rejected": -0.332527756690979,
      "logps/chosen": -357.08392333984375,
      "logps/rejected": -664.712158203125,
      "loss": 0.7495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1512961387634277,
      "rewards/margins": 1.9439175128936768,
      "rewards/rejected": -4.095213890075684,
      "step": 13385
    },
    {
      "epoch": 1.7516479921485237,
      "grad_norm": 9.8975830078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2012566179037094,
      "logits/rejected": -0.14552029967308044,
      "logps/chosen": -564.5916137695312,
      "logps/rejected": -488.4029846191406,
      "loss": 0.402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7410945892333984,
      "rewards/margins": 1.3435072898864746,
      "rewards/rejected": -2.084601879119873,
      "step": 13386
    },
    {
      "epoch": 1.7517788500858755,
      "grad_norm": 12.55810260772705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1949380785226822,
      "logits/rejected": -0.14779122173786163,
      "logps/chosen": -503.3605041503906,
      "logps/rejected": -384.142822265625,
      "loss": 0.7232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9890354871749878,
      "rewards/margins": 0.48023033142089844,
      "rewards/rejected": -1.4692658185958862,
      "step": 13387
    },
    {
      "epoch": 1.7519097080232273,
      "grad_norm": 11.871674537658691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4025765359401703,
      "logits/rejected": -0.13523022830486298,
      "logps/chosen": -397.13275146484375,
      "logps/rejected": -529.955322265625,
      "loss": 0.6993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8604912161827087,
      "rewards/margins": 0.3723072409629822,
      "rewards/rejected": -1.232798457145691,
      "step": 13388
    },
    {
      "epoch": 1.7520405659605789,
      "grad_norm": 4.99778413772583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20803219079971313,
      "logits/rejected": -0.13769879937171936,
      "logps/chosen": -437.6385498046875,
      "logps/rejected": -535.0462646484375,
      "loss": 0.2054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9275395274162292,
      "rewards/margins": 1.9812723398208618,
      "rewards/rejected": -2.9088120460510254,
      "step": 13389
    },
    {
      "epoch": 1.7521714238979307,
      "grad_norm": 9.827425956726074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02216966450214386,
      "logits/rejected": 0.004068970680236816,
      "logps/chosen": -553.8092041015625,
      "logps/rejected": -557.7325439453125,
      "loss": 0.4279,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4160258173942566,
      "rewards/margins": 1.5494786500930786,
      "rewards/rejected": -1.96550452709198,
      "step": 13390
    },
    {
      "epoch": 1.7523022818352825,
      "grad_norm": 8.9053373336792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48686960339546204,
      "logits/rejected": -0.6755169034004211,
      "logps/chosen": -415.2191162109375,
      "logps/rejected": -370.85528564453125,
      "loss": 0.4753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7022230625152588,
      "rewards/margins": 0.8280418515205383,
      "rewards/rejected": -2.5302648544311523,
      "step": 13391
    },
    {
      "epoch": 1.7524331397726343,
      "grad_norm": 11.126199722290039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.293058305978775,
      "logits/rejected": -0.3228118419647217,
      "logps/chosen": -324.7092590332031,
      "logps/rejected": -409.6789855957031,
      "loss": 0.7665,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.551520586013794,
      "rewards/margins": 0.5470021367073059,
      "rewards/rejected": -2.098522901535034,
      "step": 13392
    },
    {
      "epoch": 1.752563997709986,
      "grad_norm": 4.074131011962891,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16084572672843933,
      "logits/rejected": -0.31845247745513916,
      "logps/chosen": -573.0543212890625,
      "logps/rejected": -474.49371337890625,
      "loss": 0.182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17316266894340515,
      "rewards/margins": 2.1772890090942383,
      "rewards/rejected": -2.350451707839966,
      "step": 13393
    },
    {
      "epoch": 1.7526948556473378,
      "grad_norm": 13.785561561584473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.054460473358631134,
      "logits/rejected": -0.28121957182884216,
      "logps/chosen": -529.7144165039062,
      "logps/rejected": -486.93359375,
      "loss": 0.8411,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12668341398239136,
      "rewards/margins": 0.4845436215400696,
      "rewards/rejected": -0.6112270355224609,
      "step": 13394
    },
    {
      "epoch": 1.7528257135846896,
      "grad_norm": 7.8587117195129395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1700945496559143,
      "logits/rejected": -0.061056412756443024,
      "logps/chosen": -448.7562561035156,
      "logps/rejected": -431.3812561035156,
      "loss": 0.3959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3919249176979065,
      "rewards/margins": 1.010114312171936,
      "rewards/rejected": -1.4020390510559082,
      "step": 13395
    },
    {
      "epoch": 1.7529565715220414,
      "grad_norm": 7.923561096191406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.353049635887146,
      "logits/rejected": -0.48519936203956604,
      "logps/chosen": -559.8987426757812,
      "logps/rejected": -478.6981506347656,
      "loss": 0.4081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5298565030097961,
      "rewards/margins": 1.0955851078033447,
      "rewards/rejected": -1.6254416704177856,
      "step": 13396
    },
    {
      "epoch": 1.7530874294593932,
      "grad_norm": 10.642529487609863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13104790449142456,
      "logits/rejected": -0.3531910479068756,
      "logps/chosen": -613.4108276367188,
      "logps/rejected": -541.110595703125,
      "loss": 0.4802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1856606006622314,
      "rewards/margins": 1.1064562797546387,
      "rewards/rejected": -2.29211688041687,
      "step": 13397
    },
    {
      "epoch": 1.753218287396745,
      "grad_norm": 12.045605659484863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1557227224111557,
      "logits/rejected": -0.032869528979063034,
      "logps/chosen": -444.0262451171875,
      "logps/rejected": -452.2865295410156,
      "loss": 0.5139,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5896399021148682,
      "rewards/margins": 1.5066546201705933,
      "rewards/rejected": -2.096294641494751,
      "step": 13398
    },
    {
      "epoch": 1.7533491453340968,
      "grad_norm": 7.301632404327393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29018571972846985,
      "logits/rejected": -0.4548586308956146,
      "logps/chosen": -516.7777099609375,
      "logps/rejected": -401.2218017578125,
      "loss": 0.3094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7946234345436096,
      "rewards/margins": 1.5037356615066528,
      "rewards/rejected": -2.2983591556549072,
      "step": 13399
    },
    {
      "epoch": 1.7534800032714486,
      "grad_norm": 6.878363132476807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32221144437789917,
      "logits/rejected": -0.5435789227485657,
      "logps/chosen": -503.6350402832031,
      "logps/rejected": -527.3970336914062,
      "loss": 0.3449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.000612258911133,
      "rewards/margins": 1.5936256647109985,
      "rewards/rejected": -3.594238042831421,
      "step": 13400
    },
    {
      "epoch": 1.7536108612088002,
      "grad_norm": 10.87913703918457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37349799275398254,
      "logits/rejected": -0.49557679891586304,
      "logps/chosen": -318.2877502441406,
      "logps/rejected": -500.52117919921875,
      "loss": 0.4886,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3954237699508667,
      "rewards/margins": 1.4345059394836426,
      "rewards/rejected": -2.8299295902252197,
      "step": 13401
    },
    {
      "epoch": 1.753741719146152,
      "grad_norm": 9.730822563171387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04093589633703232,
      "logits/rejected": -0.2555663585662842,
      "logps/chosen": -466.8136291503906,
      "logps/rejected": -330.3006591796875,
      "loss": 0.4479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41586875915527344,
      "rewards/margins": 1.1285037994384766,
      "rewards/rejected": -1.54437255859375,
      "step": 13402
    },
    {
      "epoch": 1.7538725770835037,
      "grad_norm": 11.45166301727295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10213660448789597,
      "logits/rejected": 0.16148625314235687,
      "logps/chosen": -477.9384765625,
      "logps/rejected": -444.2801513671875,
      "loss": 0.42,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5266977548599243,
      "rewards/margins": 1.45476496219635,
      "rewards/rejected": -1.9814627170562744,
      "step": 13403
    },
    {
      "epoch": 1.7540034350208555,
      "grad_norm": 15.538530349731445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3663690984249115,
      "logits/rejected": -0.2979668080806732,
      "logps/chosen": -564.691162109375,
      "logps/rejected": -543.7697143554688,
      "loss": 1.1584,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2477904558181763,
      "rewards/margins": -0.3200598955154419,
      "rewards/rejected": -0.9277305006980896,
      "step": 13404
    },
    {
      "epoch": 1.754134292958207,
      "grad_norm": 9.007550239562988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04048487916588783,
      "logits/rejected": -0.21797224879264832,
      "logps/chosen": -611.8765258789062,
      "logps/rejected": -455.32427978515625,
      "loss": 0.3239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8488262295722961,
      "rewards/margins": 2.511190891265869,
      "rewards/rejected": -3.3600168228149414,
      "step": 13405
    },
    {
      "epoch": 1.754265150895559,
      "grad_norm": 8.43669319152832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12283600866794586,
      "logits/rejected": -0.23841843008995056,
      "logps/chosen": -568.0127563476562,
      "logps/rejected": -520.7774047851562,
      "loss": 0.294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6936454176902771,
      "rewards/margins": 1.8478119373321533,
      "rewards/rejected": -2.5414576530456543,
      "step": 13406
    },
    {
      "epoch": 1.7543960088329107,
      "grad_norm": 9.372781753540039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14000625908374786,
      "logits/rejected": -0.17712661623954773,
      "logps/chosen": -592.378662109375,
      "logps/rejected": -464.3303527832031,
      "loss": 0.4846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1056512594223022,
      "rewards/margins": 1.6169517040252686,
      "rewards/rejected": -2.7226028442382812,
      "step": 13407
    },
    {
      "epoch": 1.7545268667702625,
      "grad_norm": 11.186137199401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25195783376693726,
      "logits/rejected": -0.3605498671531677,
      "logps/chosen": -480.3830871582031,
      "logps/rejected": -396.2680969238281,
      "loss": 0.4779,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1498526334762573,
      "rewards/margins": 1.3478680849075317,
      "rewards/rejected": -2.497720241546631,
      "step": 13408
    },
    {
      "epoch": 1.7546577247076143,
      "grad_norm": 8.418998718261719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33574777841567993,
      "logits/rejected": 0.019887275993824005,
      "logps/chosen": -399.19683837890625,
      "logps/rejected": -366.6036682128906,
      "loss": 0.5003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9554719924926758,
      "rewards/margins": 0.7751529812812805,
      "rewards/rejected": -1.730625033378601,
      "step": 13409
    },
    {
      "epoch": 1.754788582644966,
      "grad_norm": 3.994816303253174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06343488395214081,
      "logits/rejected": -0.21745413541793823,
      "logps/chosen": -363.7959899902344,
      "logps/rejected": -497.422119140625,
      "loss": 0.142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7687808871269226,
      "rewards/margins": 2.975374937057495,
      "rewards/rejected": -3.7441558837890625,
      "step": 13410
    },
    {
      "epoch": 1.7549194405823179,
      "grad_norm": 9.275092124938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05548980087041855,
      "logits/rejected": -0.37612324953079224,
      "logps/chosen": -353.884033203125,
      "logps/rejected": -423.3469543457031,
      "loss": 0.3292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2235748767852783,
      "rewards/margins": 1.701336145401001,
      "rewards/rejected": -2.9249110221862793,
      "step": 13411
    },
    {
      "epoch": 1.7550502985196696,
      "grad_norm": 8.727152824401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09200736880302429,
      "logits/rejected": -0.1787550300359726,
      "logps/chosen": -484.1959533691406,
      "logps/rejected": -315.7353515625,
      "loss": 0.2279,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20661486685276031,
      "rewards/margins": 2.954535961151123,
      "rewards/rejected": -2.7479212284088135,
      "step": 13412
    },
    {
      "epoch": 1.7551811564570214,
      "grad_norm": 7.902124881744385,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017718099057674408,
      "logits/rejected": -0.04063110053539276,
      "logps/chosen": -536.454345703125,
      "logps/rejected": -455.94830322265625,
      "loss": 0.4134,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.008674256503582,
      "rewards/margins": 1.1853443384170532,
      "rewards/rejected": -1.1940186023712158,
      "step": 13413
    },
    {
      "epoch": 1.7553120143943732,
      "grad_norm": 9.233201026916504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4898706376552582,
      "logits/rejected": -0.5487191677093506,
      "logps/chosen": -769.0762939453125,
      "logps/rejected": -507.03924560546875,
      "loss": 0.3568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3718066215515137,
      "rewards/margins": 1.3894140720367432,
      "rewards/rejected": -2.761220932006836,
      "step": 13414
    },
    {
      "epoch": 1.755442872331725,
      "grad_norm": 6.3301167488098145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.163563072681427,
      "logits/rejected": -0.040075868368148804,
      "logps/chosen": -496.7771911621094,
      "logps/rejected": -515.6607055664062,
      "loss": 0.214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5547338128089905,
      "rewards/margins": 1.820893406867981,
      "rewards/rejected": -2.375627279281616,
      "step": 13415
    },
    {
      "epoch": 1.7555737302690766,
      "grad_norm": 9.00935173034668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0163672287017107,
      "logits/rejected": -0.07564304769039154,
      "logps/chosen": -513.32763671875,
      "logps/rejected": -367.3682556152344,
      "loss": 0.3651,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8822012543678284,
      "rewards/margins": 1.456723690032959,
      "rewards/rejected": -2.3389248847961426,
      "step": 13416
    },
    {
      "epoch": 1.7557045882064284,
      "grad_norm": 7.981457233428955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15666210651397705,
      "logits/rejected": -0.3101886808872223,
      "logps/chosen": -545.2657470703125,
      "logps/rejected": -528.2139282226562,
      "loss": 0.3457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0549736022949219,
      "rewards/margins": 1.1798840761184692,
      "rewards/rejected": -2.2348577976226807,
      "step": 13417
    },
    {
      "epoch": 1.7558354461437802,
      "grad_norm": 10.054832458496094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13663700222969055,
      "logits/rejected": -0.13237264752388,
      "logps/chosen": -458.123046875,
      "logps/rejected": -305.1127624511719,
      "loss": 0.4857,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9186787009239197,
      "rewards/margins": 1.301363468170166,
      "rewards/rejected": -2.2200422286987305,
      "step": 13418
    },
    {
      "epoch": 1.755966304081132,
      "grad_norm": 6.0236945152282715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2180362045764923,
      "logits/rejected": -0.15855267643928528,
      "logps/chosen": -543.4451293945312,
      "logps/rejected": -353.3652038574219,
      "loss": 0.2609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7487019896507263,
      "rewards/margins": 2.409388542175293,
      "rewards/rejected": -3.158090591430664,
      "step": 13419
    },
    {
      "epoch": 1.7560971620184835,
      "grad_norm": 6.546275615692139,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14372220635414124,
      "logits/rejected": 0.1254250705242157,
      "logps/chosen": -443.54803466796875,
      "logps/rejected": -413.85406494140625,
      "loss": 0.3305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24838274717330933,
      "rewards/margins": 1.8360341787338257,
      "rewards/rejected": -2.0844168663024902,
      "step": 13420
    },
    {
      "epoch": 1.7562280199558353,
      "grad_norm": 10.578597068786621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34850335121154785,
      "logits/rejected": 0.02499542012810707,
      "logps/chosen": -555.22998046875,
      "logps/rejected": -565.7308349609375,
      "loss": 0.637,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3023343086242676,
      "rewards/margins": 0.4440368711948395,
      "rewards/rejected": -1.7463711500167847,
      "step": 13421
    },
    {
      "epoch": 1.7563588778931871,
      "grad_norm": 14.19268798828125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037855010479688644,
      "logits/rejected": -0.14386437833309174,
      "logps/chosen": -649.1268920898438,
      "logps/rejected": -656.024169921875,
      "loss": 0.616,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0088528394699097,
      "rewards/margins": 0.9174267053604126,
      "rewards/rejected": -1.9262795448303223,
      "step": 13422
    },
    {
      "epoch": 1.756489735830539,
      "grad_norm": 10.345036506652832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29147571325302124,
      "logits/rejected": 0.01988382078707218,
      "logps/chosen": -269.8400573730469,
      "logps/rejected": -408.1661682128906,
      "loss": 0.6003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3460758328437805,
      "rewards/margins": 0.5590242147445679,
      "rewards/rejected": -0.9051001071929932,
      "step": 13423
    },
    {
      "epoch": 1.7566205937678907,
      "grad_norm": 10.514372825622559,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10158558934926987,
      "logits/rejected": 0.10518883168697357,
      "logps/chosen": -405.7308349609375,
      "logps/rejected": -349.7789001464844,
      "loss": 0.6795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.145829200744629,
      "rewards/margins": 0.47819799184799194,
      "rewards/rejected": -1.6240272521972656,
      "step": 13424
    },
    {
      "epoch": 1.7567514517052425,
      "grad_norm": 9.004439353942871,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06072056293487549,
      "logits/rejected": -0.21942153573036194,
      "logps/chosen": -567.0140380859375,
      "logps/rejected": -534.7586669921875,
      "loss": 0.3862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1886141002178192,
      "rewards/margins": 1.959030032157898,
      "rewards/rejected": -1.770416021347046,
      "step": 13425
    },
    {
      "epoch": 1.7568823096425943,
      "grad_norm": 4.543817520141602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1588033139705658,
      "logits/rejected": -0.10654538869857788,
      "logps/chosen": -585.357421875,
      "logps/rejected": -516.7314453125,
      "loss": 0.2491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03907202184200287,
      "rewards/margins": 1.4917223453521729,
      "rewards/rejected": -1.530794382095337,
      "step": 13426
    },
    {
      "epoch": 1.757013167579946,
      "grad_norm": 9.514776229858398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3394700884819031,
      "logits/rejected": -0.19110620021820068,
      "logps/chosen": -542.35791015625,
      "logps/rejected": -530.02783203125,
      "loss": 0.634,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4408767819404602,
      "rewards/margins": 0.781091570854187,
      "rewards/rejected": -1.221968173980713,
      "step": 13427
    },
    {
      "epoch": 1.7571440255172979,
      "grad_norm": 9.312288284301758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18838846683502197,
      "logits/rejected": -0.3011988401412964,
      "logps/chosen": -436.3526916503906,
      "logps/rejected": -483.5194091796875,
      "loss": 0.4851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1954773664474487,
      "rewards/margins": 1.4731072187423706,
      "rewards/rejected": -2.6685845851898193,
      "step": 13428
    },
    {
      "epoch": 1.7572748834546497,
      "grad_norm": 15.59144115447998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13732510805130005,
      "logits/rejected": -0.42176151275634766,
      "logps/chosen": -390.5763244628906,
      "logps/rejected": -411.2294616699219,
      "loss": 0.6396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.648678183555603,
      "rewards/margins": 1.2448617219924927,
      "rewards/rejected": -2.8935396671295166,
      "step": 13429
    },
    {
      "epoch": 1.7574057413920015,
      "grad_norm": 8.447458267211914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34215691685676575,
      "logits/rejected": -0.573853611946106,
      "logps/chosen": -425.96209716796875,
      "logps/rejected": -412.8547058105469,
      "loss": 0.3874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8318408727645874,
      "rewards/margins": 1.48240065574646,
      "rewards/rejected": -2.314241647720337,
      "step": 13430
    },
    {
      "epoch": 1.757536599329353,
      "grad_norm": 6.150256156921387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14068228006362915,
      "logits/rejected": -0.1364929974079132,
      "logps/chosen": -529.1626586914062,
      "logps/rejected": -432.2575988769531,
      "loss": 0.3197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04635611176490784,
      "rewards/margins": 1.5364999771118164,
      "rewards/rejected": -1.5828561782836914,
      "step": 13431
    },
    {
      "epoch": 1.7576674572667048,
      "grad_norm": 14.485780715942383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3285701274871826,
      "logits/rejected": -0.007502362132072449,
      "logps/chosen": -513.7567138671875,
      "logps/rejected": -572.4821166992188,
      "loss": 0.8468,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3348944187164307,
      "rewards/margins": 0.6214576363563538,
      "rewards/rejected": -1.9563519954681396,
      "step": 13432
    },
    {
      "epoch": 1.7577983152040566,
      "grad_norm": 7.0590386390686035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3050227463245392,
      "logits/rejected": -0.22468429803848267,
      "logps/chosen": -228.20335388183594,
      "logps/rejected": -277.1405029296875,
      "loss": 0.4856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.289841890335083,
      "rewards/margins": 0.6697930097579956,
      "rewards/rejected": -1.9596349000930786,
      "step": 13433
    },
    {
      "epoch": 1.7579291731414084,
      "grad_norm": 8.168254852294922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.026359617710113525,
      "logits/rejected": -0.10867241024971008,
      "logps/chosen": -511.3627624511719,
      "logps/rejected": -555.9541625976562,
      "loss": 0.3473,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18584822118282318,
      "rewards/margins": 1.1903294324874878,
      "rewards/rejected": -1.3761775493621826,
      "step": 13434
    },
    {
      "epoch": 1.75806003107876,
      "grad_norm": 5.992012023925781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4699060618877411,
      "logits/rejected": -0.32350149750709534,
      "logps/chosen": -487.5561828613281,
      "logps/rejected": -510.52825927734375,
      "loss": 0.3071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.258297324180603,
      "rewards/margins": 1.6386001110076904,
      "rewards/rejected": -1.8968974351882935,
      "step": 13435
    },
    {
      "epoch": 1.7581908890161118,
      "grad_norm": 11.995755195617676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.094159796833992,
      "logits/rejected": -0.18923255801200867,
      "logps/chosen": -406.68218994140625,
      "logps/rejected": -496.29217529296875,
      "loss": 0.7795,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.906777560710907,
      "rewards/margins": 0.11948377639055252,
      "rewards/rejected": -1.026261329650879,
      "step": 13436
    },
    {
      "epoch": 1.7583217469534635,
      "grad_norm": 6.757328510284424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3607534170150757,
      "logits/rejected": -0.3738378882408142,
      "logps/chosen": -562.1207275390625,
      "logps/rejected": -425.733642578125,
      "loss": 0.4428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3709051012992859,
      "rewards/margins": 1.4635249376296997,
      "rewards/rejected": -1.83443021774292,
      "step": 13437
    },
    {
      "epoch": 1.7584526048908153,
      "grad_norm": 7.148034572601318,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3817425072193146,
      "logits/rejected": -0.30642107129096985,
      "logps/chosen": -458.8329162597656,
      "logps/rejected": -416.9390869140625,
      "loss": 0.4573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.092968463897705,
      "rewards/margins": 1.195937991142273,
      "rewards/rejected": -2.2889063358306885,
      "step": 13438
    },
    {
      "epoch": 1.7585834628281671,
      "grad_norm": 8.752205848693848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3188855051994324,
      "logits/rejected": -0.140851691365242,
      "logps/chosen": -580.320068359375,
      "logps/rejected": -497.741455078125,
      "loss": 0.4124,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7805607318878174,
      "rewards/margins": 1.4402035474777222,
      "rewards/rejected": -2.22076416015625,
      "step": 13439
    },
    {
      "epoch": 1.758714320765519,
      "grad_norm": 4.818070411682129,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10405541211366653,
      "logits/rejected": 0.13450247049331665,
      "logps/chosen": -577.5911865234375,
      "logps/rejected": -649.46484375,
      "loss": 0.2096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8424968719482422,
      "rewards/margins": 1.7775695323944092,
      "rewards/rejected": -0.9350727200508118,
      "step": 13440
    },
    {
      "epoch": 1.7588451787028707,
      "grad_norm": 13.556286811828613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1934855878353119,
      "logits/rejected": -0.3036023676395416,
      "logps/chosen": -403.9779052734375,
      "logps/rejected": -408.2064514160156,
      "loss": 0.6292,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2495133876800537,
      "rewards/margins": 0.4744489789009094,
      "rewards/rejected": -1.7239623069763184,
      "step": 13441
    },
    {
      "epoch": 1.7589760366402225,
      "grad_norm": 6.3473124504089355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04928593337535858,
      "logits/rejected": -0.20746856927871704,
      "logps/chosen": -314.1658020019531,
      "logps/rejected": -342.10906982421875,
      "loss": 0.2941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3731672465801239,
      "rewards/margins": 1.7330095767974854,
      "rewards/rejected": -2.1061766147613525,
      "step": 13442
    },
    {
      "epoch": 1.7591068945775743,
      "grad_norm": 5.93696403503418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18462078273296356,
      "logits/rejected": 0.09810560941696167,
      "logps/chosen": -432.43408203125,
      "logps/rejected": -596.621337890625,
      "loss": 0.2072,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02033841609954834,
      "rewards/margins": 2.547161102294922,
      "rewards/rejected": -2.526822805404663,
      "step": 13443
    },
    {
      "epoch": 1.759237752514926,
      "grad_norm": 11.810050964355469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13268572092056274,
      "logits/rejected": -0.29516103863716125,
      "logps/chosen": -419.2696228027344,
      "logps/rejected": -500.9414367675781,
      "loss": 0.6325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9281783699989319,
      "rewards/margins": 0.4853583574295044,
      "rewards/rejected": -1.413536548614502,
      "step": 13444
    },
    {
      "epoch": 1.7593686104522779,
      "grad_norm": 10.990887641906738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2096388041973114,
      "logits/rejected": -0.3204362094402313,
      "logps/chosen": -370.6607971191406,
      "logps/rejected": -422.25537109375,
      "loss": 0.4249,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1285052299499512,
      "rewards/margins": 1.8496787548065186,
      "rewards/rejected": -2.9781839847564697,
      "step": 13445
    },
    {
      "epoch": 1.7594994683896297,
      "grad_norm": 12.03596019744873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30897510051727295,
      "logits/rejected": -0.14856848120689392,
      "logps/chosen": -382.45050048828125,
      "logps/rejected": -357.34930419921875,
      "loss": 0.753,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7904205322265625,
      "rewards/margins": 0.8010330200195312,
      "rewards/rejected": -2.5914533138275146,
      "step": 13446
    },
    {
      "epoch": 1.7596303263269812,
      "grad_norm": 11.908004760742188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023333847522735596,
      "logits/rejected": 0.08010488748550415,
      "logps/chosen": -638.544677734375,
      "logps/rejected": -616.6676635742188,
      "loss": 0.548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4564967751502991,
      "rewards/margins": 1.3978850841522217,
      "rewards/rejected": -1.854382038116455,
      "step": 13447
    },
    {
      "epoch": 1.759761184264333,
      "grad_norm": 8.429394721984863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21768303215503693,
      "logits/rejected": -0.17432229220867157,
      "logps/chosen": -527.6795654296875,
      "logps/rejected": -490.2260437011719,
      "loss": 0.375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3567615747451782,
      "rewards/margins": 1.743950605392456,
      "rewards/rejected": -2.1007118225097656,
      "step": 13448
    },
    {
      "epoch": 1.7598920422016848,
      "grad_norm": 10.654921531677246,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14064882695674896,
      "logits/rejected": 0.02980373054742813,
      "logps/chosen": -608.5225830078125,
      "logps/rejected": -548.6292724609375,
      "loss": 0.5474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6132761836051941,
      "rewards/margins": 0.8451669216156006,
      "rewards/rejected": -1.45844304561615,
      "step": 13449
    },
    {
      "epoch": 1.7600229001390364,
      "grad_norm": 8.102201461791992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3279179334640503,
      "logits/rejected": -0.3340407609939575,
      "logps/chosen": -332.46734619140625,
      "logps/rejected": -356.5606994628906,
      "loss": 0.4436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5884720087051392,
      "rewards/margins": 0.9939565658569336,
      "rewards/rejected": -1.5824285745620728,
      "step": 13450
    },
    {
      "epoch": 1.7601537580763882,
      "grad_norm": 7.9817328453063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12567460536956787,
      "logits/rejected": -0.29963767528533936,
      "logps/chosen": -477.0248718261719,
      "logps/rejected": -331.8633117675781,
      "loss": 0.4282,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1862119734287262,
      "rewards/margins": 1.4926153421401978,
      "rewards/rejected": -1.6788274049758911,
      "step": 13451
    },
    {
      "epoch": 1.76028461601374,
      "grad_norm": 9.058748245239258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16888917982578278,
      "logits/rejected": 0.014768749475479126,
      "logps/chosen": -466.48553466796875,
      "logps/rejected": -459.0939636230469,
      "loss": 0.4747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4789096117019653,
      "rewards/margins": 1.0467815399169922,
      "rewards/rejected": -2.525691032409668,
      "step": 13452
    },
    {
      "epoch": 1.7604154739510918,
      "grad_norm": 11.607714653015137,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14672765135765076,
      "logits/rejected": -0.005837973207235336,
      "logps/chosen": -628.4317626953125,
      "logps/rejected": -601.630126953125,
      "loss": 0.4405,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44155311584472656,
      "rewards/margins": 1.5258575677871704,
      "rewards/rejected": -1.9674108028411865,
      "step": 13453
    },
    {
      "epoch": 1.7605463318884436,
      "grad_norm": 11.684657096862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47441890835762024,
      "logits/rejected": 0.04962480813264847,
      "logps/chosen": -256.1334533691406,
      "logps/rejected": -369.4684143066406,
      "loss": 0.7598,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5824419260025024,
      "rewards/margins": 0.0585705041885376,
      "rewards/rejected": -1.6410125494003296,
      "step": 13454
    },
    {
      "epoch": 1.7606771898257954,
      "grad_norm": 6.013425827026367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06746591627597809,
      "logits/rejected": -0.3322833180427551,
      "logps/chosen": -582.9174194335938,
      "logps/rejected": -527.9436645507812,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1989341825246811,
      "rewards/margins": 2.696563482284546,
      "rewards/rejected": -2.497629165649414,
      "step": 13455
    },
    {
      "epoch": 1.7608080477631471,
      "grad_norm": 11.413350105285645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.195596843957901,
      "logits/rejected": -0.21866658329963684,
      "logps/chosen": -395.969482421875,
      "logps/rejected": -486.15106201171875,
      "loss": 0.5195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5565402507781982,
      "rewards/margins": 0.8544612526893616,
      "rewards/rejected": -2.411001443862915,
      "step": 13456
    },
    {
      "epoch": 1.760938905700499,
      "grad_norm": 9.590514183044434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.412126362323761,
      "logits/rejected": -0.525987982749939,
      "logps/chosen": -514.6055908203125,
      "logps/rejected": -996.8328857421875,
      "loss": 0.6153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0543296337127686,
      "rewards/margins": 0.9273037910461426,
      "rewards/rejected": -2.9816336631774902,
      "step": 13457
    },
    {
      "epoch": 1.7610697636378507,
      "grad_norm": 8.65514087677002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05096207931637764,
      "logits/rejected": -0.2863020598888397,
      "logps/chosen": -433.527099609375,
      "logps/rejected": -556.6376342773438,
      "loss": 0.4308,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0206114053726196,
      "rewards/margins": 1.6756547689437866,
      "rewards/rejected": -2.6962661743164062,
      "step": 13458
    },
    {
      "epoch": 1.7612006215752025,
      "grad_norm": 5.123533725738525,
      "learning_rate": 5e-06,
      "logits/chosen": 0.018218576908111572,
      "logits/rejected": -0.11674581468105316,
      "logps/chosen": -416.5335693359375,
      "logps/rejected": -500.6620178222656,
      "loss": 0.2523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33331814408302307,
      "rewards/margins": 1.9499776363372803,
      "rewards/rejected": -2.2832956314086914,
      "step": 13459
    },
    {
      "epoch": 1.7613314795125543,
      "grad_norm": 4.536540985107422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22577589750289917,
      "logits/rejected": -0.28524965047836304,
      "logps/chosen": -387.63043212890625,
      "logps/rejected": -389.6697998046875,
      "loss": 0.2171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8060630559921265,
      "rewards/margins": 2.1717381477355957,
      "rewards/rejected": -2.9778013229370117,
      "step": 13460
    },
    {
      "epoch": 1.761462337449906,
      "grad_norm": 10.41192626953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09358716011047363,
      "logits/rejected": -0.25742197036743164,
      "logps/chosen": -351.4533996582031,
      "logps/rejected": -387.34173583984375,
      "loss": 0.4435,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4492518901824951,
      "rewards/margins": 1.1574429273605347,
      "rewards/rejected": -2.6066949367523193,
      "step": 13461
    },
    {
      "epoch": 1.7615931953872577,
      "grad_norm": 11.490029335021973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10378210991621017,
      "logits/rejected": -0.0484129823744297,
      "logps/chosen": -414.80859375,
      "logps/rejected": -459.54254150390625,
      "loss": 0.6139,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.386397123336792,
      "rewards/margins": 0.7884420156478882,
      "rewards/rejected": -2.1748392581939697,
      "step": 13462
    },
    {
      "epoch": 1.7617240533246095,
      "grad_norm": 3.7982547283172607,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32178252935409546,
      "logits/rejected": -0.4534164071083069,
      "logps/chosen": -310.49639892578125,
      "logps/rejected": -389.300048828125,
      "loss": 0.1615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1052608489990234,
      "rewards/margins": 2.429511547088623,
      "rewards/rejected": -3.5347723960876465,
      "step": 13463
    },
    {
      "epoch": 1.7618549112619613,
      "grad_norm": 3.9545071125030518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0712367594242096,
      "logits/rejected": -0.39237719774246216,
      "logps/chosen": -352.63067626953125,
      "logps/rejected": -574.75146484375,
      "loss": 0.1564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.558845043182373,
      "rewards/margins": 2.477911949157715,
      "rewards/rejected": -4.03675651550293,
      "step": 13464
    },
    {
      "epoch": 1.761985769199313,
      "grad_norm": 7.839081287384033,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4336007237434387,
      "logits/rejected": -0.38532334566116333,
      "logps/chosen": -449.35455322265625,
      "logps/rejected": -314.1549987792969,
      "loss": 0.3776,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.947672724723816,
      "rewards/margins": 1.179853916168213,
      "rewards/rejected": -3.1275267601013184,
      "step": 13465
    },
    {
      "epoch": 1.7621166271366646,
      "grad_norm": 7.777982234954834,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0289338119328022,
      "logits/rejected": -0.10504016280174255,
      "logps/chosen": -355.36322021484375,
      "logps/rejected": -446.7500915527344,
      "loss": 0.3889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9454845786094666,
      "rewards/margins": 1.452410340309143,
      "rewards/rejected": -2.397894859313965,
      "step": 13466
    },
    {
      "epoch": 1.7622474850740164,
      "grad_norm": 14.689153671264648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14126789569854736,
      "logits/rejected": -0.45281240344047546,
      "logps/chosen": -508.2398376464844,
      "logps/rejected": -402.2913818359375,
      "loss": 0.7499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7110531330108643,
      "rewards/margins": 0.6619107723236084,
      "rewards/rejected": -2.3729641437530518,
      "step": 13467
    },
    {
      "epoch": 1.7623783430113682,
      "grad_norm": 6.662759780883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08237996697425842,
      "logits/rejected": -0.40652650594711304,
      "logps/chosen": -467.08477783203125,
      "logps/rejected": -656.513671875,
      "loss": 0.2429,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.526148796081543,
      "rewards/margins": 1.9108326435089111,
      "rewards/rejected": -2.436981678009033,
      "step": 13468
    },
    {
      "epoch": 1.76250920094872,
      "grad_norm": 7.3386383056640625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23936741054058075,
      "logits/rejected": -0.21286699175834656,
      "logps/chosen": -443.2613525390625,
      "logps/rejected": -366.4849853515625,
      "loss": 0.472,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.622002124786377,
      "rewards/margins": 1.1787291765213013,
      "rewards/rejected": -2.8007311820983887,
      "step": 13469
    },
    {
      "epoch": 1.7626400588860718,
      "grad_norm": 6.697105407714844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3008902966976166,
      "logits/rejected": -0.25581154227256775,
      "logps/chosen": -374.535888671875,
      "logps/rejected": -348.96929931640625,
      "loss": 0.407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.472472071647644,
      "rewards/margins": 1.2426165342330933,
      "rewards/rejected": -2.7150886058807373,
      "step": 13470
    },
    {
      "epoch": 1.7627709168234236,
      "grad_norm": 7.74301290512085,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0513324961066246,
      "logits/rejected": -0.2543048560619354,
      "logps/chosen": -419.08282470703125,
      "logps/rejected": -404.7348937988281,
      "loss": 0.3605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2526226043701172,
      "rewards/margins": 2.4527554512023926,
      "rewards/rejected": -3.705378293991089,
      "step": 13471
    },
    {
      "epoch": 1.7629017747607754,
      "grad_norm": 11.630988121032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13897716999053955,
      "logits/rejected": -0.4657832384109497,
      "logps/chosen": -338.7561950683594,
      "logps/rejected": -334.6034851074219,
      "loss": 0.7902,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6412098407745361,
      "rewards/margins": 0.9758976697921753,
      "rewards/rejected": -2.617107629776001,
      "step": 13472
    },
    {
      "epoch": 1.7630326326981272,
      "grad_norm": 6.009936809539795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21971815824508667,
      "logits/rejected": -0.26735320687294006,
      "logps/chosen": -529.5150146484375,
      "logps/rejected": -404.9486083984375,
      "loss": 0.3299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5305939316749573,
      "rewards/margins": 2.4715521335601807,
      "rewards/rejected": -3.002146005630493,
      "step": 13473
    },
    {
      "epoch": 1.763163490635479,
      "grad_norm": 4.117116928100586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11762424558401108,
      "logits/rejected": -0.1671137511730194,
      "logps/chosen": -502.0388488769531,
      "logps/rejected": -549.5127563476562,
      "loss": 0.1923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7789831161499023,
      "rewards/margins": 2.3129889965057373,
      "rewards/rejected": -3.0919718742370605,
      "step": 13474
    },
    {
      "epoch": 1.7632943485728307,
      "grad_norm": 11.83969783782959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4928792119026184,
      "logits/rejected": -0.007131204009056091,
      "logps/chosen": -355.72705078125,
      "logps/rejected": -361.44329833984375,
      "loss": 0.7704,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.968856930732727,
      "rewards/margins": 0.5340068936347961,
      "rewards/rejected": -2.502863645553589,
      "step": 13475
    },
    {
      "epoch": 1.7634252065101825,
      "grad_norm": 13.566970825195312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18258976936340332,
      "logits/rejected": -0.08047336339950562,
      "logps/chosen": -345.8628845214844,
      "logps/rejected": -400.21417236328125,
      "loss": 0.6792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0539339780807495,
      "rewards/margins": 0.8744192123413086,
      "rewards/rejected": -1.928353190422058,
      "step": 13476
    },
    {
      "epoch": 1.763556064447534,
      "grad_norm": 3.9223227500915527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11680445075035095,
      "logits/rejected": -0.25298845767974854,
      "logps/chosen": -472.1733703613281,
      "logps/rejected": -476.7176208496094,
      "loss": 0.2444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.728593111038208,
      "rewards/margins": 1.761242151260376,
      "rewards/rejected": -2.489835262298584,
      "step": 13477
    },
    {
      "epoch": 1.763686922384886,
      "grad_norm": 10.5064697265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4075523018836975,
      "logits/rejected": -0.38918280601501465,
      "logps/chosen": -467.89385986328125,
      "logps/rejected": -414.2046813964844,
      "loss": 0.3498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5086070895195007,
      "rewards/margins": 2.191528797149658,
      "rewards/rejected": -2.700136423110962,
      "step": 13478
    },
    {
      "epoch": 1.7638177803222377,
      "grad_norm": 8.52600383758545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5671995878219604,
      "logits/rejected": -0.5484066605567932,
      "logps/chosen": -400.9215087890625,
      "logps/rejected": -346.6974182128906,
      "loss": 0.4408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.329559564590454,
      "rewards/margins": 1.0571041107177734,
      "rewards/rejected": -2.3866639137268066,
      "step": 13479
    },
    {
      "epoch": 1.7639486382595895,
      "grad_norm": 8.09172248840332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10090039670467377,
      "logits/rejected": -0.39481836557388306,
      "logps/chosen": -332.230224609375,
      "logps/rejected": -385.3304748535156,
      "loss": 0.3805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3620684146881104,
      "rewards/margins": 1.5660368204116821,
      "rewards/rejected": -2.928105115890503,
      "step": 13480
    },
    {
      "epoch": 1.764079496196941,
      "grad_norm": 8.239482879638672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7086859941482544,
      "logits/rejected": -0.4776817560195923,
      "logps/chosen": -263.83306884765625,
      "logps/rejected": -425.0680236816406,
      "loss": 0.4931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.821847677230835,
      "rewards/margins": 1.5416007041931152,
      "rewards/rejected": -3.3634486198425293,
      "step": 13481
    },
    {
      "epoch": 1.7642103541342928,
      "grad_norm": 9.637709617614746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19813655316829681,
      "logits/rejected": -0.027859196066856384,
      "logps/chosen": -358.01324462890625,
      "logps/rejected": -386.94598388671875,
      "loss": 0.4146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5444746017456055,
      "rewards/margins": 1.4095380306243896,
      "rewards/rejected": -2.954012632369995,
      "step": 13482
    },
    {
      "epoch": 1.7643412120716446,
      "grad_norm": 12.78478717803955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37893563508987427,
      "logits/rejected": -0.2263377606868744,
      "logps/chosen": -511.76556396484375,
      "logps/rejected": -507.1883850097656,
      "loss": 0.7206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1439403295516968,
      "rewards/margins": 1.0296943187713623,
      "rewards/rejected": -2.1736345291137695,
      "step": 13483
    },
    {
      "epoch": 1.7644720700089964,
      "grad_norm": 10.857420921325684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.040781088173389435,
      "logits/rejected": -0.21420451998710632,
      "logps/chosen": -428.7457580566406,
      "logps/rejected": -404.37908935546875,
      "loss": 0.4845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9831507205963135,
      "rewards/margins": 0.7483797073364258,
      "rewards/rejected": -2.73153018951416,
      "step": 13484
    },
    {
      "epoch": 1.7646029279463482,
      "grad_norm": 8.69197940826416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5251511931419373,
      "logits/rejected": -0.5817661285400391,
      "logps/chosen": -404.20831298828125,
      "logps/rejected": -379.73858642578125,
      "loss": 0.4032,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0574166774749756,
      "rewards/margins": 1.0995025634765625,
      "rewards/rejected": -3.156919002532959,
      "step": 13485
    },
    {
      "epoch": 1.7647337858837,
      "grad_norm": 9.927386283874512,
      "learning_rate": 5e-06,
      "logits/chosen": 0.3132026493549347,
      "logits/rejected": 0.16519784927368164,
      "logps/chosen": -572.2296142578125,
      "logps/rejected": -469.73162841796875,
      "loss": 0.3141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.014491319656372,
      "rewards/margins": 1.8067972660064697,
      "rewards/rejected": -2.8212883472442627,
      "step": 13486
    },
    {
      "epoch": 1.7648646438210518,
      "grad_norm": 12.718886375427246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3842298984527588,
      "logits/rejected": -0.26713505387306213,
      "logps/chosen": -495.80316162109375,
      "logps/rejected": -377.86376953125,
      "loss": 0.8828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6516448259353638,
      "rewards/margins": 0.028356581926345825,
      "rewards/rejected": -1.6800014972686768,
      "step": 13487
    },
    {
      "epoch": 1.7649955017584036,
      "grad_norm": 7.692934036254883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3389066755771637,
      "logits/rejected": -0.37976640462875366,
      "logps/chosen": -496.40618896484375,
      "logps/rejected": -414.49774169921875,
      "loss": 0.4159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8069844245910645,
      "rewards/margins": 1.7968968152999878,
      "rewards/rejected": -2.603881359100342,
      "step": 13488
    },
    {
      "epoch": 1.7651263596957554,
      "grad_norm": 5.571345806121826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4358859658241272,
      "logits/rejected": -0.3146872818470001,
      "logps/chosen": -573.1952514648438,
      "logps/rejected": -398.5089111328125,
      "loss": 0.2397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.813416063785553,
      "rewards/margins": 1.8130285739898682,
      "rewards/rejected": -2.6264445781707764,
      "step": 13489
    },
    {
      "epoch": 1.7652572176331072,
      "grad_norm": 10.188919067382812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2978366017341614,
      "logits/rejected": -0.20253817737102509,
      "logps/chosen": -399.2508850097656,
      "logps/rejected": -454.1477355957031,
      "loss": 0.4988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7858655452728271,
      "rewards/margins": 0.8568044900894165,
      "rewards/rejected": -2.642669916152954,
      "step": 13490
    },
    {
      "epoch": 1.765388075570459,
      "grad_norm": 9.69792652130127,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07335147261619568,
      "logits/rejected": -0.10273948311805725,
      "logps/chosen": -487.3358154296875,
      "logps/rejected": -516.75341796875,
      "loss": 0.4615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6453163623809814,
      "rewards/margins": 1.1121610403060913,
      "rewards/rejected": -2.7574775218963623,
      "step": 13491
    },
    {
      "epoch": 1.7655189335078105,
      "grad_norm": 8.747682571411133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4028291404247284,
      "logits/rejected": -0.36703255772590637,
      "logps/chosen": -589.1235961914062,
      "logps/rejected": -466.9911193847656,
      "loss": 0.2743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5754567384719849,
      "rewards/margins": 1.850206732749939,
      "rewards/rejected": -3.425663471221924,
      "step": 13492
    },
    {
      "epoch": 1.7656497914451623,
      "grad_norm": 8.477028846740723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17213135957717896,
      "logits/rejected": -0.4356863498687744,
      "logps/chosen": -506.823974609375,
      "logps/rejected": -448.7219543457031,
      "loss": 0.4687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2765551805496216,
      "rewards/margins": 1.737921118736267,
      "rewards/rejected": -3.0144765377044678,
      "step": 13493
    },
    {
      "epoch": 1.7657806493825141,
      "grad_norm": 9.928352355957031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12635859847068787,
      "logits/rejected": -0.2813686728477478,
      "logps/chosen": -352.94061279296875,
      "logps/rejected": -464.28875732421875,
      "loss": 0.38,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8835192322731018,
      "rewards/margins": 2.1578595638275146,
      "rewards/rejected": -3.0413787364959717,
      "step": 13494
    },
    {
      "epoch": 1.765911507319866,
      "grad_norm": 8.004132270812988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16320312023162842,
      "logits/rejected": -0.43935829401016235,
      "logps/chosen": -374.7407531738281,
      "logps/rejected": -513.5852661132812,
      "loss": 0.4137,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.582776427268982,
      "rewards/margins": 1.5553691387176514,
      "rewards/rejected": -3.138145685195923,
      "step": 13495
    },
    {
      "epoch": 1.7660423652572175,
      "grad_norm": 12.246187210083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17040231823921204,
      "logits/rejected": -0.4950564503669739,
      "logps/chosen": -718.660888671875,
      "logps/rejected": -479.987060546875,
      "loss": 0.5182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5208884477615356,
      "rewards/margins": 1.514535665512085,
      "rewards/rejected": -3.03542423248291,
      "step": 13496
    },
    {
      "epoch": 1.7661732231945693,
      "grad_norm": 5.760526657104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20716246962547302,
      "logits/rejected": -0.08063197135925293,
      "logps/chosen": -708.11279296875,
      "logps/rejected": -627.2877197265625,
      "loss": 0.1982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2760116457939148,
      "rewards/margins": 2.5710160732269287,
      "rewards/rejected": -2.84702730178833,
      "step": 13497
    },
    {
      "epoch": 1.766304081131921,
      "grad_norm": 5.344732284545898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3564736843109131,
      "logits/rejected": -0.3785232901573181,
      "logps/chosen": -645.3737182617188,
      "logps/rejected": -611.3035888671875,
      "loss": 0.1953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9288719296455383,
      "rewards/margins": 2.2170329093933105,
      "rewards/rejected": -3.1459052562713623,
      "step": 13498
    },
    {
      "epoch": 1.7664349390692728,
      "grad_norm": 6.097219467163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4183301627635956,
      "logits/rejected": -0.5738771557807922,
      "logps/chosen": -362.2831726074219,
      "logps/rejected": -442.0321044921875,
      "loss": 0.3111,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.223970651626587,
      "rewards/margins": 1.7772140502929688,
      "rewards/rejected": -3.0011849403381348,
      "step": 13499
    },
    {
      "epoch": 1.7665657970066246,
      "grad_norm": 8.050856590270996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06963745504617691,
      "logits/rejected": -0.03755645081400871,
      "logps/chosen": -491.460205078125,
      "logps/rejected": -391.69927978515625,
      "loss": 0.3985,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4589424133300781,
      "rewards/margins": 1.6982789039611816,
      "rewards/rejected": -3.1572213172912598,
      "step": 13500
    },
    {
      "epoch": 1.7666966549439764,
      "grad_norm": 7.442277431488037,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21732792258262634,
      "logits/rejected": -0.16267666220664978,
      "logps/chosen": -627.8341064453125,
      "logps/rejected": -571.9464111328125,
      "loss": 0.3554,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44087183475494385,
      "rewards/margins": 1.2727344036102295,
      "rewards/rejected": -1.7136061191558838,
      "step": 13501
    },
    {
      "epoch": 1.7668275128813282,
      "grad_norm": 6.444464683532715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29528313875198364,
      "logits/rejected": -0.4852709174156189,
      "logps/chosen": -390.2039794921875,
      "logps/rejected": -431.3717041015625,
      "loss": 0.4158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5593137741088867,
      "rewards/margins": 0.9814305305480957,
      "rewards/rejected": -1.5407443046569824,
      "step": 13502
    },
    {
      "epoch": 1.76695837081868,
      "grad_norm": 9.422711372375488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11330925673246384,
      "logits/rejected": -0.4920436143875122,
      "logps/chosen": -473.5218811035156,
      "logps/rejected": -352.9238586425781,
      "loss": 0.6464,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3788284063339233,
      "rewards/margins": 0.6150569915771484,
      "rewards/rejected": -1.9938852787017822,
      "step": 13503
    },
    {
      "epoch": 1.7670892287560318,
      "grad_norm": 8.324830055236816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3915882110595703,
      "logits/rejected": -0.7206150889396667,
      "logps/chosen": -479.9715576171875,
      "logps/rejected": -622.7244873046875,
      "loss": 0.3668,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9298944473266602,
      "rewards/margins": 3.0327486991882324,
      "rewards/rejected": -3.9626429080963135,
      "step": 13504
    },
    {
      "epoch": 1.7672200866933836,
      "grad_norm": 10.042143821716309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1806267350912094,
      "logits/rejected": -0.33499667048454285,
      "logps/chosen": -367.84649658203125,
      "logps/rejected": -438.54638671875,
      "loss": 0.5705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7470600605010986,
      "rewards/margins": 1.663792371749878,
      "rewards/rejected": -3.4108524322509766,
      "step": 13505
    },
    {
      "epoch": 1.7673509446307354,
      "grad_norm": 8.492234230041504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2984084486961365,
      "logits/rejected": -0.3710814118385315,
      "logps/chosen": -511.24530029296875,
      "logps/rejected": -330.9788513183594,
      "loss": 0.4663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9190689325332642,
      "rewards/margins": 1.7155308723449707,
      "rewards/rejected": -2.6346001625061035,
      "step": 13506
    },
    {
      "epoch": 1.7674818025680872,
      "grad_norm": 8.13984489440918,
      "learning_rate": 5e-06,
      "logits/chosen": 0.005415413063019514,
      "logits/rejected": 0.048054710030555725,
      "logps/chosen": -498.7992248535156,
      "logps/rejected": -522.6622314453125,
      "loss": 0.4094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8023866415023804,
      "rewards/margins": 2.1417384147644043,
      "rewards/rejected": -3.944124698638916,
      "step": 13507
    },
    {
      "epoch": 1.7676126605054387,
      "grad_norm": 6.037178993225098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08377274870872498,
      "logits/rejected": -0.053761936724185944,
      "logps/chosen": -579.8021240234375,
      "logps/rejected": -517.520751953125,
      "loss": 0.3089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6706344485282898,
      "rewards/margins": 1.447424054145813,
      "rewards/rejected": -2.118058443069458,
      "step": 13508
    },
    {
      "epoch": 1.7677435184427905,
      "grad_norm": 8.192547798156738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30386167764663696,
      "logits/rejected": -0.3751541078090668,
      "logps/chosen": -399.7461242675781,
      "logps/rejected": -492.85504150390625,
      "loss": 0.3225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5674587488174438,
      "rewards/margins": 2.044743537902832,
      "rewards/rejected": -3.6122026443481445,
      "step": 13509
    },
    {
      "epoch": 1.7678743763801423,
      "grad_norm": 7.777634143829346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11372045427560806,
      "logits/rejected": -0.1177707314491272,
      "logps/chosen": -335.03997802734375,
      "logps/rejected": -367.01959228515625,
      "loss": 0.25,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8935132026672363,
      "rewards/margins": 2.3475451469421387,
      "rewards/rejected": -3.241058349609375,
      "step": 13510
    },
    {
      "epoch": 1.7680052343174941,
      "grad_norm": 5.095685005187988,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1238035261631012,
      "logits/rejected": -0.1248689517378807,
      "logps/chosen": -394.63232421875,
      "logps/rejected": -439.7124328613281,
      "loss": 0.2423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7226299047470093,
      "rewards/margins": 2.4935214519500732,
      "rewards/rejected": -3.216151475906372,
      "step": 13511
    },
    {
      "epoch": 1.7681360922548457,
      "grad_norm": 10.215010643005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3285233676433563,
      "logits/rejected": -0.3907034397125244,
      "logps/chosen": -392.7784423828125,
      "logps/rejected": -436.5948181152344,
      "loss": 0.6586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9657900929450989,
      "rewards/margins": 1.1744009256362915,
      "rewards/rejected": -2.140191078186035,
      "step": 13512
    },
    {
      "epoch": 1.7682669501921975,
      "grad_norm": 4.668676376342773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07676254212856293,
      "logits/rejected": -0.4557858109474182,
      "logps/chosen": -483.7010498046875,
      "logps/rejected": -464.1432189941406,
      "loss": 0.2273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1331030130386353,
      "rewards/margins": 2.5119619369506836,
      "rewards/rejected": -3.6450648307800293,
      "step": 13513
    },
    {
      "epoch": 1.7683978081295493,
      "grad_norm": 8.868973731994629,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06186696141958237,
      "logits/rejected": -0.02500251680612564,
      "logps/chosen": -239.25515747070312,
      "logps/rejected": -356.407470703125,
      "loss": 0.5515,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.957786202430725,
      "rewards/margins": 2.0179712772369385,
      "rewards/rejected": -3.975757598876953,
      "step": 13514
    },
    {
      "epoch": 1.768528666066901,
      "grad_norm": 13.838390350341797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35033050179481506,
      "logits/rejected": -0.15293912589550018,
      "logps/chosen": -736.3056640625,
      "logps/rejected": -691.0946655273438,
      "loss": 0.5517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7322654724121094,
      "rewards/margins": 1.0956883430480957,
      "rewards/rejected": -2.827954053878784,
      "step": 13515
    },
    {
      "epoch": 1.7686595240042529,
      "grad_norm": 5.571542263031006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027314621955156326,
      "logits/rejected": -0.3502920866012573,
      "logps/chosen": -487.48443603515625,
      "logps/rejected": -456.64385986328125,
      "loss": 0.2169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9917194247245789,
      "rewards/margins": 2.0912163257598877,
      "rewards/rejected": -3.0829358100891113,
      "step": 13516
    },
    {
      "epoch": 1.7687903819416047,
      "grad_norm": 8.854790687561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4023786187171936,
      "logits/rejected": -0.35216647386550903,
      "logps/chosen": -393.18316650390625,
      "logps/rejected": -421.60400390625,
      "loss": 0.4696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1964257955551147,
      "rewards/margins": 0.8528023958206177,
      "rewards/rejected": -2.0492281913757324,
      "step": 13517
    },
    {
      "epoch": 1.7689212398789564,
      "grad_norm": 5.978557109832764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3455362617969513,
      "logits/rejected": -0.5337755680084229,
      "logps/chosen": -331.64306640625,
      "logps/rejected": -356.1065979003906,
      "loss": 0.3587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9999733567237854,
      "rewards/margins": 1.1908738613128662,
      "rewards/rejected": -2.190847158432007,
      "step": 13518
    },
    {
      "epoch": 1.7690520978163082,
      "grad_norm": 17.672801971435547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2610117197036743,
      "logits/rejected": -0.25320398807525635,
      "logps/chosen": -615.6726684570312,
      "logps/rejected": -746.8941040039062,
      "loss": 0.8715,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.3163416385650635,
      "rewards/margins": -0.09524162113666534,
      "rewards/rejected": -2.221099853515625,
      "step": 13519
    },
    {
      "epoch": 1.76918295575366,
      "grad_norm": 7.096145153045654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25772732496261597,
      "logits/rejected": -0.480368971824646,
      "logps/chosen": -455.46343994140625,
      "logps/rejected": -447.9320068359375,
      "loss": 0.353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5575220584869385,
      "rewards/margins": 1.0045859813690186,
      "rewards/rejected": -2.562108039855957,
      "step": 13520
    },
    {
      "epoch": 1.7693138136910118,
      "grad_norm": 7.667313575744629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11334449052810669,
      "logits/rejected": -0.3231409788131714,
      "logps/chosen": -373.9445495605469,
      "logps/rejected": -433.4389343261719,
      "loss": 0.3001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9623032808303833,
      "rewards/margins": 2.2111563682556152,
      "rewards/rejected": -4.173460006713867,
      "step": 13521
    },
    {
      "epoch": 1.7694446716283636,
      "grad_norm": 3.972224473953247,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15941326320171356,
      "logits/rejected": -0.2738727033138275,
      "logps/chosen": -455.5337219238281,
      "logps/rejected": -452.7957458496094,
      "loss": 0.1439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1517150402069092,
      "rewards/margins": 2.5525400638580322,
      "rewards/rejected": -3.7042553424835205,
      "step": 13522
    },
    {
      "epoch": 1.7695755295657152,
      "grad_norm": 6.84015417098999,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40710294246673584,
      "logits/rejected": -0.48964256048202515,
      "logps/chosen": -304.7037658691406,
      "logps/rejected": -279.3158874511719,
      "loss": 0.4244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9415239691734314,
      "rewards/margins": 1.1965118646621704,
      "rewards/rejected": -2.138036012649536,
      "step": 13523
    },
    {
      "epoch": 1.769706387503067,
      "grad_norm": 7.803854465484619,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39845454692840576,
      "logits/rejected": -0.24271942675113678,
      "logps/chosen": -341.8293151855469,
      "logps/rejected": -437.24847412109375,
      "loss": 0.3446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38518258929252625,
      "rewards/margins": 1.5505599975585938,
      "rewards/rejected": -1.935742735862732,
      "step": 13524
    },
    {
      "epoch": 1.7698372454404188,
      "grad_norm": 13.78132438659668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25845271348953247,
      "logits/rejected": -0.4304317533969879,
      "logps/chosen": -300.7580261230469,
      "logps/rejected": -396.45037841796875,
      "loss": 0.7725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.0544376373291016,
      "rewards/margins": 0.25080668926239014,
      "rewards/rejected": -3.3052444458007812,
      "step": 13525
    },
    {
      "epoch": 1.7699681033777706,
      "grad_norm": 6.9550628662109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15298104286193848,
      "logits/rejected": -0.03120991215109825,
      "logps/chosen": -518.2102661132812,
      "logps/rejected": -558.816162109375,
      "loss": 0.355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5147119760513306,
      "rewards/margins": 1.1566035747528076,
      "rewards/rejected": -1.6713156700134277,
      "step": 13526
    },
    {
      "epoch": 1.7700989613151221,
      "grad_norm": 4.315977573394775,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25371402502059937,
      "logits/rejected": -0.13346700370311737,
      "logps/chosen": -453.94482421875,
      "logps/rejected": -505.808837890625,
      "loss": 0.2405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.404030680656433,
      "rewards/margins": 1.8730034828186035,
      "rewards/rejected": -3.277034044265747,
      "step": 13527
    },
    {
      "epoch": 1.770229819252474,
      "grad_norm": 10.776483535766602,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010518625378608704,
      "logits/rejected": -0.1021227091550827,
      "logps/chosen": -416.6295166015625,
      "logps/rejected": -302.90594482421875,
      "loss": 0.5096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0124564170837402,
      "rewards/margins": 1.314500093460083,
      "rewards/rejected": -2.3269567489624023,
      "step": 13528
    },
    {
      "epoch": 1.7703606771898257,
      "grad_norm": 9.6094388961792,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1738288849592209,
      "logits/rejected": -0.24664132297039032,
      "logps/chosen": -675.3533325195312,
      "logps/rejected": -626.1495361328125,
      "loss": 0.542,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5265983939170837,
      "rewards/margins": 1.1986900568008423,
      "rewards/rejected": -1.7252885103225708,
      "step": 13529
    },
    {
      "epoch": 1.7704915351271775,
      "grad_norm": 10.641898155212402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4224585294723511,
      "logits/rejected": -0.14461997151374817,
      "logps/chosen": -454.13946533203125,
      "logps/rejected": -542.765380859375,
      "loss": 0.4694,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.15176260471344,
      "rewards/margins": 1.5136046409606934,
      "rewards/rejected": -2.665367364883423,
      "step": 13530
    },
    {
      "epoch": 1.7706223930645293,
      "grad_norm": 13.228139877319336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3506464958190918,
      "logits/rejected": -0.44391584396362305,
      "logps/chosen": -617.4906005859375,
      "logps/rejected": -484.7178955078125,
      "loss": 0.5924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4344381093978882,
      "rewards/margins": 0.8185530304908752,
      "rewards/rejected": -1.2529910802841187,
      "step": 13531
    },
    {
      "epoch": 1.770753251001881,
      "grad_norm": 19.817516326904297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34935444593429565,
      "logits/rejected": -0.6165614724159241,
      "logps/chosen": -531.339111328125,
      "logps/rejected": -356.5568542480469,
      "loss": 1.1338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1172502040863037,
      "rewards/margins": 0.4182378649711609,
      "rewards/rejected": -2.5354881286621094,
      "step": 13532
    },
    {
      "epoch": 1.7708841089392329,
      "grad_norm": 8.094338417053223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06927238404750824,
      "logits/rejected": -0.10306274890899658,
      "logps/chosen": -347.4539489746094,
      "logps/rejected": -324.7891540527344,
      "loss": 0.4252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.622121810913086,
      "rewards/margins": 1.4447941780090332,
      "rewards/rejected": -3.06691575050354,
      "step": 13533
    },
    {
      "epoch": 1.7710149668765847,
      "grad_norm": 7.851129055023193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32823020219802856,
      "logits/rejected": -0.2661314904689789,
      "logps/chosen": -444.96160888671875,
      "logps/rejected": -339.4358825683594,
      "loss": 0.4107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9585727453231812,
      "rewards/margins": 1.0731865167617798,
      "rewards/rejected": -2.031759262084961,
      "step": 13534
    },
    {
      "epoch": 1.7711458248139365,
      "grad_norm": 9.349196434020996,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08682715147733688,
      "logits/rejected": -0.07423071563243866,
      "logps/chosen": -598.9100952148438,
      "logps/rejected": -602.6751098632812,
      "loss": 0.4112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4948531985282898,
      "rewards/margins": 1.1194782257080078,
      "rewards/rejected": -1.6143314838409424,
      "step": 13535
    },
    {
      "epoch": 1.7712766827512882,
      "grad_norm": 2.079498767852783,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0033160336315631866,
      "logits/rejected": -0.2559534013271332,
      "logps/chosen": -354.8731689453125,
      "logps/rejected": -423.6405029296875,
      "loss": 0.1704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8912037014961243,
      "rewards/margins": 2.902681589126587,
      "rewards/rejected": -3.7938849925994873,
      "step": 13536
    },
    {
      "epoch": 1.77140754068864,
      "grad_norm": 6.668931007385254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4343791604042053,
      "logits/rejected": -0.3159967362880707,
      "logps/chosen": -444.6678771972656,
      "logps/rejected": -560.5092163085938,
      "loss": 0.3127,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2643924951553345,
      "rewards/margins": 2.085399866104126,
      "rewards/rejected": -2.34979248046875,
      "step": 13537
    },
    {
      "epoch": 1.7715383986259916,
      "grad_norm": 11.838447570800781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16399520635604858,
      "logits/rejected": -0.26749494671821594,
      "logps/chosen": -424.079345703125,
      "logps/rejected": -431.8961181640625,
      "loss": 0.5704,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9134352207183838,
      "rewards/margins": 0.9260246753692627,
      "rewards/rejected": -2.8394598960876465,
      "step": 13538
    },
    {
      "epoch": 1.7716692565633434,
      "grad_norm": 8.943828582763672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2327682375907898,
      "logits/rejected": -0.3546367287635803,
      "logps/chosen": -378.1434631347656,
      "logps/rejected": -394.4123229980469,
      "loss": 0.4899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9128103852272034,
      "rewards/margins": 1.2344796657562256,
      "rewards/rejected": -2.1472902297973633,
      "step": 13539
    },
    {
      "epoch": 1.7718001145006952,
      "grad_norm": 6.202132701873779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10790698975324631,
      "logits/rejected": -0.2803354859352112,
      "logps/chosen": -415.2608337402344,
      "logps/rejected": -413.13299560546875,
      "loss": 0.2647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7977064251899719,
      "rewards/margins": 1.6164757013320923,
      "rewards/rejected": -2.41418194770813,
      "step": 13540
    },
    {
      "epoch": 1.771930972438047,
      "grad_norm": 4.237295627593994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26103341579437256,
      "logits/rejected": -0.4187470078468323,
      "logps/chosen": -334.6295471191406,
      "logps/rejected": -409.8887939453125,
      "loss": 0.2797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3507682681083679,
      "rewards/margins": 2.289865255355835,
      "rewards/rejected": -2.6406335830688477,
      "step": 13541
    },
    {
      "epoch": 1.7720618303753986,
      "grad_norm": 8.780899047851562,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23470744490623474,
      "logits/rejected": -0.06799868494272232,
      "logps/chosen": -274.695068359375,
      "logps/rejected": -267.14251708984375,
      "loss": 0.531,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.385372281074524,
      "rewards/margins": 1.3335223197937012,
      "rewards/rejected": -2.7188944816589355,
      "step": 13542
    },
    {
      "epoch": 1.7721926883127503,
      "grad_norm": 12.900741577148438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.025804869830608368,
      "logits/rejected": -0.04634793847799301,
      "logps/chosen": -375.5914306640625,
      "logps/rejected": -476.930419921875,
      "loss": 0.8613,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.233569860458374,
      "rewards/margins": -0.09937997162342072,
      "rewards/rejected": -2.1341896057128906,
      "step": 13543
    },
    {
      "epoch": 1.7723235462501021,
      "grad_norm": 12.548686027526855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4486468434333801,
      "logits/rejected": -0.33426451683044434,
      "logps/chosen": -547.54248046875,
      "logps/rejected": -404.7201232910156,
      "loss": 0.738,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5353624820709229,
      "rewards/margins": 0.9328929781913757,
      "rewards/rejected": -2.4682552814483643,
      "step": 13544
    },
    {
      "epoch": 1.772454404187454,
      "grad_norm": 4.649957180023193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027408188208937645,
      "logits/rejected": -0.11245498061180115,
      "logps/chosen": -529.5376586914062,
      "logps/rejected": -510.0071105957031,
      "loss": 0.3188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6621201038360596,
      "rewards/margins": 2.6328117847442627,
      "rewards/rejected": -3.2949318885803223,
      "step": 13545
    },
    {
      "epoch": 1.7725852621248057,
      "grad_norm": 11.047689437866211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17337091267108917,
      "logits/rejected": -0.16404959559440613,
      "logps/chosen": -581.6587524414062,
      "logps/rejected": -495.4368896484375,
      "loss": 0.495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3090834617614746,
      "rewards/margins": 1.2444262504577637,
      "rewards/rejected": -3.5535097122192383,
      "step": 13546
    },
    {
      "epoch": 1.7727161200621575,
      "grad_norm": 10.290078163146973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06427450478076935,
      "logits/rejected": -0.1826033741235733,
      "logps/chosen": -487.31890869140625,
      "logps/rejected": -319.5532531738281,
      "loss": 0.5378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9288879632949829,
      "rewards/margins": 0.520978569984436,
      "rewards/rejected": -1.449866533279419,
      "step": 13547
    },
    {
      "epoch": 1.7728469779995093,
      "grad_norm": 8.466948509216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5750643014907837,
      "logits/rejected": -0.7123535871505737,
      "logps/chosen": -396.35003662109375,
      "logps/rejected": -472.72100830078125,
      "loss": 0.5022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6578145027160645,
      "rewards/margins": 0.9622843265533447,
      "rewards/rejected": -2.620098829269409,
      "step": 13548
    },
    {
      "epoch": 1.772977835936861,
      "grad_norm": 9.029741287231445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3694305121898651,
      "logits/rejected": -0.37049055099487305,
      "logps/chosen": -453.7640075683594,
      "logps/rejected": -523.902587890625,
      "loss": 0.356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5447087287902832,
      "rewards/margins": 1.922776222229004,
      "rewards/rejected": -3.467484951019287,
      "step": 13549
    },
    {
      "epoch": 1.7731086938742129,
      "grad_norm": 12.266124725341797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.410419762134552,
      "logits/rejected": -0.3626282513141632,
      "logps/chosen": -371.5162048339844,
      "logps/rejected": -406.0615539550781,
      "loss": 0.4499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9174991846084595,
      "rewards/margins": 1.9542478322982788,
      "rewards/rejected": -3.871746778488159,
      "step": 13550
    },
    {
      "epoch": 1.7732395518115647,
      "grad_norm": 11.140323638916016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3233332633972168,
      "logits/rejected": -0.2914549708366394,
      "logps/chosen": -436.1421813964844,
      "logps/rejected": -459.35821533203125,
      "loss": 0.4242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.249574065208435,
      "rewards/margins": 1.2121385335922241,
      "rewards/rejected": -2.461712598800659,
      "step": 13551
    },
    {
      "epoch": 1.7733704097489165,
      "grad_norm": 10.515704154968262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03097914159297943,
      "logits/rejected": -0.058371651917696,
      "logps/chosen": -534.535400390625,
      "logps/rejected": -464.474609375,
      "loss": 0.4727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8244290351867676,
      "rewards/margins": 1.7088849544525146,
      "rewards/rejected": -2.533313512802124,
      "step": 13552
    },
    {
      "epoch": 1.7735012676862683,
      "grad_norm": 7.687005996704102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12613630294799805,
      "logits/rejected": -0.14018307626247406,
      "logps/chosen": -388.1872863769531,
      "logps/rejected": -425.9588623046875,
      "loss": 0.472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8042107820510864,
      "rewards/margins": 1.0055527687072754,
      "rewards/rejected": -2.809763193130493,
      "step": 13553
    },
    {
      "epoch": 1.7736321256236198,
      "grad_norm": 8.0743408203125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29310131072998047,
      "logits/rejected": -0.2514607310295105,
      "logps/chosen": -515.2191772460938,
      "logps/rejected": -376.29193115234375,
      "loss": 0.3526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7069165706634521,
      "rewards/margins": 1.4825304746627808,
      "rewards/rejected": -3.1894469261169434,
      "step": 13554
    },
    {
      "epoch": 1.7737629835609716,
      "grad_norm": 6.719058990478516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14261622726917267,
      "logits/rejected": -0.35962289571762085,
      "logps/chosen": -442.2458801269531,
      "logps/rejected": -461.2779541015625,
      "loss": 0.3404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9033740758895874,
      "rewards/margins": 0.9649373888969421,
      "rewards/rejected": -2.8683114051818848,
      "step": 13555
    },
    {
      "epoch": 1.7738938414983234,
      "grad_norm": 11.72775650024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2868707478046417,
      "logits/rejected": -0.25480151176452637,
      "logps/chosen": -500.0598449707031,
      "logps/rejected": -550.2171630859375,
      "loss": 0.5599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6536669731140137,
      "rewards/margins": 0.8606351613998413,
      "rewards/rejected": -2.5143020153045654,
      "step": 13556
    },
    {
      "epoch": 1.774024699435675,
      "grad_norm": 10.212545394897461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0447113923728466,
      "logits/rejected": -0.06509188562631607,
      "logps/chosen": -498.3533935546875,
      "logps/rejected": -453.5563659667969,
      "loss": 0.5538,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3907920122146606,
      "rewards/margins": 1.1304199695587158,
      "rewards/rejected": -2.521212100982666,
      "step": 13557
    },
    {
      "epoch": 1.7741555573730268,
      "grad_norm": 11.166563987731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4301300048828125,
      "logits/rejected": -0.4356592297554016,
      "logps/chosen": -431.7279968261719,
      "logps/rejected": -675.3336791992188,
      "loss": 0.4726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7391576766967773,
      "rewards/margins": 1.3754469156265259,
      "rewards/rejected": -3.1146047115325928,
      "step": 13558
    },
    {
      "epoch": 1.7742864153103786,
      "grad_norm": 9.044853210449219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08162906020879745,
      "logits/rejected": -0.18212544918060303,
      "logps/chosen": -517.1327514648438,
      "logps/rejected": -622.169677734375,
      "loss": 0.4853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8793171644210815,
      "rewards/margins": 1.0977892875671387,
      "rewards/rejected": -1.9771063327789307,
      "step": 13559
    },
    {
      "epoch": 1.7744172732477304,
      "grad_norm": 10.635952949523926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04583171010017395,
      "logits/rejected": -0.05447077378630638,
      "logps/chosen": -418.5700378417969,
      "logps/rejected": -565.1013793945312,
      "loss": 0.6374,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.7927757501602173,
      "rewards/margins": 1.1991974115371704,
      "rewards/rejected": -2.9919729232788086,
      "step": 13560
    },
    {
      "epoch": 1.7745481311850821,
      "grad_norm": 5.337988376617432,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3309185206890106,
      "logits/rejected": -0.2830691933631897,
      "logps/chosen": -517.5659790039062,
      "logps/rejected": -451.2912292480469,
      "loss": 0.2343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8513097763061523,
      "rewards/margins": 1.9179073572158813,
      "rewards/rejected": -2.769217014312744,
      "step": 13561
    },
    {
      "epoch": 1.774678989122434,
      "grad_norm": 17.317169189453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5811598896980286,
      "logits/rejected": -0.48106417059898376,
      "logps/chosen": -413.5039978027344,
      "logps/rejected": -323.82958984375,
      "loss": 1.0402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.705918550491333,
      "rewards/margins": 0.4578760266304016,
      "rewards/rejected": -3.16379451751709,
      "step": 13562
    },
    {
      "epoch": 1.7748098470597857,
      "grad_norm": 5.6229352951049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49140334129333496,
      "logits/rejected": -0.5242243409156799,
      "logps/chosen": -329.87701416015625,
      "logps/rejected": -660.290283203125,
      "loss": 0.3304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5350040793418884,
      "rewards/margins": 1.542768955230713,
      "rewards/rejected": -2.077772855758667,
      "step": 13563
    },
    {
      "epoch": 1.7749407049971375,
      "grad_norm": 8.451387405395508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5444254279136658,
      "logits/rejected": -0.26185375452041626,
      "logps/chosen": -284.0257263183594,
      "logps/rejected": -359.6646728515625,
      "loss": 0.5115,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1831564903259277,
      "rewards/margins": 0.8065086603164673,
      "rewards/rejected": -2.9896652698516846,
      "step": 13564
    },
    {
      "epoch": 1.7750715629344893,
      "grad_norm": 5.960862636566162,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3023301064968109,
      "logits/rejected": -0.27971363067626953,
      "logps/chosen": -354.1270446777344,
      "logps/rejected": -315.68133544921875,
      "loss": 0.4623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0283570289611816,
      "rewards/margins": 1.3078492879867554,
      "rewards/rejected": -2.3362061977386475,
      "step": 13565
    },
    {
      "epoch": 1.775202420871841,
      "grad_norm": 10.228813171386719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45338356494903564,
      "logits/rejected": -0.6249449253082275,
      "logps/chosen": -401.71234130859375,
      "logps/rejected": -359.5329284667969,
      "loss": 0.4845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.491176128387451,
      "rewards/margins": 0.9932479858398438,
      "rewards/rejected": -3.484424114227295,
      "step": 13566
    },
    {
      "epoch": 1.775333278809193,
      "grad_norm": 7.48959493637085,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11075768619775772,
      "logits/rejected": -0.25855305790901184,
      "logps/chosen": -411.9792785644531,
      "logps/rejected": -442.02325439453125,
      "loss": 0.3609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4990674257278442,
      "rewards/margins": 0.9910063147544861,
      "rewards/rejected": -2.4900739192962646,
      "step": 13567
    },
    {
      "epoch": 1.7754641367465447,
      "grad_norm": 10.778260231018066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41061532497406006,
      "logits/rejected": -0.41187700629234314,
      "logps/chosen": -361.0301818847656,
      "logps/rejected": -448.0895080566406,
      "loss": 0.47,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.436299443244934,
      "rewards/margins": 1.040399432182312,
      "rewards/rejected": -2.476698875427246,
      "step": 13568
    },
    {
      "epoch": 1.7755949946838963,
      "grad_norm": 9.516308784484863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05832008644938469,
      "logits/rejected": -0.03769683465361595,
      "logps/chosen": -473.1611022949219,
      "logps/rejected": -500.2802734375,
      "loss": 0.4395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9065709710121155,
      "rewards/margins": 1.4040412902832031,
      "rewards/rejected": -2.310612201690674,
      "step": 13569
    },
    {
      "epoch": 1.775725852621248,
      "grad_norm": 7.033353805541992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4850579798221588,
      "logits/rejected": -0.44632017612457275,
      "logps/chosen": -412.1622314453125,
      "logps/rejected": -349.08575439453125,
      "loss": 0.3993,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1483981609344482,
      "rewards/margins": 1.0113873481750488,
      "rewards/rejected": -2.159785509109497,
      "step": 13570
    },
    {
      "epoch": 1.7758567105585998,
      "grad_norm": 7.398598670959473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3109813928604126,
      "logits/rejected": -0.10726270079612732,
      "logps/chosen": -420.5836181640625,
      "logps/rejected": -394.1304626464844,
      "loss": 0.3486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45893043279647827,
      "rewards/margins": 1.3272548913955688,
      "rewards/rejected": -1.7861852645874023,
      "step": 13571
    },
    {
      "epoch": 1.7759875684959516,
      "grad_norm": 12.674440383911133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13023006916046143,
      "logits/rejected": -0.124329574406147,
      "logps/chosen": -423.2779541015625,
      "logps/rejected": -432.90838623046875,
      "loss": 0.6317,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1415224075317383,
      "rewards/margins": 0.41952818632125854,
      "rewards/rejected": -1.5610506534576416,
      "step": 13572
    },
    {
      "epoch": 1.7761184264333032,
      "grad_norm": 15.523499488830566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4353952407836914,
      "logits/rejected": -0.19294220209121704,
      "logps/chosen": -596.2380981445312,
      "logps/rejected": -463.3158874511719,
      "loss": 0.8566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1631108522415161,
      "rewards/margins": 0.1418757140636444,
      "rewards/rejected": -1.304986596107483,
      "step": 13573
    },
    {
      "epoch": 1.776249284370655,
      "grad_norm": 10.29637622833252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40957745909690857,
      "logits/rejected": -0.6004675626754761,
      "logps/chosen": -448.7530822753906,
      "logps/rejected": -410.3670349121094,
      "loss": 0.4827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.615194320678711,
      "rewards/margins": 1.292062520980835,
      "rewards/rejected": -2.907256841659546,
      "step": 13574
    },
    {
      "epoch": 1.7763801423080068,
      "grad_norm": 9.453049659729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12164559960365295,
      "logits/rejected": -0.201851949095726,
      "logps/chosen": -509.04180908203125,
      "logps/rejected": -603.1265869140625,
      "loss": 0.3523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.203517198562622,
      "rewards/margins": 1.6636496782302856,
      "rewards/rejected": -2.8671669960021973,
      "step": 13575
    },
    {
      "epoch": 1.7765110002453586,
      "grad_norm": 13.896904945373535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15323355793952942,
      "logits/rejected": -0.36651062965393066,
      "logps/chosen": -449.7690124511719,
      "logps/rejected": -429.41046142578125,
      "loss": 0.6071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1992483139038086,
      "rewards/margins": 1.105785608291626,
      "rewards/rejected": -3.3050339221954346,
      "step": 13576
    },
    {
      "epoch": 1.7766418581827104,
      "grad_norm": 8.577990531921387,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02812250889837742,
      "logits/rejected": 0.037764567881822586,
      "logps/chosen": -392.3692321777344,
      "logps/rejected": -464.41900634765625,
      "loss": 0.5002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.521031379699707,
      "rewards/margins": 1.253413438796997,
      "rewards/rejected": -1.7744446992874146,
      "step": 13577
    },
    {
      "epoch": 1.7767727161200622,
      "grad_norm": 8.651976585388184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28462448716163635,
      "logits/rejected": -0.33907350897789,
      "logps/chosen": -493.6249084472656,
      "logps/rejected": -495.2549133300781,
      "loss": 0.2978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9409023523330688,
      "rewards/margins": 2.033747434616089,
      "rewards/rejected": -2.974649429321289,
      "step": 13578
    },
    {
      "epoch": 1.776903574057414,
      "grad_norm": 11.247126579284668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30867087841033936,
      "logits/rejected": -0.34137097001075745,
      "logps/chosen": -438.83245849609375,
      "logps/rejected": -447.70037841796875,
      "loss": 0.7017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6717333793640137,
      "rewards/margins": 0.13660414516925812,
      "rewards/rejected": -1.8083374500274658,
      "step": 13579
    },
    {
      "epoch": 1.7770344319947657,
      "grad_norm": 4.896518230438232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20005209743976593,
      "logits/rejected": -0.18301160633563995,
      "logps/chosen": -544.9481201171875,
      "logps/rejected": -479.8894958496094,
      "loss": 0.2434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.784812331199646,
      "rewards/margins": 1.598257303237915,
      "rewards/rejected": -2.3830699920654297,
      "step": 13580
    },
    {
      "epoch": 1.7771652899321175,
      "grad_norm": 8.321717262268066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17685849964618683,
      "logits/rejected": -0.13647393882274628,
      "logps/chosen": -509.351318359375,
      "logps/rejected": -504.61370849609375,
      "loss": 0.4195,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6891288757324219,
      "rewards/margins": 1.8673818111419678,
      "rewards/rejected": -2.5565106868743896,
      "step": 13581
    },
    {
      "epoch": 1.7772961478694693,
      "grad_norm": 8.187376976013184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14621537923812866,
      "logits/rejected": -0.011986598372459412,
      "logps/chosen": -488.82073974609375,
      "logps/rejected": -510.9692687988281,
      "loss": 0.4295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8661052584648132,
      "rewards/margins": 1.1951873302459717,
      "rewards/rejected": -2.0612926483154297,
      "step": 13582
    },
    {
      "epoch": 1.7774270058068211,
      "grad_norm": 10.511054039001465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1679321825504303,
      "logits/rejected": -0.15082213282585144,
      "logps/chosen": -494.1469421386719,
      "logps/rejected": -542.067138671875,
      "loss": 0.4254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.161376714706421,
      "rewards/margins": 1.1501517295837402,
      "rewards/rejected": -2.311528444290161,
      "step": 13583
    },
    {
      "epoch": 1.7775578637441727,
      "grad_norm": 4.713611125946045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41625353693962097,
      "logits/rejected": -0.43892624974250793,
      "logps/chosen": -526.5604248046875,
      "logps/rejected": -468.695068359375,
      "loss": 0.173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5739746689796448,
      "rewards/margins": 2.0112221240997314,
      "rewards/rejected": -2.5851967334747314,
      "step": 13584
    },
    {
      "epoch": 1.7776887216815245,
      "grad_norm": 8.287519454956055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3221356272697449,
      "logits/rejected": -0.16235420107841492,
      "logps/chosen": -302.67474365234375,
      "logps/rejected": -434.7756042480469,
      "loss": 0.4062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4832490682601929,
      "rewards/margins": 1.281619668006897,
      "rewards/rejected": -2.764868974685669,
      "step": 13585
    },
    {
      "epoch": 1.7778195796188763,
      "grad_norm": 4.382716178894043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.326366126537323,
      "logits/rejected": -0.11174590140581131,
      "logps/chosen": -449.82440185546875,
      "logps/rejected": -492.1936950683594,
      "loss": 0.1887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4430614113807678,
      "rewards/margins": 1.9307913780212402,
      "rewards/rejected": -2.3738527297973633,
      "step": 13586
    },
    {
      "epoch": 1.777950437556228,
      "grad_norm": 6.185682773590088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.179986834526062,
      "logits/rejected": -0.14878502488136292,
      "logps/chosen": -460.5054626464844,
      "logps/rejected": -557.0071411132812,
      "loss": 0.3818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7673537135124207,
      "rewards/margins": 1.3941136598587036,
      "rewards/rejected": -2.1614675521850586,
      "step": 13587
    },
    {
      "epoch": 1.7780812954935796,
      "grad_norm": 15.044999122619629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5231269001960754,
      "logits/rejected": -0.31224220991134644,
      "logps/chosen": -638.887451171875,
      "logps/rejected": -473.79864501953125,
      "loss": 0.6982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.597545623779297,
      "rewards/margins": 0.6965879797935486,
      "rewards/rejected": -3.2941336631774902,
      "step": 13588
    },
    {
      "epoch": 1.7782121534309314,
      "grad_norm": 8.434612274169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23605209589004517,
      "logits/rejected": -0.2267129123210907,
      "logps/chosen": -356.9000244140625,
      "logps/rejected": -417.049560546875,
      "loss": 0.4697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7191358804702759,
      "rewards/margins": 0.984661877155304,
      "rewards/rejected": -1.7037978172302246,
      "step": 13589
    },
    {
      "epoch": 1.7783430113682832,
      "grad_norm": 9.9403715133667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32763469219207764,
      "logits/rejected": -0.36575019359588623,
      "logps/chosen": -504.2134704589844,
      "logps/rejected": -552.7217407226562,
      "loss": 0.4389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4483194947242737,
      "rewards/margins": 2.196547746658325,
      "rewards/rejected": -2.644866943359375,
      "step": 13590
    },
    {
      "epoch": 1.778473869305635,
      "grad_norm": 10.89919662475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24302953481674194,
      "logits/rejected": -0.2618173360824585,
      "logps/chosen": -444.7684020996094,
      "logps/rejected": -551.43701171875,
      "loss": 0.555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3020589351654053,
      "rewards/margins": 1.3114349842071533,
      "rewards/rejected": -2.6134939193725586,
      "step": 13591
    },
    {
      "epoch": 1.7786047272429868,
      "grad_norm": 10.344796180725098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1840747892856598,
      "logits/rejected": -0.190353661775589,
      "logps/chosen": -385.14453125,
      "logps/rejected": -382.2159423828125,
      "loss": 0.6361,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7709247469902039,
      "rewards/margins": 1.0888326168060303,
      "rewards/rejected": -1.859757423400879,
      "step": 13592
    },
    {
      "epoch": 1.7787355851803386,
      "grad_norm": 6.245495319366455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5205968022346497,
      "logits/rejected": -0.6250739693641663,
      "logps/chosen": -406.5244140625,
      "logps/rejected": -432.6429748535156,
      "loss": 0.2867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2763340473175049,
      "rewards/margins": 1.7675933837890625,
      "rewards/rejected": -3.0439274311065674,
      "step": 13593
    },
    {
      "epoch": 1.7788664431176904,
      "grad_norm": 16.2910213470459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01514233648777008,
      "logits/rejected": -0.2724717855453491,
      "logps/chosen": -592.6698608398438,
      "logps/rejected": -432.2267761230469,
      "loss": 0.7683,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4473881721496582,
      "rewards/margins": 0.1726575791835785,
      "rewards/rejected": -1.6200456619262695,
      "step": 13594
    },
    {
      "epoch": 1.7789973010550422,
      "grad_norm": 8.389080047607422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3502946197986603,
      "logits/rejected": -0.49118900299072266,
      "logps/chosen": -451.0626220703125,
      "logps/rejected": -434.7837829589844,
      "loss": 0.4785,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5956556797027588,
      "rewards/margins": 0.8240450024604797,
      "rewards/rejected": -1.4197007417678833,
      "step": 13595
    },
    {
      "epoch": 1.779128158992394,
      "grad_norm": 16.681209564208984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23346811532974243,
      "logits/rejected": -0.3365088403224945,
      "logps/chosen": -531.2677001953125,
      "logps/rejected": -329.40863037109375,
      "loss": 0.6073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.57708740234375,
      "rewards/margins": 0.8974753022193909,
      "rewards/rejected": -2.474562644958496,
      "step": 13596
    },
    {
      "epoch": 1.7792590169297458,
      "grad_norm": 10.79969310760498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2550998032093048,
      "logits/rejected": -0.3750823140144348,
      "logps/chosen": -439.52825927734375,
      "logps/rejected": -370.6775207519531,
      "loss": 0.4749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9309909343719482,
      "rewards/margins": 1.6367840766906738,
      "rewards/rejected": -2.567774772644043,
      "step": 13597
    },
    {
      "epoch": 1.7793898748670975,
      "grad_norm": 9.226439476013184,
      "learning_rate": 5e-06,
      "logits/chosen": 0.003716807346791029,
      "logits/rejected": -0.11782635003328323,
      "logps/chosen": -489.8419189453125,
      "logps/rejected": -533.6941528320312,
      "loss": 0.4487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7974395751953125,
      "rewards/margins": 1.1473382711410522,
      "rewards/rejected": -2.9447779655456543,
      "step": 13598
    },
    {
      "epoch": 1.7795207328044493,
      "grad_norm": 5.973768711090088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19618213176727295,
      "logits/rejected": -0.3940838575363159,
      "logps/chosen": -679.931640625,
      "logps/rejected": -485.59613037109375,
      "loss": 0.2836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6787874698638916,
      "rewards/margins": 1.216867446899414,
      "rewards/rejected": -1.8956549167633057,
      "step": 13599
    },
    {
      "epoch": 1.779651590741801,
      "grad_norm": 9.935033798217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.312248170375824,
      "logits/rejected": -0.48809874057769775,
      "logps/chosen": -451.08160400390625,
      "logps/rejected": -417.632080078125,
      "loss": 0.359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9858724474906921,
      "rewards/margins": 1.5691146850585938,
      "rewards/rejected": -2.5549874305725098,
      "step": 13600
    },
    {
      "epoch": 1.7797824486791527,
      "grad_norm": 8.525532722473145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.23367591202259064,
      "logits/rejected": 0.16134169697761536,
      "logps/chosen": -783.082763671875,
      "logps/rejected": -791.7346801757812,
      "loss": 0.3499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.25075453519821167,
      "rewards/margins": 1.453152060508728,
      "rewards/rejected": -1.703906774520874,
      "step": 13601
    },
    {
      "epoch": 1.7799133066165045,
      "grad_norm": 8.988438606262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6454236507415771,
      "logits/rejected": -0.6593753099441528,
      "logps/chosen": -315.5435791015625,
      "logps/rejected": -449.224365234375,
      "loss": 0.4719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.985980749130249,
      "rewards/margins": 1.428922176361084,
      "rewards/rejected": -3.414902925491333,
      "step": 13602
    },
    {
      "epoch": 1.780044164553856,
      "grad_norm": 10.65799331665039,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08014121651649475,
      "logits/rejected": 0.15748000144958496,
      "logps/chosen": -424.49896240234375,
      "logps/rejected": -416.0945129394531,
      "loss": 0.7859,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9577425718307495,
      "rewards/margins": 0.06216489523649216,
      "rewards/rejected": -1.0199074745178223,
      "step": 13603
    },
    {
      "epoch": 1.7801750224912078,
      "grad_norm": 6.3230085372924805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20883984863758087,
      "logits/rejected": -0.3308143615722656,
      "logps/chosen": -637.8201293945312,
      "logps/rejected": -350.71649169921875,
      "loss": 0.3203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7747082114219666,
      "rewards/margins": 2.215641975402832,
      "rewards/rejected": -2.9903500080108643,
      "step": 13604
    },
    {
      "epoch": 1.7803058804285596,
      "grad_norm": 11.985920906066895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03181175887584686,
      "logits/rejected": 0.0018142163753509521,
      "logps/chosen": -467.618896484375,
      "logps/rejected": -441.6126708984375,
      "loss": 0.7338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.103184461593628,
      "rewards/margins": 0.574150800704956,
      "rewards/rejected": -1.6773353815078735,
      "step": 13605
    },
    {
      "epoch": 1.7804367383659114,
      "grad_norm": 6.0378336906433105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28169792890548706,
      "logits/rejected": -0.186240091919899,
      "logps/chosen": -442.4003601074219,
      "logps/rejected": -448.89263916015625,
      "loss": 0.3109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01429346576333046,
      "rewards/margins": 1.565038800239563,
      "rewards/rejected": -1.5793323516845703,
      "step": 13606
    },
    {
      "epoch": 1.7805675963032632,
      "grad_norm": 15.804323196411133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35696327686309814,
      "logits/rejected": -0.43169480562210083,
      "logps/chosen": -498.82958984375,
      "logps/rejected": -592.607666015625,
      "loss": 0.878,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4836821556091309,
      "rewards/margins": -0.056403905153274536,
      "rewards/rejected": -1.4272781610488892,
      "step": 13607
    },
    {
      "epoch": 1.780698454240615,
      "grad_norm": 4.011977195739746,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07535000145435333,
      "logits/rejected": 0.017117202281951904,
      "logps/chosen": -657.3452758789062,
      "logps/rejected": -508.4263610839844,
      "loss": 0.1657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1127641201019287,
      "rewards/margins": 2.0847537517547607,
      "rewards/rejected": -3.1975176334381104,
      "step": 13608
    },
    {
      "epoch": 1.7808293121779668,
      "grad_norm": 5.840559959411621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3219510614871979,
      "logits/rejected": -0.5930062532424927,
      "logps/chosen": -487.38671875,
      "logps/rejected": -467.1144714355469,
      "loss": 0.2041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5711082220077515,
      "rewards/margins": 1.8677159547805786,
      "rewards/rejected": -2.43882417678833,
      "step": 13609
    },
    {
      "epoch": 1.7809601701153186,
      "grad_norm": 8.862871170043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45200276374816895,
      "logits/rejected": -0.7526226043701172,
      "logps/chosen": -481.70501708984375,
      "logps/rejected": -291.33245849609375,
      "loss": 0.3943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6024438738822937,
      "rewards/margins": 1.0370758771896362,
      "rewards/rejected": -1.6395196914672852,
      "step": 13610
    },
    {
      "epoch": 1.7810910280526704,
      "grad_norm": 15.266752243041992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047098949551582336,
      "logits/rejected": -0.38719093799591064,
      "logps/chosen": -460.4866638183594,
      "logps/rejected": -393.5595397949219,
      "loss": 0.7437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1981587409973145,
      "rewards/margins": 0.2671215534210205,
      "rewards/rejected": -2.465280532836914,
      "step": 13611
    },
    {
      "epoch": 1.7812218859900222,
      "grad_norm": 7.6068572998046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12455043941736221,
      "logits/rejected": -0.34302857518196106,
      "logps/chosen": -462.85369873046875,
      "logps/rejected": -332.947265625,
      "loss": 0.3367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7391031384468079,
      "rewards/margins": 1.8335760831832886,
      "rewards/rejected": -2.572679042816162,
      "step": 13612
    },
    {
      "epoch": 1.781352743927374,
      "grad_norm": 5.420420169830322,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06640275567770004,
      "logits/rejected": -0.1215580403804779,
      "logps/chosen": -364.57440185546875,
      "logps/rejected": -670.9934692382812,
      "loss": 0.2239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.46149808168411255,
      "rewards/margins": 1.9691317081451416,
      "rewards/rejected": -2.4306297302246094,
      "step": 13613
    },
    {
      "epoch": 1.7814836018647258,
      "grad_norm": 7.604635238647461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18212689459323883,
      "logits/rejected": -0.5869913697242737,
      "logps/chosen": -383.0782775878906,
      "logps/rejected": -514.1202392578125,
      "loss": 0.2528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8651166558265686,
      "rewards/margins": 2.57179856300354,
      "rewards/rejected": -3.436915397644043,
      "step": 13614
    },
    {
      "epoch": 1.7816144598020773,
      "grad_norm": 3.7541770935058594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14666880667209625,
      "logits/rejected": -0.2514655590057373,
      "logps/chosen": -384.880615234375,
      "logps/rejected": -487.3968200683594,
      "loss": 0.157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4970976710319519,
      "rewards/margins": 2.4757189750671387,
      "rewards/rejected": -2.9728164672851562,
      "step": 13615
    },
    {
      "epoch": 1.7817453177394291,
      "grad_norm": 8.323171615600586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.013809707015752792,
      "logits/rejected": 0.01796739175915718,
      "logps/chosen": -419.76043701171875,
      "logps/rejected": -477.2032775878906,
      "loss": 0.4961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1314932107925415,
      "rewards/margins": 1.1475955247879028,
      "rewards/rejected": -2.2790887355804443,
      "step": 13616
    },
    {
      "epoch": 1.781876175676781,
      "grad_norm": 5.399707317352295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4070437252521515,
      "logits/rejected": -0.5559636950492859,
      "logps/chosen": -534.87939453125,
      "logps/rejected": -522.29736328125,
      "loss": 0.2396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41043561697006226,
      "rewards/margins": 1.6002293825149536,
      "rewards/rejected": -2.010664939880371,
      "step": 13617
    },
    {
      "epoch": 1.7820070336141327,
      "grad_norm": 8.762639999389648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6328721046447754,
      "logits/rejected": -0.5338639616966248,
      "logps/chosen": -369.8301086425781,
      "logps/rejected": -355.06854248046875,
      "loss": 0.3729,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8429001569747925,
      "rewards/margins": 1.4572030305862427,
      "rewards/rejected": -2.300102949142456,
      "step": 13618
    },
    {
      "epoch": 1.7821378915514843,
      "grad_norm": 15.157331466674805,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014024458825588226,
      "logits/rejected": -0.3962025046348572,
      "logps/chosen": -578.57177734375,
      "logps/rejected": -501.0801696777344,
      "loss": 0.8541,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.534745216369629,
      "rewards/margins": 0.6718463897705078,
      "rewards/rejected": -2.2065913677215576,
      "step": 13619
    },
    {
      "epoch": 1.782268749488836,
      "grad_norm": 10.117158889770508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2517732083797455,
      "logits/rejected": -0.11888310313224792,
      "logps/chosen": -466.8077697753906,
      "logps/rejected": -410.64404296875,
      "loss": 0.5753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5107675790786743,
      "rewards/margins": 1.2112680673599243,
      "rewards/rejected": -1.7220357656478882,
      "step": 13620
    },
    {
      "epoch": 1.7823996074261879,
      "grad_norm": 8.716191291809082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31292223930358887,
      "logits/rejected": -0.5264009237289429,
      "logps/chosen": -445.6418762207031,
      "logps/rejected": -385.2964172363281,
      "loss": 0.3988,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46477416157722473,
      "rewards/margins": 2.196406364440918,
      "rewards/rejected": -2.6611804962158203,
      "step": 13621
    },
    {
      "epoch": 1.7825304653635397,
      "grad_norm": 7.827609539031982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18559113144874573,
      "logits/rejected": -0.33630189299583435,
      "logps/chosen": -470.90179443359375,
      "logps/rejected": -411.3982238769531,
      "loss": 0.4856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4145204722881317,
      "rewards/margins": 0.711321234703064,
      "rewards/rejected": -1.125841736793518,
      "step": 13622
    },
    {
      "epoch": 1.7826613233008914,
      "grad_norm": 7.461280822753906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07128950208425522,
      "logits/rejected": -0.17182987928390503,
      "logps/chosen": -466.32415771484375,
      "logps/rejected": -466.731689453125,
      "loss": 0.3879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3754426836967468,
      "rewards/margins": 1.3664510250091553,
      "rewards/rejected": -1.7418938875198364,
      "step": 13623
    },
    {
      "epoch": 1.7827921812382432,
      "grad_norm": 7.649009704589844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22459223866462708,
      "logits/rejected": -0.37637776136398315,
      "logps/chosen": -507.4776611328125,
      "logps/rejected": -503.94384765625,
      "loss": 0.3427,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47194942831993103,
      "rewards/margins": 1.4296098947525024,
      "rewards/rejected": -1.9015593528747559,
      "step": 13624
    },
    {
      "epoch": 1.782923039175595,
      "grad_norm": 10.151433944702148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2097865641117096,
      "logits/rejected": -0.1789173185825348,
      "logps/chosen": -732.7576904296875,
      "logps/rejected": -686.833251953125,
      "loss": 0.5134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07593116164207458,
      "rewards/margins": 0.9447570443153381,
      "rewards/rejected": -0.8688258528709412,
      "step": 13625
    },
    {
      "epoch": 1.7830538971129468,
      "grad_norm": 9.965730667114258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.062218714505434036,
      "logits/rejected": -0.12891605496406555,
      "logps/chosen": -546.910400390625,
      "logps/rejected": -463.2860107421875,
      "loss": 0.5801,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2706142067909241,
      "rewards/margins": 0.7435846328735352,
      "rewards/rejected": -1.0141987800598145,
      "step": 13626
    },
    {
      "epoch": 1.7831847550502986,
      "grad_norm": 4.970691204071045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1730797290802002,
      "logits/rejected": -0.22183507680892944,
      "logps/chosen": -313.51470947265625,
      "logps/rejected": -385.1370849609375,
      "loss": 0.1347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3330157995223999,
      "rewards/margins": 2.5724170207977295,
      "rewards/rejected": -2.23940110206604,
      "step": 13627
    },
    {
      "epoch": 1.7833156129876504,
      "grad_norm": 13.203680992126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16684114933013916,
      "logits/rejected": -0.24087122082710266,
      "logps/chosen": -320.4961853027344,
      "logps/rejected": -372.656494140625,
      "loss": 0.8886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9382789134979248,
      "rewards/margins": 0.5418745279312134,
      "rewards/rejected": -1.4801534414291382,
      "step": 13628
    },
    {
      "epoch": 1.7834464709250022,
      "grad_norm": 12.221489906311035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030694272369146347,
      "logits/rejected": -0.4444209039211273,
      "logps/chosen": -574.432861328125,
      "logps/rejected": -403.0080261230469,
      "loss": 0.7318,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8636768460273743,
      "rewards/margins": 0.9764333963394165,
      "rewards/rejected": -1.840110182762146,
      "step": 13629
    },
    {
      "epoch": 1.7835773288623538,
      "grad_norm": 9.144411087036133,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1794421374797821,
      "logits/rejected": -0.1589391827583313,
      "logps/chosen": -682.4142456054688,
      "logps/rejected": -732.52099609375,
      "loss": 0.4266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46824225783348083,
      "rewards/margins": 1.0824310779571533,
      "rewards/rejected": -1.550673246383667,
      "step": 13630
    },
    {
      "epoch": 1.7837081867997056,
      "grad_norm": 8.628153800964355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3970217704772949,
      "logits/rejected": -0.6683751344680786,
      "logps/chosen": -617.956787109375,
      "logps/rejected": -432.0002746582031,
      "loss": 0.3262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0379507541656494,
      "rewards/margins": 2.0128254890441895,
      "rewards/rejected": -3.050776243209839,
      "step": 13631
    },
    {
      "epoch": 1.7838390447370573,
      "grad_norm": 9.51063060760498,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03422772139310837,
      "logits/rejected": -0.21567144989967346,
      "logps/chosen": -473.0702209472656,
      "logps/rejected": -400.24078369140625,
      "loss": 0.5256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5164225101470947,
      "rewards/margins": 0.7523149251937866,
      "rewards/rejected": -2.268737554550171,
      "step": 13632
    },
    {
      "epoch": 1.7839699026744091,
      "grad_norm": 7.299829006195068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5918728113174438,
      "logits/rejected": -0.695676326751709,
      "logps/chosen": -398.05865478515625,
      "logps/rejected": -384.6407165527344,
      "loss": 0.4669,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6555957794189453,
      "rewards/margins": 0.8728035092353821,
      "rewards/rejected": -1.5283992290496826,
      "step": 13633
    },
    {
      "epoch": 1.7841007606117607,
      "grad_norm": 8.554743766784668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2893771529197693,
      "logits/rejected": -0.155329629778862,
      "logps/chosen": -523.677001953125,
      "logps/rejected": -446.67529296875,
      "loss": 0.4403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3790576756000519,
      "rewards/margins": 0.9743592143058777,
      "rewards/rejected": -1.3534170389175415,
      "step": 13634
    },
    {
      "epoch": 1.7842316185491125,
      "grad_norm": 6.842617034912109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14514274895191193,
      "logits/rejected": -0.11975591629743576,
      "logps/chosen": -315.92327880859375,
      "logps/rejected": -292.7767333984375,
      "loss": 0.4093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.517886757850647,
      "rewards/margins": 1.0352540016174316,
      "rewards/rejected": -1.553140640258789,
      "step": 13635
    },
    {
      "epoch": 1.7843624764864643,
      "grad_norm": 5.885508060455322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20478618144989014,
      "logits/rejected": -0.31423720717430115,
      "logps/chosen": -434.41314697265625,
      "logps/rejected": -370.7585144042969,
      "loss": 0.2409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.644052267074585,
      "rewards/margins": 1.7474851608276367,
      "rewards/rejected": -2.391537666320801,
      "step": 13636
    },
    {
      "epoch": 1.784493334423816,
      "grad_norm": 16.8426570892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4564211368560791,
      "logits/rejected": -0.33757561445236206,
      "logps/chosen": -374.4035949707031,
      "logps/rejected": -548.3414916992188,
      "loss": 1.0387,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.285369634628296,
      "rewards/margins": 0.04062610864639282,
      "rewards/rejected": -1.3259958028793335,
      "step": 13637
    },
    {
      "epoch": 1.7846241923611679,
      "grad_norm": 7.252462387084961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4366448223590851,
      "logits/rejected": -0.4664938747882843,
      "logps/chosen": -382.9992980957031,
      "logps/rejected": -505.20654296875,
      "loss": 0.3653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.613501787185669,
      "rewards/margins": 1.6409149169921875,
      "rewards/rejected": -2.2544167041778564,
      "step": 13638
    },
    {
      "epoch": 1.7847550502985197,
      "grad_norm": 5.803460121154785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4106651544570923,
      "logits/rejected": -0.25266629457473755,
      "logps/chosen": -325.0538024902344,
      "logps/rejected": -400.1298522949219,
      "loss": 0.2856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0992313623428345,
      "rewards/margins": 1.8620370626449585,
      "rewards/rejected": -2.961268663406372,
      "step": 13639
    },
    {
      "epoch": 1.7848859082358715,
      "grad_norm": 9.601579666137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37170910835266113,
      "logits/rejected": -0.5615170001983643,
      "logps/chosen": -416.5456848144531,
      "logps/rejected": -406.49151611328125,
      "loss": 0.5335,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4122754335403442,
      "rewards/margins": 1.223249077796936,
      "rewards/rejected": -2.6355245113372803,
      "step": 13640
    },
    {
      "epoch": 1.7850167661732232,
      "grad_norm": 8.965014457702637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022443711757659912,
      "logits/rejected": -0.1885118931531906,
      "logps/chosen": -338.357421875,
      "logps/rejected": -507.327392578125,
      "loss": 0.419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.295717477798462,
      "rewards/margins": 1.434692144393921,
      "rewards/rejected": -2.730409622192383,
      "step": 13641
    },
    {
      "epoch": 1.785147624110575,
      "grad_norm": 10.81509780883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2688121497631073,
      "logits/rejected": -0.26816049218177795,
      "logps/chosen": -595.98291015625,
      "logps/rejected": -584.6448364257812,
      "loss": 0.6025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5348292589187622,
      "rewards/margins": 0.7760348916053772,
      "rewards/rejected": -2.310864210128784,
      "step": 13642
    },
    {
      "epoch": 1.7852784820479268,
      "grad_norm": 10.03625774383545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44397515058517456,
      "logits/rejected": -0.3749932050704956,
      "logps/chosen": -544.630126953125,
      "logps/rejected": -760.78125,
      "loss": 0.5485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7884474992752075,
      "rewards/margins": 1.056782603263855,
      "rewards/rejected": -1.8452301025390625,
      "step": 13643
    },
    {
      "epoch": 1.7854093399852786,
      "grad_norm": 4.638136863708496,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0027744844555854797,
      "logits/rejected": 0.049332961440086365,
      "logps/chosen": -373.7876892089844,
      "logps/rejected": -506.90826416015625,
      "loss": 0.2135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5841957330703735,
      "rewards/margins": 2.422356128692627,
      "rewards/rejected": -3.006551742553711,
      "step": 13644
    },
    {
      "epoch": 1.7855401979226302,
      "grad_norm": 11.554370880126953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23587624728679657,
      "logits/rejected": -0.3500480353832245,
      "logps/chosen": -427.81646728515625,
      "logps/rejected": -473.6024169921875,
      "loss": 0.6166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7189826965332031,
      "rewards/margins": 1.690728783607483,
      "rewards/rejected": -2.4097115993499756,
      "step": 13645
    },
    {
      "epoch": 1.785671055859982,
      "grad_norm": 11.994758605957031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3722783327102661,
      "logits/rejected": -0.5027831792831421,
      "logps/chosen": -432.10247802734375,
      "logps/rejected": -437.80023193359375,
      "loss": 0.5878,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3187150955200195,
      "rewards/margins": 0.7084863781929016,
      "rewards/rejected": -2.0272011756896973,
      "step": 13646
    },
    {
      "epoch": 1.7858019137973338,
      "grad_norm": 10.981328964233398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10073508322238922,
      "logits/rejected": -0.3444735109806061,
      "logps/chosen": -670.5340576171875,
      "logps/rejected": -392.3995056152344,
      "loss": 0.467,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8931012153625488,
      "rewards/margins": 1.660771369934082,
      "rewards/rejected": -2.553872585296631,
      "step": 13647
    },
    {
      "epoch": 1.7859327717346856,
      "grad_norm": 6.536908149719238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05812137573957443,
      "logits/rejected": -0.12182310223579407,
      "logps/chosen": -535.6039428710938,
      "logps/rejected": -614.8450317382812,
      "loss": 0.2738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3601264953613281,
      "rewards/margins": 2.1936116218566895,
      "rewards/rejected": -2.5537381172180176,
      "step": 13648
    },
    {
      "epoch": 1.7860636296720371,
      "grad_norm": 11.202176094055176,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09379579871892929,
      "logits/rejected": -0.34110209345817566,
      "logps/chosen": -518.7852783203125,
      "logps/rejected": -507.0461730957031,
      "loss": 0.4225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8459370136260986,
      "rewards/margins": 1.4503581523895264,
      "rewards/rejected": -2.296295166015625,
      "step": 13649
    },
    {
      "epoch": 1.786194487609389,
      "grad_norm": 14.944347381591797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2388063669204712,
      "logits/rejected": 0.006613731384277344,
      "logps/chosen": -540.5831298828125,
      "logps/rejected": -636.6981811523438,
      "loss": 0.8835,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.635932445526123,
      "rewards/margins": -0.1583062708377838,
      "rewards/rejected": -1.4776263236999512,
      "step": 13650
    },
    {
      "epoch": 1.7863253455467407,
      "grad_norm": 6.256989479064941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3745045065879822,
      "logits/rejected": -0.5586017370223999,
      "logps/chosen": -450.95733642578125,
      "logps/rejected": -323.41107177734375,
      "loss": 0.3061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34679123759269714,
      "rewards/margins": 1.9113627672195435,
      "rewards/rejected": -2.2581539154052734,
      "step": 13651
    },
    {
      "epoch": 1.7864562034840925,
      "grad_norm": 6.504138469696045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32326191663742065,
      "logits/rejected": -0.32848620414733887,
      "logps/chosen": -439.35687255859375,
      "logps/rejected": -376.1023254394531,
      "loss": 0.3003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5864944458007812,
      "rewards/margins": 1.4387125968933105,
      "rewards/rejected": -2.025207042694092,
      "step": 13652
    },
    {
      "epoch": 1.7865870614214443,
      "grad_norm": 11.555098533630371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3685571551322937,
      "logits/rejected": -0.3603600263595581,
      "logps/chosen": -375.1903381347656,
      "logps/rejected": -454.4534912109375,
      "loss": 0.7842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.218219041824341,
      "rewards/margins": 0.8187515735626221,
      "rewards/rejected": -3.036970615386963,
      "step": 13653
    },
    {
      "epoch": 1.786717919358796,
      "grad_norm": 11.667332649230957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2682608962059021,
      "logits/rejected": -0.35582685470581055,
      "logps/chosen": -373.5627746582031,
      "logps/rejected": -393.3738098144531,
      "loss": 0.5869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9010729789733887,
      "rewards/margins": 0.5595500469207764,
      "rewards/rejected": -2.460623025894165,
      "step": 13654
    },
    {
      "epoch": 1.7868487772961479,
      "grad_norm": 6.561811923980713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33433499932289124,
      "logits/rejected": -0.22764573991298676,
      "logps/chosen": -391.25079345703125,
      "logps/rejected": -409.5322570800781,
      "loss": 0.4362,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8920316696166992,
      "rewards/margins": 1.1466253995895386,
      "rewards/rejected": -2.0386569499969482,
      "step": 13655
    },
    {
      "epoch": 1.7869796352334997,
      "grad_norm": 6.948044776916504,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0321844220161438,
      "logits/rejected": -0.45471397042274475,
      "logps/chosen": -563.80078125,
      "logps/rejected": -492.24822998046875,
      "loss": 0.2671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.135331630706787,
      "rewards/margins": 1.9475690126419067,
      "rewards/rejected": -3.0829010009765625,
      "step": 13656
    },
    {
      "epoch": 1.7871104931708515,
      "grad_norm": 3.883345127105713,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33903539180755615,
      "logits/rejected": -0.3111421763896942,
      "logps/chosen": -473.72052001953125,
      "logps/rejected": -428.79693603515625,
      "loss": 0.2412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5899589657783508,
      "rewards/margins": 2.288119077682495,
      "rewards/rejected": -2.878077983856201,
      "step": 13657
    },
    {
      "epoch": 1.7872413511082033,
      "grad_norm": 7.050009250640869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060228101909160614,
      "logits/rejected": -0.3155937194824219,
      "logps/chosen": -387.5001525878906,
      "logps/rejected": -301.9072570800781,
      "loss": 0.4164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9884655475616455,
      "rewards/margins": 1.7098605632781982,
      "rewards/rejected": -2.698326349258423,
      "step": 13658
    },
    {
      "epoch": 1.787372209045555,
      "grad_norm": 12.200803756713867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2879392206668854,
      "logits/rejected": -0.40924087166786194,
      "logps/chosen": -571.1988525390625,
      "logps/rejected": -468.913818359375,
      "loss": 0.6983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6064876317977905,
      "rewards/margins": 0.16626495122909546,
      "rewards/rejected": -1.7727526426315308,
      "step": 13659
    },
    {
      "epoch": 1.7875030669829068,
      "grad_norm": 12.394891738891602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.302861750125885,
      "logits/rejected": -0.23391437530517578,
      "logps/chosen": -488.73687744140625,
      "logps/rejected": -581.4007568359375,
      "loss": 0.6983,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9288589358329773,
      "rewards/margins": 0.46086451411247253,
      "rewards/rejected": -1.3897234201431274,
      "step": 13660
    },
    {
      "epoch": 1.7876339249202584,
      "grad_norm": 9.950249671936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46370142698287964,
      "logits/rejected": -0.3745344579219818,
      "logps/chosen": -310.4747314453125,
      "logps/rejected": -320.13653564453125,
      "loss": 0.514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0713863372802734,
      "rewards/margins": 0.7528441548347473,
      "rewards/rejected": -2.824230432510376,
      "step": 13661
    },
    {
      "epoch": 1.7877647828576102,
      "grad_norm": 10.43368148803711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24425312876701355,
      "logits/rejected": -0.490157812833786,
      "logps/chosen": -509.5890808105469,
      "logps/rejected": -413.589111328125,
      "loss": 0.7115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8965831995010376,
      "rewards/margins": 0.3596622347831726,
      "rewards/rejected": -1.256245493888855,
      "step": 13662
    },
    {
      "epoch": 1.787895640794962,
      "grad_norm": 7.789229393005371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4558962285518646,
      "logits/rejected": -0.47685664892196655,
      "logps/chosen": -545.2171020507812,
      "logps/rejected": -483.2978515625,
      "loss": 0.3157,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0709646940231323,
      "rewards/margins": 1.5804709196090698,
      "rewards/rejected": -2.651435613632202,
      "step": 13663
    },
    {
      "epoch": 1.7880264987323138,
      "grad_norm": 8.584678649902344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3438865840435028,
      "logits/rejected": -0.26950642466545105,
      "logps/chosen": -494.54400634765625,
      "logps/rejected": -436.32525634765625,
      "loss": 0.4267,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9341081976890564,
      "rewards/margins": 0.8416395783424377,
      "rewards/rejected": -1.7757478952407837,
      "step": 13664
    },
    {
      "epoch": 1.7881573566696654,
      "grad_norm": 8.93699836730957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2580817639827728,
      "logits/rejected": -0.4000629782676697,
      "logps/chosen": -327.1080627441406,
      "logps/rejected": -383.68243408203125,
      "loss": 0.5383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2301630973815918,
      "rewards/margins": 0.6455288529396057,
      "rewards/rejected": -1.8756917715072632,
      "step": 13665
    },
    {
      "epoch": 1.7882882146070171,
      "grad_norm": 5.613373756408691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03197088465094566,
      "logits/rejected": -0.4548702538013458,
      "logps/chosen": -469.8082275390625,
      "logps/rejected": -531.998046875,
      "loss": 0.2318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0564039945602417,
      "rewards/margins": 2.3676774501800537,
      "rewards/rejected": -3.424081325531006,
      "step": 13666
    },
    {
      "epoch": 1.788419072544369,
      "grad_norm": 11.515226364135742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27723538875579834,
      "logits/rejected": -0.2475666105747223,
      "logps/chosen": -525.9442749023438,
      "logps/rejected": -446.3771667480469,
      "loss": 0.481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.098438024520874,
      "rewards/margins": 1.5056147575378418,
      "rewards/rejected": -2.604052782058716,
      "step": 13667
    },
    {
      "epoch": 1.7885499304817207,
      "grad_norm": 9.544414520263672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09315270185470581,
      "logits/rejected": 0.1587599217891693,
      "logps/chosen": -330.636962890625,
      "logps/rejected": -461.7367248535156,
      "loss": 0.3966,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8998399972915649,
      "rewards/margins": 1.472523808479309,
      "rewards/rejected": -2.372363567352295,
      "step": 13668
    },
    {
      "epoch": 1.7886807884190725,
      "grad_norm": 11.274969100952148,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06262008845806122,
      "logits/rejected": 0.015889715403318405,
      "logps/chosen": -430.06121826171875,
      "logps/rejected": -555.7367553710938,
      "loss": 0.5565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.320763349533081,
      "rewards/margins": 0.9654205441474915,
      "rewards/rejected": -2.2861838340759277,
      "step": 13669
    },
    {
      "epoch": 1.7888116463564243,
      "grad_norm": 14.956893920898438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34213924407958984,
      "logits/rejected": -0.3131294846534729,
      "logps/chosen": -486.9814758300781,
      "logps/rejected": -541.5399780273438,
      "loss": 0.5815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4591383934020996,
      "rewards/margins": 0.9339511394500732,
      "rewards/rejected": -2.393089532852173,
      "step": 13670
    },
    {
      "epoch": 1.788942504293776,
      "grad_norm": 4.717459201812744,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3017638027667999,
      "logits/rejected": -0.19996023178100586,
      "logps/chosen": -383.5648498535156,
      "logps/rejected": -392.2120056152344,
      "loss": 0.206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.029581822454929352,
      "rewards/margins": 1.9652976989746094,
      "rewards/rejected": -1.9948796033859253,
      "step": 13671
    },
    {
      "epoch": 1.789073362231128,
      "grad_norm": 2.9169514179229736,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07989632338285446,
      "logits/rejected": -0.44718798995018005,
      "logps/chosen": -559.8002319335938,
      "logps/rejected": -348.9564514160156,
      "loss": 0.1323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6175258755683899,
      "rewards/margins": 2.51967191696167,
      "rewards/rejected": -1.9021461009979248,
      "step": 13672
    },
    {
      "epoch": 1.7892042201684797,
      "grad_norm": 9.550004959106445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2106979489326477,
      "logits/rejected": -0.3637189269065857,
      "logps/chosen": -521.5718383789062,
      "logps/rejected": -557.6790771484375,
      "loss": 0.5076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.46527937054634094,
      "rewards/margins": 0.8154782056808472,
      "rewards/rejected": -1.2807575464248657,
      "step": 13673
    },
    {
      "epoch": 1.7893350781058315,
      "grad_norm": 8.388476371765137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22508186101913452,
      "logits/rejected": -0.2435055673122406,
      "logps/chosen": -499.25048828125,
      "logps/rejected": -518.4241333007812,
      "loss": 0.4379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1191489696502686,
      "rewards/margins": 1.286123275756836,
      "rewards/rejected": -2.4052724838256836,
      "step": 13674
    },
    {
      "epoch": 1.7894659360431833,
      "grad_norm": 10.793037414550781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020607339218258858,
      "logits/rejected": -0.06024133786559105,
      "logps/chosen": -356.9185485839844,
      "logps/rejected": -388.310546875,
      "loss": 0.8034,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3694782257080078,
      "rewards/margins": -0.09159965813159943,
      "rewards/rejected": -1.2778786420822144,
      "step": 13675
    },
    {
      "epoch": 1.7895967939805348,
      "grad_norm": 8.059967994689941,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02589784935116768,
      "logits/rejected": -0.201998770236969,
      "logps/chosen": -643.6865234375,
      "logps/rejected": -561.4824829101562,
      "loss": 0.3476,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3263263702392578,
      "rewards/margins": 2.0499215126037598,
      "rewards/rejected": -1.723595142364502,
      "step": 13676
    },
    {
      "epoch": 1.7897276519178866,
      "grad_norm": 4.772189617156982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29347583651542664,
      "logits/rejected": -0.3640449047088623,
      "logps/chosen": -450.604248046875,
      "logps/rejected": -520.0009155273438,
      "loss": 0.1901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.32995548844337463,
      "rewards/margins": 2.5934417247772217,
      "rewards/rejected": -2.9233973026275635,
      "step": 13677
    },
    {
      "epoch": 1.7898585098552384,
      "grad_norm": 11.15854263305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011075703427195549,
      "logits/rejected": -0.1627860963344574,
      "logps/chosen": -306.5022888183594,
      "logps/rejected": -416.5821838378906,
      "loss": 0.7554,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.60066819190979,
      "rewards/margins": 0.300061970949173,
      "rewards/rejected": -0.9007301926612854,
      "step": 13678
    },
    {
      "epoch": 1.7899893677925902,
      "grad_norm": 7.764555931091309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3150569796562195,
      "logits/rejected": -0.4084326922893524,
      "logps/chosen": -543.2994995117188,
      "logps/rejected": -559.8135986328125,
      "loss": 0.4269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6823707222938538,
      "rewards/margins": 1.4120367765426636,
      "rewards/rejected": -2.094407558441162,
      "step": 13679
    },
    {
      "epoch": 1.7901202257299418,
      "grad_norm": 9.231209754943848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21519555151462555,
      "logits/rejected": -0.1459960639476776,
      "logps/chosen": -543.133056640625,
      "logps/rejected": -418.7568359375,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0419050455093384,
      "rewards/margins": 1.6735848188400269,
      "rewards/rejected": -2.715489625930786,
      "step": 13680
    },
    {
      "epoch": 1.7902510836672936,
      "grad_norm": 7.20966100692749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29918092489242554,
      "logits/rejected": -0.5164064764976501,
      "logps/chosen": -538.5070190429688,
      "logps/rejected": -518.5969848632812,
      "loss": 0.3326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5689318180084229,
      "rewards/margins": 1.668989896774292,
      "rewards/rejected": -3.237921953201294,
      "step": 13681
    },
    {
      "epoch": 1.7903819416046454,
      "grad_norm": 13.15355396270752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10211889445781708,
      "logits/rejected": -0.253449022769928,
      "logps/chosen": -401.22869873046875,
      "logps/rejected": -374.0791015625,
      "loss": 0.6812,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2100317478179932,
      "rewards/margins": 0.44247493147850037,
      "rewards/rejected": -1.652506709098816,
      "step": 13682
    },
    {
      "epoch": 1.7905127995419972,
      "grad_norm": 6.047854423522949,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5343218445777893,
      "logits/rejected": -0.55463707447052,
      "logps/chosen": -329.9652099609375,
      "logps/rejected": -274.6407470703125,
      "loss": 0.3272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.006885290145874,
      "rewards/margins": 1.5615310668945312,
      "rewards/rejected": -2.5684163570404053,
      "step": 13683
    },
    {
      "epoch": 1.790643657479349,
      "grad_norm": 5.535285949707031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3224129378795624,
      "logits/rejected": -0.08817479759454727,
      "logps/chosen": -511.2276611328125,
      "logps/rejected": -511.69537353515625,
      "loss": 0.283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3629549741744995,
      "rewards/margins": 1.899340271949768,
      "rewards/rejected": -2.2622952461242676,
      "step": 13684
    },
    {
      "epoch": 1.7907745154167007,
      "grad_norm": 5.544183731079102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18815262615680695,
      "logits/rejected": -0.5028881430625916,
      "logps/chosen": -384.4404296875,
      "logps/rejected": -391.0809326171875,
      "loss": 0.2051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45551320910453796,
      "rewards/margins": 1.9365463256835938,
      "rewards/rejected": -2.392059564590454,
      "step": 13685
    },
    {
      "epoch": 1.7909053733540525,
      "grad_norm": 6.712215423583984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1110776886343956,
      "logits/rejected": -0.34172573685646057,
      "logps/chosen": -489.61566162109375,
      "logps/rejected": -411.9025573730469,
      "loss": 0.2582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0305724143981934,
      "rewards/margins": 1.5910451412200928,
      "rewards/rejected": -2.621617317199707,
      "step": 13686
    },
    {
      "epoch": 1.7910362312914043,
      "grad_norm": 10.825140953063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12876340746879578,
      "logits/rejected": -0.14520114660263062,
      "logps/chosen": -390.736083984375,
      "logps/rejected": -473.9338073730469,
      "loss": 0.4652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0398200750350952,
      "rewards/margins": 1.0667304992675781,
      "rewards/rejected": -2.106550693511963,
      "step": 13687
    },
    {
      "epoch": 1.7911670892287561,
      "grad_norm": 9.989370346069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2089131623506546,
      "logits/rejected": -0.31674471497535706,
      "logps/chosen": -404.1243896484375,
      "logps/rejected": -430.19964599609375,
      "loss": 0.5091,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8921457529067993,
      "rewards/margins": 1.4321163892745972,
      "rewards/rejected": -2.3242623805999756,
      "step": 13688
    },
    {
      "epoch": 1.791297947166108,
      "grad_norm": 12.467245101928711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3560003936290741,
      "logits/rejected": -0.2922915518283844,
      "logps/chosen": -384.6359558105469,
      "logps/rejected": -382.28656005859375,
      "loss": 1.069,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.9737967252731323,
      "rewards/margins": 0.22369563579559326,
      "rewards/rejected": -2.1974925994873047,
      "step": 13689
    },
    {
      "epoch": 1.7914288051034597,
      "grad_norm": 10.411555290222168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3009563386440277,
      "logits/rejected": -0.19804228842258453,
      "logps/chosen": -380.57098388671875,
      "logps/rejected": -350.816650390625,
      "loss": 0.4313,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6857702732086182,
      "rewards/margins": 1.2340978384017944,
      "rewards/rejected": -2.919868230819702,
      "step": 13690
    },
    {
      "epoch": 1.7915596630408113,
      "grad_norm": 5.627569198608398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25053471326828003,
      "logits/rejected": -0.20030713081359863,
      "logps/chosen": -560.2863159179688,
      "logps/rejected": -600.4725952148438,
      "loss": 0.1593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0712226629257202,
      "rewards/margins": 1.9288747310638428,
      "rewards/rejected": -3.0000972747802734,
      "step": 13691
    },
    {
      "epoch": 1.791690520978163,
      "grad_norm": 3.570775270462036,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5164196491241455,
      "logits/rejected": -0.4515756368637085,
      "logps/chosen": -620.7216186523438,
      "logps/rejected": -375.6885986328125,
      "loss": 0.14,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1184656098484993,
      "rewards/margins": 2.1891441345214844,
      "rewards/rejected": -2.307609796524048,
      "step": 13692
    },
    {
      "epoch": 1.7918213789155149,
      "grad_norm": 19.23781394958496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7799726724624634,
      "logits/rejected": -0.8392696380615234,
      "logps/chosen": -438.3822937011719,
      "logps/rejected": -436.9803161621094,
      "loss": 0.6322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9045541286468506,
      "rewards/margins": 1.8020662069320679,
      "rewards/rejected": -3.706620216369629,
      "step": 13693
    },
    {
      "epoch": 1.7919522368528666,
      "grad_norm": 10.177444458007812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.470888614654541,
      "logits/rejected": -0.5664750337600708,
      "logps/chosen": -349.07293701171875,
      "logps/rejected": -409.9962158203125,
      "loss": 0.575,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6205499172210693,
      "rewards/margins": 0.8921205997467041,
      "rewards/rejected": -2.5126702785491943,
      "step": 13694
    },
    {
      "epoch": 1.7920830947902182,
      "grad_norm": 6.346591472625732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21678021550178528,
      "logits/rejected": -0.2745339870452881,
      "logps/chosen": -511.664306640625,
      "logps/rejected": -762.9260864257812,
      "loss": 0.3105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8722022771835327,
      "rewards/margins": 1.7826684713363647,
      "rewards/rejected": -3.6548707485198975,
      "step": 13695
    },
    {
      "epoch": 1.79221395272757,
      "grad_norm": 14.10438060760498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3422839343547821,
      "logits/rejected": -0.14027296006679535,
      "logps/chosen": -553.4732666015625,
      "logps/rejected": -437.1634521484375,
      "loss": 0.4344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9484478235244751,
      "rewards/margins": 1.1732124090194702,
      "rewards/rejected": -2.1216604709625244,
      "step": 13696
    },
    {
      "epoch": 1.7923448106649218,
      "grad_norm": 4.685356140136719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42494502663612366,
      "logits/rejected": -0.6395487785339355,
      "logps/chosen": -464.3373107910156,
      "logps/rejected": -405.29998779296875,
      "loss": 0.151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5743723511695862,
      "rewards/margins": 2.5324580669403076,
      "rewards/rejected": -3.106830358505249,
      "step": 13697
    },
    {
      "epoch": 1.7924756686022736,
      "grad_norm": 6.835647106170654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03013608232140541,
      "logits/rejected": -0.09770031273365021,
      "logps/chosen": -557.564697265625,
      "logps/rejected": -477.8116149902344,
      "loss": 0.294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.01776503026485443,
      "rewards/margins": 2.11287784576416,
      "rewards/rejected": -2.0951130390167236,
      "step": 13698
    },
    {
      "epoch": 1.7926065265396254,
      "grad_norm": 4.334182262420654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23298776149749756,
      "logits/rejected": -0.32203447818756104,
      "logps/chosen": -385.6134033203125,
      "logps/rejected": -375.2142028808594,
      "loss": 0.1933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24375401437282562,
      "rewards/margins": 2.2140750885009766,
      "rewards/rejected": -2.457829236984253,
      "step": 13699
    },
    {
      "epoch": 1.7927373844769772,
      "grad_norm": 7.300633907318115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15671148896217346,
      "logits/rejected": -0.3770621716976166,
      "logps/chosen": -382.0758972167969,
      "logps/rejected": -347.66650390625,
      "loss": 0.4198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3298313617706299,
      "rewards/margins": 0.8568956255912781,
      "rewards/rejected": -2.1867270469665527,
      "step": 13700
    },
    {
      "epoch": 1.792868242414329,
      "grad_norm": 9.048412322998047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6258262991905212,
      "logits/rejected": -0.49762269854545593,
      "logps/chosen": -361.10888671875,
      "logps/rejected": -381.1034851074219,
      "loss": 0.7026,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.059425711631775,
      "rewards/margins": 0.8036090731620789,
      "rewards/rejected": -1.8630348443984985,
      "step": 13701
    },
    {
      "epoch": 1.7929991003516808,
      "grad_norm": 9.631295204162598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2574153542518616,
      "logits/rejected": -0.22568044066429138,
      "logps/chosen": -346.8311767578125,
      "logps/rejected": -413.26678466796875,
      "loss": 0.5261,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2269467115402222,
      "rewards/margins": 0.6081826686859131,
      "rewards/rejected": -1.8351293802261353,
      "step": 13702
    },
    {
      "epoch": 1.7931299582890325,
      "grad_norm": 8.384997367858887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29270243644714355,
      "logits/rejected": -0.29454365372657776,
      "logps/chosen": -420.4492492675781,
      "logps/rejected": -468.2709655761719,
      "loss": 0.3895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5727571249008179,
      "rewards/margins": 2.203828811645508,
      "rewards/rejected": -2.7765860557556152,
      "step": 13703
    },
    {
      "epoch": 1.7932608162263843,
      "grad_norm": 8.578545570373535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08850833028554916,
      "logits/rejected": -0.2379642128944397,
      "logps/chosen": -438.5138854980469,
      "logps/rejected": -437.517822265625,
      "loss": 0.4144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5423859357833862,
      "rewards/margins": 1.6803196668624878,
      "rewards/rejected": -2.222705602645874,
      "step": 13704
    },
    {
      "epoch": 1.7933916741637361,
      "grad_norm": 7.508193492889404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06001216918230057,
      "logits/rejected": -0.1347649097442627,
      "logps/chosen": -402.402099609375,
      "logps/rejected": -382.514404296875,
      "loss": 0.4755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6044648289680481,
      "rewards/margins": 1.1684584617614746,
      "rewards/rejected": -1.7729233503341675,
      "step": 13705
    },
    {
      "epoch": 1.793522532101088,
      "grad_norm": 8.674198150634766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17175032198429108,
      "logits/rejected": -0.21227316558361053,
      "logps/chosen": -580.81787109375,
      "logps/rejected": -624.7724609375,
      "loss": 0.4181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8203043937683105,
      "rewards/margins": 1.5351468324661255,
      "rewards/rejected": -3.3554511070251465,
      "step": 13706
    },
    {
      "epoch": 1.7936533900384395,
      "grad_norm": 7.496090412139893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5780498385429382,
      "logits/rejected": -0.4561734199523926,
      "logps/chosen": -214.38636779785156,
      "logps/rejected": -393.3638610839844,
      "loss": 0.3387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0160404443740845,
      "rewards/margins": 1.629391074180603,
      "rewards/rejected": -2.6454312801361084,
      "step": 13707
    },
    {
      "epoch": 1.7937842479757913,
      "grad_norm": 12.181723594665527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04110089689493179,
      "logits/rejected": -0.10599295794963837,
      "logps/chosen": -365.5003356933594,
      "logps/rejected": -373.2595520019531,
      "loss": 0.5333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9492725133895874,
      "rewards/margins": 0.7310589551925659,
      "rewards/rejected": -2.680331230163574,
      "step": 13708
    },
    {
      "epoch": 1.793915105913143,
      "grad_norm": 8.734626770019531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08341071009635925,
      "logits/rejected": -0.17532899975776672,
      "logps/chosen": -392.4903869628906,
      "logps/rejected": -421.9611511230469,
      "loss": 0.365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7144266366958618,
      "rewards/margins": 1.9354456663131714,
      "rewards/rejected": -2.649872303009033,
      "step": 13709
    },
    {
      "epoch": 1.7940459638504946,
      "grad_norm": 6.327110767364502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014185599982738495,
      "logits/rejected": -0.24945306777954102,
      "logps/chosen": -641.7552490234375,
      "logps/rejected": -543.1431884765625,
      "loss": 0.3533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3161700963973999,
      "rewards/margins": 1.181593894958496,
      "rewards/rejected": -1.4977641105651855,
      "step": 13710
    },
    {
      "epoch": 1.7941768217878464,
      "grad_norm": 9.459142684936523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.494009405374527,
      "logits/rejected": -0.595733642578125,
      "logps/chosen": -487.3182678222656,
      "logps/rejected": -449.8978576660156,
      "loss": 0.552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.355308175086975,
      "rewards/margins": 0.6190773248672485,
      "rewards/rejected": -1.9743854999542236,
      "step": 13711
    },
    {
      "epoch": 1.7943076797251982,
      "grad_norm": 5.4128193855285645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07887586951255798,
      "logits/rejected": -0.14753051102161407,
      "logps/chosen": -475.46038818359375,
      "logps/rejected": -510.1395263671875,
      "loss": 0.2677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17278406023979187,
      "rewards/margins": 1.4465934038162231,
      "rewards/rejected": -1.619377613067627,
      "step": 13712
    },
    {
      "epoch": 1.79443853766255,
      "grad_norm": 10.724204063415527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08353261649608612,
      "logits/rejected": -0.32134580612182617,
      "logps/chosen": -561.0831298828125,
      "logps/rejected": -312.0438232421875,
      "loss": 0.4676,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.512499451637268,
      "rewards/margins": 0.8318254947662354,
      "rewards/rejected": -2.344324827194214,
      "step": 13713
    },
    {
      "epoch": 1.7945693955999018,
      "grad_norm": 8.269874572753906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28054410219192505,
      "logits/rejected": -0.33755871653556824,
      "logps/chosen": -328.28436279296875,
      "logps/rejected": -344.008544921875,
      "loss": 0.2783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.073563814163208,
      "rewards/margins": 2.0368645191192627,
      "rewards/rejected": -3.11042857170105,
      "step": 13714
    },
    {
      "epoch": 1.7947002535372536,
      "grad_norm": 5.961443901062012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3162410259246826,
      "logits/rejected": -0.5865849852561951,
      "logps/chosen": -457.73724365234375,
      "logps/rejected": -416.14874267578125,
      "loss": 0.3184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5967975854873657,
      "rewards/margins": 1.6464089155197144,
      "rewards/rejected": -2.24320650100708,
      "step": 13715
    },
    {
      "epoch": 1.7948311114746054,
      "grad_norm": 6.554896831512451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5127555131912231,
      "logits/rejected": -0.5064732432365417,
      "logps/chosen": -608.5238647460938,
      "logps/rejected": -512.4701538085938,
      "loss": 0.3586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4649967551231384,
      "rewards/margins": 1.4788609743118286,
      "rewards/rejected": -1.9438577890396118,
      "step": 13716
    },
    {
      "epoch": 1.7949619694119572,
      "grad_norm": 18.072681427001953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29871392250061035,
      "logits/rejected": -0.23025333881378174,
      "logps/chosen": -361.20770263671875,
      "logps/rejected": -339.2479553222656,
      "loss": 0.8221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.558961033821106,
      "rewards/margins": 0.6042984127998352,
      "rewards/rejected": -2.163259267807007,
      "step": 13717
    },
    {
      "epoch": 1.795092827349309,
      "grad_norm": 8.11196517944336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.031626880168914795,
      "logits/rejected": -0.22116146981716156,
      "logps/chosen": -396.9901123046875,
      "logps/rejected": -477.6119079589844,
      "loss": 0.2684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8329184055328369,
      "rewards/margins": 2.417567014694214,
      "rewards/rejected": -3.250485420227051,
      "step": 13718
    },
    {
      "epoch": 1.7952236852866608,
      "grad_norm": 5.324838161468506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31036457419395447,
      "logits/rejected": -0.470589280128479,
      "logps/chosen": -354.6287841796875,
      "logps/rejected": -466.456298828125,
      "loss": 0.2186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6205580234527588,
      "rewards/margins": 2.062626361846924,
      "rewards/rejected": -3.6831843852996826,
      "step": 13719
    },
    {
      "epoch": 1.7953545432240126,
      "grad_norm": 8.97351360321045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25112390518188477,
      "logits/rejected": -0.16206499934196472,
      "logps/chosen": -283.8078918457031,
      "logps/rejected": -296.68865966796875,
      "loss": 0.7462,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4138914346694946,
      "rewards/margins": 0.0945802628993988,
      "rewards/rejected": -1.5084717273712158,
      "step": 13720
    },
    {
      "epoch": 1.7954854011613643,
      "grad_norm": 9.086797714233398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4780401587486267,
      "logits/rejected": -0.3795725405216217,
      "logps/chosen": -436.3895263671875,
      "logps/rejected": -450.36297607421875,
      "loss": 0.4274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9763447046279907,
      "rewards/margins": 1.32562255859375,
      "rewards/rejected": -2.3019673824310303,
      "step": 13721
    },
    {
      "epoch": 1.795616259098716,
      "grad_norm": 9.987937927246094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1353863626718521,
      "logits/rejected": -0.1971043050289154,
      "logps/chosen": -413.4328308105469,
      "logps/rejected": -381.0190124511719,
      "loss": 0.6776,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9685543179512024,
      "rewards/margins": 0.43465083837509155,
      "rewards/rejected": -1.403205156326294,
      "step": 13722
    },
    {
      "epoch": 1.7957471170360677,
      "grad_norm": 6.571103096008301,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24701789021492004,
      "logits/rejected": -0.08570399135351181,
      "logps/chosen": -337.9913330078125,
      "logps/rejected": -485.5804443359375,
      "loss": 0.3029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3728663921356201,
      "rewards/margins": 1.4058727025985718,
      "rewards/rejected": -2.7787392139434814,
      "step": 13723
    },
    {
      "epoch": 1.7958779749734195,
      "grad_norm": 10.810179710388184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013585470616817474,
      "logits/rejected": 0.02470993623137474,
      "logps/chosen": -597.1259765625,
      "logps/rejected": -520.2139892578125,
      "loss": 0.4748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3950741291046143,
      "rewards/margins": 1.0376873016357422,
      "rewards/rejected": -2.4327611923217773,
      "step": 13724
    },
    {
      "epoch": 1.7960088329107713,
      "grad_norm": 9.136009216308594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3586736023426056,
      "logits/rejected": -0.5260027050971985,
      "logps/chosen": -327.0687561035156,
      "logps/rejected": -353.47235107421875,
      "loss": 0.3949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1358065605163574,
      "rewards/margins": 1.8240163326263428,
      "rewards/rejected": -2.9598228931427,
      "step": 13725
    },
    {
      "epoch": 1.7961396908481229,
      "grad_norm": 8.154983520507812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3203325569629669,
      "logits/rejected": -0.46394097805023193,
      "logps/chosen": -486.53472900390625,
      "logps/rejected": -473.3765563964844,
      "loss": 0.4664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.856869101524353,
      "rewards/margins": 0.8569101095199585,
      "rewards/rejected": -2.7137792110443115,
      "step": 13726
    },
    {
      "epoch": 1.7962705487854747,
      "grad_norm": 15.436590194702148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.161195307970047,
      "logits/rejected": -0.2799370288848877,
      "logps/chosen": -460.928955078125,
      "logps/rejected": -401.87908935546875,
      "loss": 0.6458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7911522388458252,
      "rewards/margins": 0.5972482562065125,
      "rewards/rejected": -2.3884005546569824,
      "step": 13727
    },
    {
      "epoch": 1.7964014067228264,
      "grad_norm": 10.441706657409668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2449503242969513,
      "logits/rejected": -0.1299167275428772,
      "logps/chosen": -443.55487060546875,
      "logps/rejected": -373.7903747558594,
      "loss": 0.4822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7941756248474121,
      "rewards/margins": 1.0221667289733887,
      "rewards/rejected": -1.8163423538208008,
      "step": 13728
    },
    {
      "epoch": 1.7965322646601782,
      "grad_norm": 4.213345527648926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1871928572654724,
      "logits/rejected": -0.21000632643699646,
      "logps/chosen": -350.5829772949219,
      "logps/rejected": -309.89788818359375,
      "loss": 0.2386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6430569291114807,
      "rewards/margins": 1.9754095077514648,
      "rewards/rejected": -2.6184661388397217,
      "step": 13729
    },
    {
      "epoch": 1.79666312259753,
      "grad_norm": 5.3469390869140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3270007371902466,
      "logits/rejected": -0.179608553647995,
      "logps/chosen": -459.67327880859375,
      "logps/rejected": -357.3798828125,
      "loss": 0.3174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4130007028579712,
      "rewards/margins": 1.5177643299102783,
      "rewards/rejected": -2.93076491355896,
      "step": 13730
    },
    {
      "epoch": 1.7967939805348818,
      "grad_norm": 11.521702766418457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19078029692173004,
      "logits/rejected": -0.004490941762924194,
      "logps/chosen": -534.8181762695312,
      "logps/rejected": -408.751708984375,
      "loss": 0.72,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4221588373184204,
      "rewards/margins": 0.5279253721237183,
      "rewards/rejected": -1.9500840902328491,
      "step": 13731
    },
    {
      "epoch": 1.7969248384722336,
      "grad_norm": 7.458151817321777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1967972368001938,
      "logits/rejected": -0.08475125581026077,
      "logps/chosen": -411.15045166015625,
      "logps/rejected": -459.42095947265625,
      "loss": 0.3665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7421502470970154,
      "rewards/margins": 0.8888857960700989,
      "rewards/rejected": -1.6310362815856934,
      "step": 13732
    },
    {
      "epoch": 1.7970556964095854,
      "grad_norm": 9.349546432495117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3647008240222931,
      "logits/rejected": -0.33780428767204285,
      "logps/chosen": -387.3974304199219,
      "logps/rejected": -445.21044921875,
      "loss": 0.5149,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3675669431686401,
      "rewards/margins": 1.6083698272705078,
      "rewards/rejected": -2.9759366512298584,
      "step": 13733
    },
    {
      "epoch": 1.7971865543469372,
      "grad_norm": 5.45659875869751,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09887273609638214,
      "logits/rejected": -0.14843736588954926,
      "logps/chosen": -398.03948974609375,
      "logps/rejected": -376.7989196777344,
      "loss": 0.326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9705461859703064,
      "rewards/margins": 1.200336217880249,
      "rewards/rejected": -2.1708824634552,
      "step": 13734
    },
    {
      "epoch": 1.797317412284289,
      "grad_norm": 11.656764030456543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2477799654006958,
      "logits/rejected": -0.2513622045516968,
      "logps/chosen": -525.7058715820312,
      "logps/rejected": -329.0318603515625,
      "loss": 0.4629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1173691749572754,
      "rewards/margins": 1.482440710067749,
      "rewards/rejected": -3.5998096466064453,
      "step": 13735
    },
    {
      "epoch": 1.7974482702216408,
      "grad_norm": 6.7628936767578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38047218322753906,
      "logits/rejected": -0.24691081047058105,
      "logps/chosen": -373.0503234863281,
      "logps/rejected": -526.58447265625,
      "loss": 0.282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5619847774505615,
      "rewards/margins": 1.5156687498092651,
      "rewards/rejected": -3.077653408050537,
      "step": 13736
    },
    {
      "epoch": 1.7975791281589923,
      "grad_norm": 8.03003215789795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4597221612930298,
      "logits/rejected": -0.4411478638648987,
      "logps/chosen": -449.53436279296875,
      "logps/rejected": -470.8795166015625,
      "loss": 0.3443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.800008237361908,
      "rewards/margins": 1.4122306108474731,
      "rewards/rejected": -2.2122387886047363,
      "step": 13737
    },
    {
      "epoch": 1.7977099860963441,
      "grad_norm": 11.8042631149292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4081137776374817,
      "logits/rejected": -0.4474930763244629,
      "logps/chosen": -406.360107421875,
      "logps/rejected": -519.4832763671875,
      "loss": 0.6395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7069635391235352,
      "rewards/margins": 0.6450197696685791,
      "rewards/rejected": -1.3519833087921143,
      "step": 13738
    },
    {
      "epoch": 1.797840844033696,
      "grad_norm": 6.625358581542969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4276342988014221,
      "logits/rejected": -0.3233255445957184,
      "logps/chosen": -313.91961669921875,
      "logps/rejected": -347.5516357421875,
      "loss": 0.423,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0331099033355713,
      "rewards/margins": 1.9623099565505981,
      "rewards/rejected": -2.995419979095459,
      "step": 13739
    },
    {
      "epoch": 1.7979717019710477,
      "grad_norm": 11.059330940246582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.291701078414917,
      "logits/rejected": -0.41963768005371094,
      "logps/chosen": -536.9098510742188,
      "logps/rejected": -421.9913024902344,
      "loss": 0.674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8972437381744385,
      "rewards/margins": 0.422671914100647,
      "rewards/rejected": -2.319915533065796,
      "step": 13740
    },
    {
      "epoch": 1.7981025599083993,
      "grad_norm": 9.249019622802734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05226795747876167,
      "logits/rejected": -0.328033983707428,
      "logps/chosen": -369.06201171875,
      "logps/rejected": -372.5265197753906,
      "loss": 0.4036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1671295166015625,
      "rewards/margins": 1.7597404718399048,
      "rewards/rejected": -2.9268698692321777,
      "step": 13741
    },
    {
      "epoch": 1.798233417845751,
      "grad_norm": 8.618946075439453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31650739908218384,
      "logits/rejected": -0.5111054182052612,
      "logps/chosen": -483.1898498535156,
      "logps/rejected": -482.8434143066406,
      "loss": 0.4035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6597229242324829,
      "rewards/margins": 0.9660506844520569,
      "rewards/rejected": -1.6257736682891846,
      "step": 13742
    },
    {
      "epoch": 1.7983642757831029,
      "grad_norm": 7.28789758682251,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22443228960037231,
      "logits/rejected": -0.30386948585510254,
      "logps/chosen": -314.7822570800781,
      "logps/rejected": -287.74151611328125,
      "loss": 0.4114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8488277196884155,
      "rewards/margins": 0.9436779618263245,
      "rewards/rejected": -1.7925057411193848,
      "step": 13743
    },
    {
      "epoch": 1.7984951337204547,
      "grad_norm": 8.801078796386719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.297914981842041,
      "logits/rejected": -0.3159191310405731,
      "logps/chosen": -450.507568359375,
      "logps/rejected": -538.1156005859375,
      "loss": 0.4412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7006828784942627,
      "rewards/margins": 1.108885407447815,
      "rewards/rejected": -2.809568405151367,
      "step": 13744
    },
    {
      "epoch": 1.7986259916578065,
      "grad_norm": 12.528107643127441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.387236624956131,
      "logits/rejected": -0.37893572449684143,
      "logps/chosen": -306.5596923828125,
      "logps/rejected": -376.86541748046875,
      "loss": 0.4937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5689983367919922,
      "rewards/margins": 1.1888738870620728,
      "rewards/rejected": -2.7578725814819336,
      "step": 13745
    },
    {
      "epoch": 1.7987568495951582,
      "grad_norm": 13.078213691711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5506198406219482,
      "logits/rejected": -0.2493852823972702,
      "logps/chosen": -372.63470458984375,
      "logps/rejected": -529.0677490234375,
      "loss": 0.733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0566962957382202,
      "rewards/margins": 0.9103246927261353,
      "rewards/rejected": -1.9670207500457764,
      "step": 13746
    },
    {
      "epoch": 1.79888770753251,
      "grad_norm": 5.155872821807861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32278913259506226,
      "logits/rejected": -0.2285301387310028,
      "logps/chosen": -319.87579345703125,
      "logps/rejected": -287.65478515625,
      "loss": 0.2704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12694382667541504,
      "rewards/margins": 1.4271693229675293,
      "rewards/rejected": -1.5541131496429443,
      "step": 13747
    },
    {
      "epoch": 1.7990185654698618,
      "grad_norm": 7.35635232925415,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4605647027492523,
      "logits/rejected": -0.5717630386352539,
      "logps/chosen": -476.1020812988281,
      "logps/rejected": -395.84539794921875,
      "loss": 0.3025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3203124403953552,
      "rewards/margins": 2.1305606365203857,
      "rewards/rejected": -2.4508731365203857,
      "step": 13748
    },
    {
      "epoch": 1.7991494234072136,
      "grad_norm": 9.535541534423828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37279486656188965,
      "logits/rejected": -0.38883867859840393,
      "logps/chosen": -517.4296875,
      "logps/rejected": -388.9258117675781,
      "loss": 0.4383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3204618692398071,
      "rewards/margins": 0.822732150554657,
      "rewards/rejected": -2.1431939601898193,
      "step": 13749
    },
    {
      "epoch": 1.7992802813445654,
      "grad_norm": 10.847691535949707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2511883080005646,
      "logits/rejected": -0.4108130931854248,
      "logps/chosen": -539.0018920898438,
      "logps/rejected": -350.3731689453125,
      "loss": 0.3776,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1307220458984375,
      "rewards/margins": 1.3324167728424072,
      "rewards/rejected": -2.4631388187408447,
      "step": 13750
    },
    {
      "epoch": 1.7994111392819172,
      "grad_norm": 4.107723712921143,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2041543871164322,
      "logits/rejected": -0.19877567887306213,
      "logps/chosen": -473.6412048339844,
      "logps/rejected": -368.32720947265625,
      "loss": 0.1733,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8805738687515259,
      "rewards/margins": 2.6462647914886475,
      "rewards/rejected": -3.526838541030884,
      "step": 13751
    },
    {
      "epoch": 1.799541997219269,
      "grad_norm": 14.74074649810791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19336749613285065,
      "logits/rejected": -0.14608250558376312,
      "logps/chosen": -534.3840942382812,
      "logps/rejected": -628.26123046875,
      "loss": 0.7846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0436798334121704,
      "rewards/margins": 0.10264036059379578,
      "rewards/rejected": -1.1463202238082886,
      "step": 13752
    },
    {
      "epoch": 1.7996728551566206,
      "grad_norm": 12.8328275680542,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08809784799814224,
      "logits/rejected": -0.12484946846961975,
      "logps/chosen": -474.7420654296875,
      "logps/rejected": -440.7118225097656,
      "loss": 0.6187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.762203574180603,
      "rewards/margins": 1.2611885070800781,
      "rewards/rejected": -3.02339243888855,
      "step": 13753
    },
    {
      "epoch": 1.7998037130939724,
      "grad_norm": 15.030985832214355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11320716887712479,
      "logits/rejected": -0.5156353712081909,
      "logps/chosen": -388.7871398925781,
      "logps/rejected": -371.3084411621094,
      "loss": 0.4821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1832200288772583,
      "rewards/margins": 1.1806812286376953,
      "rewards/rejected": -2.363901138305664,
      "step": 13754
    },
    {
      "epoch": 1.7999345710313241,
      "grad_norm": 9.395559310913086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3818674683570862,
      "logits/rejected": -0.29666921496391296,
      "logps/chosen": -445.673583984375,
      "logps/rejected": -452.0899658203125,
      "loss": 0.4483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8122942447662354,
      "rewards/margins": 1.3044344186782837,
      "rewards/rejected": -3.1167285442352295,
      "step": 13755
    },
    {
      "epoch": 1.8000654289686757,
      "grad_norm": 17.319793701171875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18690022826194763,
      "logits/rejected": -0.17426177859306335,
      "logps/chosen": -397.7552795410156,
      "logps/rejected": -463.88165283203125,
      "loss": 0.4675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7426704168319702,
      "rewards/margins": 2.349148988723755,
      "rewards/rejected": -3.0918192863464355,
      "step": 13756
    },
    {
      "epoch": 1.8001962869060275,
      "grad_norm": 7.855309009552002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15736791491508484,
      "logits/rejected": -0.36151519417762756,
      "logps/chosen": -395.204345703125,
      "logps/rejected": -337.71026611328125,
      "loss": 0.4178,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0252283811569214,
      "rewards/margins": 1.2933260202407837,
      "rewards/rejected": -2.318554401397705,
      "step": 13757
    },
    {
      "epoch": 1.8003271448433793,
      "grad_norm": 9.850794792175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2886636257171631,
      "logits/rejected": -0.34232771396636963,
      "logps/chosen": -390.9173583984375,
      "logps/rejected": -521.3300170898438,
      "loss": 0.3718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2627524137496948,
      "rewards/margins": 1.0930085182189941,
      "rewards/rejected": -2.3557608127593994,
      "step": 13758
    },
    {
      "epoch": 1.800458002780731,
      "grad_norm": 9.612141609191895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4583158493041992,
      "logits/rejected": -0.4496613144874573,
      "logps/chosen": -533.8816528320312,
      "logps/rejected": -554.1974487304688,
      "loss": 0.5359,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.136237382888794,
      "rewards/margins": 0.5862067937850952,
      "rewards/rejected": -1.7224441766738892,
      "step": 13759
    },
    {
      "epoch": 1.8005888607180829,
      "grad_norm": 10.371448516845703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24831868708133698,
      "logits/rejected": -0.1398429125547409,
      "logps/chosen": -333.79400634765625,
      "logps/rejected": -406.61407470703125,
      "loss": 0.6283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.476030170917511,
      "rewards/margins": 0.7109454870223999,
      "rewards/rejected": -1.1869757175445557,
      "step": 13760
    },
    {
      "epoch": 1.8007197186554347,
      "grad_norm": 7.252198696136475,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2664676308631897,
      "logits/rejected": -0.38488924503326416,
      "logps/chosen": -446.004638671875,
      "logps/rejected": -443.4503173828125,
      "loss": 0.4133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.41800636053085327,
      "rewards/margins": 1.4215565919876099,
      "rewards/rejected": -1.8395628929138184,
      "step": 13761
    },
    {
      "epoch": 1.8008505765927865,
      "grad_norm": 10.649103164672852,
      "learning_rate": 5e-06,
      "logits/chosen": 0.024511054158210754,
      "logits/rejected": -0.49163323640823364,
      "logps/chosen": -615.819091796875,
      "logps/rejected": -447.84368896484375,
      "loss": 0.4365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2518293261528015,
      "rewards/margins": 2.3272364139556885,
      "rewards/rejected": -2.5790657997131348,
      "step": 13762
    },
    {
      "epoch": 1.8009814345301383,
      "grad_norm": 7.884411811828613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27888908982276917,
      "logits/rejected": -0.27321290969848633,
      "logps/chosen": -742.7742919921875,
      "logps/rejected": -538.0697631835938,
      "loss": 0.3209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5148158669471741,
      "rewards/margins": 1.9071412086486816,
      "rewards/rejected": -2.42195725440979,
      "step": 13763
    },
    {
      "epoch": 1.80111229246749,
      "grad_norm": 11.409538269042969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4348030686378479,
      "logits/rejected": -0.2614820599555969,
      "logps/chosen": -564.511962890625,
      "logps/rejected": -509.0152587890625,
      "loss": 0.4063,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7272136807441711,
      "rewards/margins": 1.7792611122131348,
      "rewards/rejected": -2.506474494934082,
      "step": 13764
    },
    {
      "epoch": 1.8012431504048418,
      "grad_norm": 11.760363578796387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36517423391342163,
      "logits/rejected": -0.4046100378036499,
      "logps/chosen": -632.341552734375,
      "logps/rejected": -446.1172180175781,
      "loss": 0.5992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.539549708366394,
      "rewards/margins": 0.7734172344207764,
      "rewards/rejected": -2.312966823577881,
      "step": 13765
    },
    {
      "epoch": 1.8013740083421936,
      "grad_norm": 6.854859828948975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32071056962013245,
      "logits/rejected": -0.5321056246757507,
      "logps/chosen": -355.5468444824219,
      "logps/rejected": -360.9297180175781,
      "loss": 0.3293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8673612475395203,
      "rewards/margins": 1.7283439636230469,
      "rewards/rejected": -2.595705270767212,
      "step": 13766
    },
    {
      "epoch": 1.8015048662795454,
      "grad_norm": 11.176934242248535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34314441680908203,
      "logits/rejected": -0.28229770064353943,
      "logps/chosen": -309.9999694824219,
      "logps/rejected": -344.680908203125,
      "loss": 0.6808,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7814627885818481,
      "rewards/margins": 0.5166146755218506,
      "rewards/rejected": -1.2980775833129883,
      "step": 13767
    },
    {
      "epoch": 1.801635724216897,
      "grad_norm": 8.521123886108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22185522317886353,
      "logits/rejected": -0.20668895542621613,
      "logps/chosen": -531.530029296875,
      "logps/rejected": -602.1404418945312,
      "loss": 0.3824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.934089183807373,
      "rewards/margins": 1.4139641523361206,
      "rewards/rejected": -2.348053216934204,
      "step": 13768
    },
    {
      "epoch": 1.8017665821542488,
      "grad_norm": 7.5409417152404785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3239850401878357,
      "logits/rejected": -0.5392029881477356,
      "logps/chosen": -504.7124328613281,
      "logps/rejected": -464.695556640625,
      "loss": 0.4168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0271656513214111,
      "rewards/margins": 1.3884254693984985,
      "rewards/rejected": -2.415591239929199,
      "step": 13769
    },
    {
      "epoch": 1.8018974400916006,
      "grad_norm": 6.058217525482178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41976600885391235,
      "logits/rejected": -0.349115252494812,
      "logps/chosen": -454.6580810546875,
      "logps/rejected": -381.67608642578125,
      "loss": 0.3159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6179478168487549,
      "rewards/margins": 1.9670902490615845,
      "rewards/rejected": -2.58503794670105,
      "step": 13770
    },
    {
      "epoch": 1.8020282980289524,
      "grad_norm": 8.401531219482422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4465988874435425,
      "logits/rejected": -0.4727494418621063,
      "logps/chosen": -454.82147216796875,
      "logps/rejected": -334.561279296875,
      "loss": 0.4065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0903503894805908,
      "rewards/margins": 1.2229948043823242,
      "rewards/rejected": -2.313345193862915,
      "step": 13771
    },
    {
      "epoch": 1.802159155966304,
      "grad_norm": 4.630316734313965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17562271654605865,
      "logits/rejected": -0.35323116183280945,
      "logps/chosen": -571.772705078125,
      "logps/rejected": -914.806396484375,
      "loss": 0.2128,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36906394362449646,
      "rewards/margins": 2.772052049636841,
      "rewards/rejected": -3.14111590385437,
      "step": 13772
    },
    {
      "epoch": 1.8022900139036557,
      "grad_norm": 9.74998950958252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4189596474170685,
      "logits/rejected": -0.3908240795135498,
      "logps/chosen": -397.0017395019531,
      "logps/rejected": -366.6322021484375,
      "loss": 0.598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0198558568954468,
      "rewards/margins": 1.0159423351287842,
      "rewards/rejected": -2.0357983112335205,
      "step": 13773
    },
    {
      "epoch": 1.8024208718410075,
      "grad_norm": 8.888603210449219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10711975395679474,
      "logits/rejected": -0.46953266859054565,
      "logps/chosen": -487.1994323730469,
      "logps/rejected": -560.1264038085938,
      "loss": 0.4122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9151061177253723,
      "rewards/margins": 2.3772544860839844,
      "rewards/rejected": -3.292360305786133,
      "step": 13774
    },
    {
      "epoch": 1.8025517297783593,
      "grad_norm": 16.486661911010742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4262440502643585,
      "logits/rejected": -0.46781325340270996,
      "logps/chosen": -373.1094665527344,
      "logps/rejected": -392.3212890625,
      "loss": 0.6185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9934511184692383,
      "rewards/margins": 0.6387732028961182,
      "rewards/rejected": -1.6322243213653564,
      "step": 13775
    },
    {
      "epoch": 1.802682587715711,
      "grad_norm": 7.79538631439209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09428616613149643,
      "logits/rejected": -0.032368253916502,
      "logps/chosen": -532.5567626953125,
      "logps/rejected": -629.7041625976562,
      "loss": 0.3488,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7403013706207275,
      "rewards/margins": 1.806578278541565,
      "rewards/rejected": -2.546879529953003,
      "step": 13776
    },
    {
      "epoch": 1.802813445653063,
      "grad_norm": 9.500592231750488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40960681438446045,
      "logits/rejected": -0.42745983600616455,
      "logps/chosen": -281.1512451171875,
      "logps/rejected": -309.5079345703125,
      "loss": 0.7751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4888286590576172,
      "rewards/margins": 1.1270411014556885,
      "rewards/rejected": -2.6158697605133057,
      "step": 13777
    },
    {
      "epoch": 1.8029443035904147,
      "grad_norm": 7.765767574310303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.548356294631958,
      "logits/rejected": -0.386769562959671,
      "logps/chosen": -599.8092041015625,
      "logps/rejected": -578.4758911132812,
      "loss": 0.343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5506008863449097,
      "rewards/margins": 1.5149257183074951,
      "rewards/rejected": -2.0655264854431152,
      "step": 13778
    },
    {
      "epoch": 1.8030751615277665,
      "grad_norm": 6.696471691131592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.270166277885437,
      "logits/rejected": -0.40429845452308655,
      "logps/chosen": -504.58026123046875,
      "logps/rejected": -412.6424865722656,
      "loss": 0.2984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9574342966079712,
      "rewards/margins": 1.3556151390075684,
      "rewards/rejected": -2.31304931640625,
      "step": 13779
    },
    {
      "epoch": 1.8032060194651183,
      "grad_norm": 8.383756637573242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.003015127032995224,
      "logits/rejected": -0.1964254081249237,
      "logps/chosen": -477.6269836425781,
      "logps/rejected": -389.0089111328125,
      "loss": 0.4626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.135994791984558,
      "rewards/margins": 1.2074216604232788,
      "rewards/rejected": -2.343416452407837,
      "step": 13780
    },
    {
      "epoch": 1.80333687740247,
      "grad_norm": 7.092403411865234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2740730345249176,
      "logits/rejected": -0.3216778635978699,
      "logps/chosen": -519.4378662109375,
      "logps/rejected": -487.19317626953125,
      "loss": 0.2982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0966975688934326,
      "rewards/margins": 1.7347835302352905,
      "rewards/rejected": -2.8314809799194336,
      "step": 13781
    },
    {
      "epoch": 1.8034677353398219,
      "grad_norm": 8.245299339294434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.630109429359436,
      "logits/rejected": -0.7224727869033813,
      "logps/chosen": -290.24627685546875,
      "logps/rejected": -306.7969970703125,
      "loss": 0.5509,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6719077825546265,
      "rewards/margins": 0.6073405146598816,
      "rewards/rejected": -2.279247999191284,
      "step": 13782
    },
    {
      "epoch": 1.8035985932771734,
      "grad_norm": 9.147054672241211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42982810735702515,
      "logits/rejected": -0.32989683747291565,
      "logps/chosen": -483.7582092285156,
      "logps/rejected": -519.7471313476562,
      "loss": 0.4765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10946312546730042,
      "rewards/margins": 0.9467847347259521,
      "rewards/rejected": -0.8373216390609741,
      "step": 13783
    },
    {
      "epoch": 1.8037294512145252,
      "grad_norm": 11.573944091796875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31691670417785645,
      "logits/rejected": -0.2765393555164337,
      "logps/chosen": -669.4994506835938,
      "logps/rejected": -543.295654296875,
      "loss": 0.4782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.333233118057251,
      "rewards/margins": 1.242138147354126,
      "rewards/rejected": -2.575371265411377,
      "step": 13784
    },
    {
      "epoch": 1.803860309151877,
      "grad_norm": 5.787972450256348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3717370331287384,
      "logits/rejected": -0.5592997670173645,
      "logps/chosen": -438.0292053222656,
      "logps/rejected": -534.3987426757812,
      "loss": 0.277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5263555645942688,
      "rewards/margins": 2.318533420562744,
      "rewards/rejected": -2.844888925552368,
      "step": 13785
    },
    {
      "epoch": 1.8039911670892288,
      "grad_norm": 16.502151489257812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30636799335479736,
      "logits/rejected": -0.48450565338134766,
      "logps/chosen": -663.7318115234375,
      "logps/rejected": -509.5935363769531,
      "loss": 0.581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8348946571350098,
      "rewards/margins": 1.4398283958435059,
      "rewards/rejected": -3.2747230529785156,
      "step": 13786
    },
    {
      "epoch": 1.8041220250265804,
      "grad_norm": 10.354578971862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2903049886226654,
      "logits/rejected": -0.397864431142807,
      "logps/chosen": -536.8389892578125,
      "logps/rejected": -523.5454711914062,
      "loss": 0.5099,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1550159454345703,
      "rewards/margins": 1.7169954776763916,
      "rewards/rejected": -2.872011661529541,
      "step": 13787
    },
    {
      "epoch": 1.8042528829639322,
      "grad_norm": 4.062870502471924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.474039763212204,
      "logits/rejected": -0.5031497478485107,
      "logps/chosen": -521.8773193359375,
      "logps/rejected": -712.1140747070312,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9628194570541382,
      "rewards/margins": 2.815049648284912,
      "rewards/rejected": -4.777868747711182,
      "step": 13788
    },
    {
      "epoch": 1.804383740901284,
      "grad_norm": 6.2735772132873535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05277208983898163,
      "logits/rejected": -0.10272901505231857,
      "logps/chosen": -301.47576904296875,
      "logps/rejected": -393.9376525878906,
      "loss": 0.2805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9267418384552002,
      "rewards/margins": 1.8972810506820679,
      "rewards/rejected": -2.8240227699279785,
      "step": 13789
    },
    {
      "epoch": 1.8045145988386357,
      "grad_norm": 12.620840072631836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3240075707435608,
      "logits/rejected": -0.4021739065647125,
      "logps/chosen": -605.5137939453125,
      "logps/rejected": -445.5337829589844,
      "loss": 0.6161,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1980888843536377,
      "rewards/margins": 0.324057936668396,
      "rewards/rejected": -2.5221469402313232,
      "step": 13790
    },
    {
      "epoch": 1.8046454567759875,
      "grad_norm": 13.435012817382812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41976308822631836,
      "logits/rejected": -0.42012819647789,
      "logps/chosen": -520.7593994140625,
      "logps/rejected": -371.8012390136719,
      "loss": 0.5639,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6861116886138916,
      "rewards/margins": 0.7662527561187744,
      "rewards/rejected": -2.452364444732666,
      "step": 13791
    },
    {
      "epoch": 1.8047763147133393,
      "grad_norm": 12.928056716918945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13167011737823486,
      "logits/rejected": -0.29122766852378845,
      "logps/chosen": -331.54168701171875,
      "logps/rejected": -325.576904296875,
      "loss": 0.6302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.239989995956421,
      "rewards/margins": 1.2704806327819824,
      "rewards/rejected": -2.5104708671569824,
      "step": 13792
    },
    {
      "epoch": 1.8049071726506911,
      "grad_norm": 6.8666276931762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3994820713996887,
      "logits/rejected": -0.4313274025917053,
      "logps/chosen": -574.2757568359375,
      "logps/rejected": -450.4660949707031,
      "loss": 0.2735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6462873816490173,
      "rewards/margins": 1.7807306051254272,
      "rewards/rejected": -2.427018165588379,
      "step": 13793
    },
    {
      "epoch": 1.805038030588043,
      "grad_norm": 7.545990943908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5104662179946899,
      "logits/rejected": -0.25678110122680664,
      "logps/chosen": -418.1912841796875,
      "logps/rejected": -569.8928833007812,
      "loss": 0.2658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1619163751602173,
      "rewards/margins": 2.3785130977630615,
      "rewards/rejected": -3.5404295921325684,
      "step": 13794
    },
    {
      "epoch": 1.8051688885253947,
      "grad_norm": 8.966082572937012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18284361064434052,
      "logits/rejected": -0.21636855602264404,
      "logps/chosen": -558.4461669921875,
      "logps/rejected": -532.1882934570312,
      "loss": 0.4265,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.658416986465454,
      "rewards/margins": 1.6126487255096436,
      "rewards/rejected": -3.2710654735565186,
      "step": 13795
    },
    {
      "epoch": 1.8052997464627465,
      "grad_norm": 7.402971267700195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06389797478914261,
      "logits/rejected": -0.30055737495422363,
      "logps/chosen": -417.5760498046875,
      "logps/rejected": -335.5531921386719,
      "loss": 0.3695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9731879234313965,
      "rewards/margins": 1.7216463088989258,
      "rewards/rejected": -3.6948342323303223,
      "step": 13796
    },
    {
      "epoch": 1.8054306044000983,
      "grad_norm": 10.062480926513672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09279725700616837,
      "logits/rejected": -0.4016713798046112,
      "logps/chosen": -542.2130126953125,
      "logps/rejected": -344.30059814453125,
      "loss": 0.4401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6910791397094727,
      "rewards/margins": 1.3265119791030884,
      "rewards/rejected": -3.0175909996032715,
      "step": 13797
    },
    {
      "epoch": 1.8055614623374499,
      "grad_norm": 9.214712142944336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5093456506729126,
      "logits/rejected": -0.3949427604675293,
      "logps/chosen": -323.9010314941406,
      "logps/rejected": -470.80859375,
      "loss": 0.3536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.995680034160614,
      "rewards/margins": 2.3138909339904785,
      "rewards/rejected": -3.309570789337158,
      "step": 13798
    },
    {
      "epoch": 1.8056923202748016,
      "grad_norm": 10.503042221069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4016241133213043,
      "logits/rejected": -0.20802795886993408,
      "logps/chosen": -296.4798278808594,
      "logps/rejected": -453.4490661621094,
      "loss": 0.4057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.725522518157959,
      "rewards/margins": 1.0405410528182983,
      "rewards/rejected": -2.7660634517669678,
      "step": 13799
    },
    {
      "epoch": 1.8058231782121534,
      "grad_norm": 6.619511604309082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28602564334869385,
      "logits/rejected": -0.4141302704811096,
      "logps/chosen": -467.0368347167969,
      "logps/rejected": -308.3306884765625,
      "loss": 0.2455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9884877800941467,
      "rewards/margins": 2.104600667953491,
      "rewards/rejected": -3.093088388442993,
      "step": 13800
    },
    {
      "epoch": 1.8059540361495052,
      "grad_norm": 8.158954620361328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15921235084533691,
      "logits/rejected": -0.2888607084751129,
      "logps/chosen": -479.05804443359375,
      "logps/rejected": -466.2507019042969,
      "loss": 0.4311,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1294728517532349,
      "rewards/margins": 1.2319860458374023,
      "rewards/rejected": -2.3614590167999268,
      "step": 13801
    },
    {
      "epoch": 1.8060848940868568,
      "grad_norm": 11.395645141601562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2213451862335205,
      "logits/rejected": -0.21032166481018066,
      "logps/chosen": -575.4369506835938,
      "logps/rejected": -558.8775634765625,
      "loss": 0.5417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9075714349746704,
      "rewards/margins": 0.52471923828125,
      "rewards/rejected": -1.43229079246521,
      "step": 13802
    },
    {
      "epoch": 1.8062157520242086,
      "grad_norm": 9.198028564453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14188240468502045,
      "logits/rejected": -0.2704468071460724,
      "logps/chosen": -596.6060791015625,
      "logps/rejected": -626.8505859375,
      "loss": 0.5514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9136579036712646,
      "rewards/margins": 2.056713342666626,
      "rewards/rejected": -3.9703712463378906,
      "step": 13803
    },
    {
      "epoch": 1.8063466099615604,
      "grad_norm": 14.386506080627441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3154424726963043,
      "logits/rejected": -0.49790212512016296,
      "logps/chosen": -519.7540283203125,
      "logps/rejected": -461.416748046875,
      "loss": 0.9319,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3256577253341675,
      "rewards/margins": 0.05230506509542465,
      "rewards/rejected": -1.3779627084732056,
      "step": 13804
    },
    {
      "epoch": 1.8064774678989122,
      "grad_norm": 10.683944702148438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2534071207046509,
      "logits/rejected": -0.26228514313697815,
      "logps/chosen": -568.5615234375,
      "logps/rejected": -555.8401489257812,
      "loss": 0.57,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9996124505996704,
      "rewards/margins": 0.7829959988594055,
      "rewards/rejected": -1.7826083898544312,
      "step": 13805
    },
    {
      "epoch": 1.806608325836264,
      "grad_norm": 12.097454071044922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2974386513233185,
      "logits/rejected": -0.2654983103275299,
      "logps/chosen": -570.3360595703125,
      "logps/rejected": -547.955810546875,
      "loss": 0.661,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3860613107681274,
      "rewards/margins": 0.6068257093429565,
      "rewards/rejected": -1.9928871393203735,
      "step": 13806
    },
    {
      "epoch": 1.8067391837736158,
      "grad_norm": 9.695013999938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29851850867271423,
      "logits/rejected": -0.36565330624580383,
      "logps/chosen": -487.8001403808594,
      "logps/rejected": -444.8175048828125,
      "loss": 0.536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6455103158950806,
      "rewards/margins": 1.1292333602905273,
      "rewards/rejected": -2.7747435569763184,
      "step": 13807
    },
    {
      "epoch": 1.8068700417109675,
      "grad_norm": 4.929449081420898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38382136821746826,
      "logits/rejected": -0.5969381928443909,
      "logps/chosen": -571.0311279296875,
      "logps/rejected": -662.9244384765625,
      "loss": 0.1931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8586693406105042,
      "rewards/margins": 1.8443570137023926,
      "rewards/rejected": -2.703026533126831,
      "step": 13808
    },
    {
      "epoch": 1.8070008996483193,
      "grad_norm": 14.706992149353027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1063447967171669,
      "logits/rejected": -0.08354431390762329,
      "logps/chosen": -456.6905822753906,
      "logps/rejected": -429.763427734375,
      "loss": 0.7692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1842854022979736,
      "rewards/margins": 0.23430775105953217,
      "rewards/rejected": -1.418593168258667,
      "step": 13809
    },
    {
      "epoch": 1.8071317575856711,
      "grad_norm": 9.951122283935547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18613146245479584,
      "logits/rejected": -0.4938047528266907,
      "logps/chosen": -466.77008056640625,
      "logps/rejected": -543.7830200195312,
      "loss": 0.6269,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0504839420318604,
      "rewards/margins": 1.7025477886199951,
      "rewards/rejected": -2.7530317306518555,
      "step": 13810
    },
    {
      "epoch": 1.807262615523023,
      "grad_norm": 8.106377601623535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2900426387786865,
      "logits/rejected": -0.40892675518989563,
      "logps/chosen": -308.3359680175781,
      "logps/rejected": -474.57073974609375,
      "loss": 0.4309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1514812707901,
      "rewards/margins": 1.4185190200805664,
      "rewards/rejected": -2.570000410079956,
      "step": 13811
    },
    {
      "epoch": 1.8073934734603747,
      "grad_norm": 7.1571173667907715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2463400810956955,
      "logits/rejected": -0.48209741711616516,
      "logps/chosen": -463.283203125,
      "logps/rejected": -719.5826416015625,
      "loss": 0.2422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0378451347351074,
      "rewards/margins": 2.1672935485839844,
      "rewards/rejected": -4.205138683319092,
      "step": 13812
    },
    {
      "epoch": 1.8075243313977265,
      "grad_norm": 6.502277374267578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13353878259658813,
      "logits/rejected": -0.06585031747817993,
      "logps/chosen": -430.52093505859375,
      "logps/rejected": -389.06439208984375,
      "loss": 0.2857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.283076286315918,
      "rewards/margins": 1.8107740879058838,
      "rewards/rejected": -3.0938501358032227,
      "step": 13813
    },
    {
      "epoch": 1.807655189335078,
      "grad_norm": 11.166911125183105,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0810355395078659,
      "logits/rejected": -0.11610321700572968,
      "logps/chosen": -366.28436279296875,
      "logps/rejected": -405.30767822265625,
      "loss": 0.5455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.402728319168091,
      "rewards/margins": 0.6530008316040039,
      "rewards/rejected": -3.0557291507720947,
      "step": 13814
    },
    {
      "epoch": 1.8077860472724299,
      "grad_norm": 10.036331176757812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3060196042060852,
      "logits/rejected": -0.4830382168292999,
      "logps/chosen": -386.3282165527344,
      "logps/rejected": -344.6703796386719,
      "loss": 0.5611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2917689085006714,
      "rewards/margins": 0.7849022150039673,
      "rewards/rejected": -2.0766713619232178,
      "step": 13815
    },
    {
      "epoch": 1.8079169052097817,
      "grad_norm": 11.728687286376953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4092528522014618,
      "logits/rejected": -0.45476749539375305,
      "logps/chosen": -478.4513854980469,
      "logps/rejected": -432.2256164550781,
      "loss": 0.6642,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.672208070755005,
      "rewards/margins": 0.31441593170166016,
      "rewards/rejected": -2.986624002456665,
      "step": 13816
    },
    {
      "epoch": 1.8080477631471334,
      "grad_norm": 14.238640785217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3133700489997864,
      "logits/rejected": -0.4374699890613556,
      "logps/chosen": -670.617919921875,
      "logps/rejected": -566.6521606445312,
      "loss": 0.6825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0166192054748535,
      "rewards/margins": 1.3043571710586548,
      "rewards/rejected": -3.320976734161377,
      "step": 13817
    },
    {
      "epoch": 1.808178621084485,
      "grad_norm": 8.703330039978027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.042463500052690506,
      "logits/rejected": -0.3755251169204712,
      "logps/chosen": -429.51092529296875,
      "logps/rejected": -377.839599609375,
      "loss": 0.39,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6623897552490234,
      "rewards/margins": 1.4521825313568115,
      "rewards/rejected": -3.114572048187256,
      "step": 13818
    },
    {
      "epoch": 1.8083094790218368,
      "grad_norm": 9.757408142089844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48223909735679626,
      "logits/rejected": -0.37945541739463806,
      "logps/chosen": -538.5399169921875,
      "logps/rejected": -481.4541015625,
      "loss": 0.2801,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3485356569290161,
      "rewards/margins": 2.3295209407806396,
      "rewards/rejected": -3.678056478500366,
      "step": 13819
    },
    {
      "epoch": 1.8084403369591886,
      "grad_norm": 8.939233779907227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5072064399719238,
      "logits/rejected": -0.5097722411155701,
      "logps/chosen": -328.73199462890625,
      "logps/rejected": -983.081298828125,
      "loss": 0.4182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8276199102401733,
      "rewards/margins": 2.867323160171509,
      "rewards/rejected": -3.6949429512023926,
      "step": 13820
    },
    {
      "epoch": 1.8085711948965404,
      "grad_norm": 12.076844215393066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5286624431610107,
      "logits/rejected": -0.4211437702178955,
      "logps/chosen": -367.201904296875,
      "logps/rejected": -323.4277648925781,
      "loss": 0.8746,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.784472107887268,
      "rewards/margins": 0.2784026265144348,
      "rewards/rejected": -2.0628745555877686,
      "step": 13821
    },
    {
      "epoch": 1.8087020528338922,
      "grad_norm": 7.354060649871826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1752415895462036,
      "logits/rejected": -0.4230771064758301,
      "logps/chosen": -483.5735778808594,
      "logps/rejected": -485.09332275390625,
      "loss": 0.3328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6995860934257507,
      "rewards/margins": 1.2445284128189087,
      "rewards/rejected": -1.9441144466400146,
      "step": 13822
    },
    {
      "epoch": 1.808832910771244,
      "grad_norm": 10.382245063781738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7828115224838257,
      "logits/rejected": -0.6084858775138855,
      "logps/chosen": -394.1836242675781,
      "logps/rejected": -437.798095703125,
      "loss": 0.6672,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7946033477783203,
      "rewards/margins": 0.6782442331314087,
      "rewards/rejected": -1.472847580909729,
      "step": 13823
    },
    {
      "epoch": 1.8089637687085958,
      "grad_norm": 4.630434036254883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19178682565689087,
      "logits/rejected": -0.40620893239974976,
      "logps/chosen": -505.0762939453125,
      "logps/rejected": -372.03851318359375,
      "loss": 0.1715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5226407051086426,
      "rewards/margins": 1.842585563659668,
      "rewards/rejected": -3.3652262687683105,
      "step": 13824
    },
    {
      "epoch": 1.8090946266459476,
      "grad_norm": 10.970376014709473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.82375568151474,
      "logits/rejected": -0.6898547410964966,
      "logps/chosen": -479.75750732421875,
      "logps/rejected": -489.41412353515625,
      "loss": 0.6091,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0624747276306152,
      "rewards/margins": 0.6815710067749023,
      "rewards/rejected": -2.7440459728240967,
      "step": 13825
    },
    {
      "epoch": 1.8092254845832993,
      "grad_norm": 10.050671577453613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10510295629501343,
      "logits/rejected": -0.32076066732406616,
      "logps/chosen": -671.6199951171875,
      "logps/rejected": -568.0206298828125,
      "loss": 0.3732,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6273944973945618,
      "rewards/margins": 1.5704210996627808,
      "rewards/rejected": -2.197815418243408,
      "step": 13826
    },
    {
      "epoch": 1.8093563425206511,
      "grad_norm": 8.24510669708252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30987444519996643,
      "logits/rejected": -0.3664921224117279,
      "logps/chosen": -340.3825378417969,
      "logps/rejected": -464.6671142578125,
      "loss": 0.2691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8964290618896484,
      "rewards/margins": 2.24191951751709,
      "rewards/rejected": -3.1383485794067383,
      "step": 13827
    },
    {
      "epoch": 1.809487200458003,
      "grad_norm": 7.5253095626831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36843347549438477,
      "logits/rejected": -0.40543344616889954,
      "logps/chosen": -376.5658264160156,
      "logps/rejected": -395.07574462890625,
      "loss": 0.3644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4084616899490356,
      "rewards/margins": 1.2342853546142578,
      "rewards/rejected": -2.642747163772583,
      "step": 13828
    },
    {
      "epoch": 1.8096180583953545,
      "grad_norm": 6.31386661529541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34392186999320984,
      "logits/rejected": -0.27948057651519775,
      "logps/chosen": -310.5290222167969,
      "logps/rejected": -434.28765869140625,
      "loss": 0.2491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.987317681312561,
      "rewards/margins": 2.1742801666259766,
      "rewards/rejected": -3.161597967147827,
      "step": 13829
    },
    {
      "epoch": 1.8097489163327063,
      "grad_norm": 6.937565326690674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3147031366825104,
      "logits/rejected": -0.4363037347793579,
      "logps/chosen": -372.8034973144531,
      "logps/rejected": -405.78466796875,
      "loss": 0.2945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0378836393356323,
      "rewards/margins": 1.5327157974243164,
      "rewards/rejected": -2.570599317550659,
      "step": 13830
    },
    {
      "epoch": 1.809879774270058,
      "grad_norm": 6.3834075927734375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21087074279785156,
      "logits/rejected": -0.4888679087162018,
      "logps/chosen": -308.6211242675781,
      "logps/rejected": -380.2978820800781,
      "loss": 0.3176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3681745529174805,
      "rewards/margins": 1.4980167150497437,
      "rewards/rejected": -2.8661913871765137,
      "step": 13831
    },
    {
      "epoch": 1.8100106322074099,
      "grad_norm": 5.846412658691406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.57319575548172,
      "logits/rejected": -0.3789770305156708,
      "logps/chosen": -440.4731750488281,
      "logps/rejected": -479.180908203125,
      "loss": 0.2462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8268455266952515,
      "rewards/margins": 2.6182494163513184,
      "rewards/rejected": -3.4450953006744385,
      "step": 13832
    },
    {
      "epoch": 1.8101414901447614,
      "grad_norm": 9.399361610412598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2909465730190277,
      "logits/rejected": -0.2614031434059143,
      "logps/chosen": -477.6637268066406,
      "logps/rejected": -394.5677490234375,
      "loss": 0.3965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9734301567077637,
      "rewards/margins": 1.2419860363006592,
      "rewards/rejected": -3.215416431427002,
      "step": 13833
    },
    {
      "epoch": 1.8102723480821132,
      "grad_norm": 9.384536743164062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24081021547317505,
      "logits/rejected": -0.22284924983978271,
      "logps/chosen": -374.1810302734375,
      "logps/rejected": -435.9585876464844,
      "loss": 0.5236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.718353271484375,
      "rewards/margins": 1.452596664428711,
      "rewards/rejected": -3.170949935913086,
      "step": 13834
    },
    {
      "epoch": 1.810403206019465,
      "grad_norm": 5.438581466674805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4671335816383362,
      "logits/rejected": -0.4179738461971283,
      "logps/chosen": -385.414794921875,
      "logps/rejected": -449.0047607421875,
      "loss": 0.2849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6641490459442139,
      "rewards/margins": 1.5819764137268066,
      "rewards/rejected": -3.2461256980895996,
      "step": 13835
    },
    {
      "epoch": 1.8105340639568168,
      "grad_norm": 14.028580665588379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33361226320266724,
      "logits/rejected": -0.23335881531238556,
      "logps/chosen": -488.7514953613281,
      "logps/rejected": -538.521728515625,
      "loss": 0.7762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5850248336791992,
      "rewards/margins": 0.5131644606590271,
      "rewards/rejected": -2.098189353942871,
      "step": 13836
    },
    {
      "epoch": 1.8106649218941686,
      "grad_norm": 8.777495384216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41615620255470276,
      "logits/rejected": -0.37014445662498474,
      "logps/chosen": -356.2493896484375,
      "logps/rejected": -376.91961669921875,
      "loss": 0.3927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9771080017089844,
      "rewards/margins": 1.810962200164795,
      "rewards/rejected": -3.7880699634552,
      "step": 13837
    },
    {
      "epoch": 1.8107957798315204,
      "grad_norm": 8.54992389678955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35308027267456055,
      "logits/rejected": -0.44929492473602295,
      "logps/chosen": -443.6268310546875,
      "logps/rejected": -439.6098327636719,
      "loss": 0.285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0375864505767822,
      "rewards/margins": 2.078610420227051,
      "rewards/rejected": -3.116196632385254,
      "step": 13838
    },
    {
      "epoch": 1.8109266377688722,
      "grad_norm": 10.15129566192627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33136889338493347,
      "logits/rejected": -0.3005003333091736,
      "logps/chosen": -487.653076171875,
      "logps/rejected": -549.380859375,
      "loss": 0.4359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2693904638290405,
      "rewards/margins": 1.3838509321212769,
      "rewards/rejected": -2.6532413959503174,
      "step": 13839
    },
    {
      "epoch": 1.811057495706224,
      "grad_norm": 15.350218772888184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34837472438812256,
      "logits/rejected": -0.38083210587501526,
      "logps/chosen": -608.580322265625,
      "logps/rejected": -670.2512817382812,
      "loss": 0.9728,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3332529067993164,
      "rewards/margins": 0.04079476743936539,
      "rewards/rejected": -2.3740479946136475,
      "step": 13840
    },
    {
      "epoch": 1.8111883536435758,
      "grad_norm": 6.033123016357422,
      "learning_rate": 5e-06,
      "logits/chosen": -0.517716646194458,
      "logits/rejected": -0.3839242458343506,
      "logps/chosen": -468.9168701171875,
      "logps/rejected": -407.42962646484375,
      "loss": 0.2695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1375693082809448,
      "rewards/margins": 1.2733359336853027,
      "rewards/rejected": -2.410905122756958,
      "step": 13841
    },
    {
      "epoch": 1.8113192115809276,
      "grad_norm": 13.210911750793457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43062034249305725,
      "logits/rejected": -0.6571809649467468,
      "logps/chosen": -446.2728271484375,
      "logps/rejected": -458.6006164550781,
      "loss": 0.6844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1298803091049194,
      "rewards/margins": 1.1114451885223389,
      "rewards/rejected": -2.241325616836548,
      "step": 13842
    },
    {
      "epoch": 1.8114500695182794,
      "grad_norm": 14.391239166259766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3807678818702698,
      "logits/rejected": -0.6265605092048645,
      "logps/chosen": -638.434326171875,
      "logps/rejected": -555.1920166015625,
      "loss": 0.46,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9517242908477783,
      "rewards/margins": 2.182053804397583,
      "rewards/rejected": -4.133778095245361,
      "step": 13843
    },
    {
      "epoch": 1.811580927455631,
      "grad_norm": 11.886943817138672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4821699857711792,
      "logits/rejected": -0.32754024863243103,
      "logps/chosen": -507.7539367675781,
      "logps/rejected": -454.11968994140625,
      "loss": 0.5832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.416533350944519,
      "rewards/margins": 0.6299787759780884,
      "rewards/rejected": -2.0465121269226074,
      "step": 13844
    },
    {
      "epoch": 1.8117117853929827,
      "grad_norm": 5.598145008087158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5712994337081909,
      "logits/rejected": -0.30505335330963135,
      "logps/chosen": -436.974853515625,
      "logps/rejected": -516.3641967773438,
      "loss": 0.2228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6780050992965698,
      "rewards/margins": 2.401512622833252,
      "rewards/rejected": -3.0795178413391113,
      "step": 13845
    },
    {
      "epoch": 1.8118426433303345,
      "grad_norm": 8.181681632995605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.357287734746933,
      "logits/rejected": -0.33595147728919983,
      "logps/chosen": -585.7213134765625,
      "logps/rejected": -537.6053466796875,
      "loss": 0.4191,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9004254341125488,
      "rewards/margins": 1.5022826194763184,
      "rewards/rejected": -3.4027082920074463,
      "step": 13846
    },
    {
      "epoch": 1.8119735012676863,
      "grad_norm": 7.6296257972717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4819909334182739,
      "logits/rejected": -0.46352648735046387,
      "logps/chosen": -352.3307800292969,
      "logps/rejected": -398.7283935546875,
      "loss": 0.2831,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5782006978988647,
      "rewards/margins": 2.07212495803833,
      "rewards/rejected": -3.6503260135650635,
      "step": 13847
    },
    {
      "epoch": 1.8121043592050379,
      "grad_norm": 5.109201431274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41303977370262146,
      "logits/rejected": -0.27415764331817627,
      "logps/chosen": -342.5094299316406,
      "logps/rejected": -455.3869934082031,
      "loss": 0.264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.573587417602539,
      "rewards/margins": 1.8828049898147583,
      "rewards/rejected": -3.4563920497894287,
      "step": 13848
    },
    {
      "epoch": 1.8122352171423897,
      "grad_norm": 10.171381950378418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05640910938382149,
      "logits/rejected": -0.1743694245815277,
      "logps/chosen": -524.8885498046875,
      "logps/rejected": -390.2666015625,
      "loss": 0.4912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.49203360080719,
      "rewards/margins": 0.9480594396591187,
      "rewards/rejected": -2.4400930404663086,
      "step": 13849
    },
    {
      "epoch": 1.8123660750797415,
      "grad_norm": 4.203946113586426,
      "learning_rate": 5e-06,
      "logits/chosen": 0.023495696485042572,
      "logits/rejected": -0.23588362336158752,
      "logps/chosen": -562.1161499023438,
      "logps/rejected": -497.01287841796875,
      "loss": 0.231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29994410276412964,
      "rewards/margins": 2.501854181289673,
      "rewards/rejected": -2.801798105239868,
      "step": 13850
    },
    {
      "epoch": 1.8124969330170932,
      "grad_norm": 7.745264053344727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35210683941841125,
      "logits/rejected": -0.4798097610473633,
      "logps/chosen": -550.9971923828125,
      "logps/rejected": -461.88836669921875,
      "loss": 0.4515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7492204904556274,
      "rewards/margins": 1.0634175539016724,
      "rewards/rejected": -2.8126375675201416,
      "step": 13851
    },
    {
      "epoch": 1.812627790954445,
      "grad_norm": 10.652796745300293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3156268298625946,
      "logits/rejected": -0.41660380363464355,
      "logps/chosen": -324.4384460449219,
      "logps/rejected": -360.9771728515625,
      "loss": 0.5614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2415246963500977,
      "rewards/margins": 0.804989755153656,
      "rewards/rejected": -3.0465145111083984,
      "step": 13852
    },
    {
      "epoch": 1.8127586488917968,
      "grad_norm": 3.806243896484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0765458345413208,
      "logits/rejected": -0.19270507991313934,
      "logps/chosen": -363.8857116699219,
      "logps/rejected": -433.0888671875,
      "loss": 0.2048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7655061483383179,
      "rewards/margins": 1.949323296546936,
      "rewards/rejected": -3.714829444885254,
      "step": 13853
    },
    {
      "epoch": 1.8128895068291486,
      "grad_norm": 9.394227981567383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08370086550712585,
      "logits/rejected": -0.2613662779331207,
      "logps/chosen": -543.1546630859375,
      "logps/rejected": -449.34228515625,
      "loss": 0.4646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5098685026168823,
      "rewards/margins": 0.968518853187561,
      "rewards/rejected": -2.4783871173858643,
      "step": 13854
    },
    {
      "epoch": 1.8130203647665004,
      "grad_norm": 7.099093914031982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30888575315475464,
      "logits/rejected": -0.4225409924983978,
      "logps/chosen": -615.364013671875,
      "logps/rejected": -582.2105712890625,
      "loss": 0.2123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9164499044418335,
      "rewards/margins": 2.557194471359253,
      "rewards/rejected": -4.473644256591797,
      "step": 13855
    },
    {
      "epoch": 1.8131512227038522,
      "grad_norm": 10.793464660644531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07503915578126907,
      "logits/rejected": -0.1514868289232254,
      "logps/chosen": -671.769775390625,
      "logps/rejected": -551.654296875,
      "loss": 0.4664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3545575141906738,
      "rewards/margins": 1.0613294839859009,
      "rewards/rejected": -2.4158871173858643,
      "step": 13856
    },
    {
      "epoch": 1.813282080641204,
      "grad_norm": 10.295109748840332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38046395778656006,
      "logits/rejected": -0.22906753420829773,
      "logps/chosen": -398.00408935546875,
      "logps/rejected": -343.4935302734375,
      "loss": 0.526,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7317744493484497,
      "rewards/margins": 1.2926745414733887,
      "rewards/rejected": -3.024449110031128,
      "step": 13857
    },
    {
      "epoch": 1.8134129385785558,
      "grad_norm": 16.223949432373047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25788265466690063,
      "logits/rejected": -0.2743512690067291,
      "logps/chosen": -289.5633850097656,
      "logps/rejected": -436.5934143066406,
      "loss": 0.7231,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3664803504943848,
      "rewards/margins": 1.4620333909988403,
      "rewards/rejected": -2.8285136222839355,
      "step": 13858
    },
    {
      "epoch": 1.8135437965159076,
      "grad_norm": 10.153606414794922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.279435932636261,
      "logits/rejected": -0.3764519989490509,
      "logps/chosen": -350.602294921875,
      "logps/rejected": -386.42327880859375,
      "loss": 0.6218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1066621541976929,
      "rewards/margins": 0.9265108108520508,
      "rewards/rejected": -2.033173084259033,
      "step": 13859
    },
    {
      "epoch": 1.8136746544532591,
      "grad_norm": 9.535128593444824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26311904191970825,
      "logits/rejected": -0.19227635860443115,
      "logps/chosen": -476.5723876953125,
      "logps/rejected": -512.9995727539062,
      "loss": 0.6028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9144365191459656,
      "rewards/margins": 0.6219891309738159,
      "rewards/rejected": -1.5364257097244263,
      "step": 13860
    },
    {
      "epoch": 1.813805512390611,
      "grad_norm": 13.29016399383545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5009065270423889,
      "logits/rejected": -0.5348224639892578,
      "logps/chosen": -494.0716552734375,
      "logps/rejected": -467.1229248046875,
      "loss": 0.6083,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.669942855834961,
      "rewards/margins": 0.6922175884246826,
      "rewards/rejected": -2.3621606826782227,
      "step": 13861
    },
    {
      "epoch": 1.8139363703279627,
      "grad_norm": 4.7909159660339355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40192553400993347,
      "logits/rejected": -0.2628139853477478,
      "logps/chosen": -705.8399658203125,
      "logps/rejected": -415.6152648925781,
      "loss": 0.161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1468650102615356,
      "rewards/margins": 2.986663341522217,
      "rewards/rejected": -4.133528709411621,
      "step": 13862
    },
    {
      "epoch": 1.8140672282653143,
      "grad_norm": 8.991814613342285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2633243203163147,
      "logits/rejected": -0.23716437816619873,
      "logps/chosen": -370.82659912109375,
      "logps/rejected": -415.613525390625,
      "loss": 0.3259,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2364521026611328,
      "rewards/margins": 2.0832886695861816,
      "rewards/rejected": -3.3197407722473145,
      "step": 13863
    },
    {
      "epoch": 1.814198086202666,
      "grad_norm": 9.393704414367676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30162739753723145,
      "logits/rejected": -0.2910921275615692,
      "logps/chosen": -361.1600646972656,
      "logps/rejected": -376.2279357910156,
      "loss": 0.5042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.009842872619629,
      "rewards/margins": 0.6216485500335693,
      "rewards/rejected": -2.6314916610717773,
      "step": 13864
    },
    {
      "epoch": 1.8143289441400179,
      "grad_norm": 7.600204944610596,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15247279405593872,
      "logits/rejected": -0.09861493855714798,
      "logps/chosen": -560.1392211914062,
      "logps/rejected": -370.9015808105469,
      "loss": 0.3977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9075485467910767,
      "rewards/margins": 1.3932560682296753,
      "rewards/rejected": -2.300804376602173,
      "step": 13865
    },
    {
      "epoch": 1.8144598020773697,
      "grad_norm": 11.724284172058105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11461442708969116,
      "logits/rejected": -0.2453155219554901,
      "logps/chosen": -528.4996337890625,
      "logps/rejected": -367.4576416015625,
      "loss": 0.5379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9072030782699585,
      "rewards/margins": 0.9353893399238586,
      "rewards/rejected": -2.842592477798462,
      "step": 13866
    },
    {
      "epoch": 1.8145906600147215,
      "grad_norm": 4.653754711151123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20193719863891602,
      "logits/rejected": -0.1604662537574768,
      "logps/chosen": -286.3273620605469,
      "logps/rejected": -436.96343994140625,
      "loss": 0.2782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4143257141113281,
      "rewards/margins": 1.9673442840576172,
      "rewards/rejected": -3.3816702365875244,
      "step": 13867
    },
    {
      "epoch": 1.8147215179520733,
      "grad_norm": 5.164233207702637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08470382541418076,
      "logits/rejected": -0.08583730459213257,
      "logps/chosen": -409.7120666503906,
      "logps/rejected": -413.3408203125,
      "loss": 0.1818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4948219358921051,
      "rewards/margins": 2.053783893585205,
      "rewards/rejected": -2.5486059188842773,
      "step": 13868
    },
    {
      "epoch": 1.814852375889425,
      "grad_norm": 8.34801959991455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2180595099925995,
      "logits/rejected": -0.22350427508354187,
      "logps/chosen": -507.54913330078125,
      "logps/rejected": -556.278564453125,
      "loss": 0.4746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0923826694488525,
      "rewards/margins": 1.1394128799438477,
      "rewards/rejected": -2.2317955493927,
      "step": 13869
    },
    {
      "epoch": 1.8149832338267768,
      "grad_norm": 7.414674282073975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19397366046905518,
      "logits/rejected": -0.5000147223472595,
      "logps/chosen": -379.71490478515625,
      "logps/rejected": -443.5086669921875,
      "loss": 0.3812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8468353748321533,
      "rewards/margins": 1.6197396516799927,
      "rewards/rejected": -3.4665749073028564,
      "step": 13870
    },
    {
      "epoch": 1.8151140917641286,
      "grad_norm": 10.202302932739258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3798794150352478,
      "logits/rejected": -0.3640657663345337,
      "logps/chosen": -416.7283020019531,
      "logps/rejected": -560.626708984375,
      "loss": 0.4825,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7409229874610901,
      "rewards/margins": 1.3445956707000732,
      "rewards/rejected": -2.0855185985565186,
      "step": 13871
    },
    {
      "epoch": 1.8152449497014804,
      "grad_norm": 5.384458065032959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29194071888923645,
      "logits/rejected": -0.172489732503891,
      "logps/chosen": -402.51165771484375,
      "logps/rejected": -417.3623352050781,
      "loss": 0.1891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2235275506973267,
      "rewards/margins": 2.2076449394226074,
      "rewards/rejected": -3.4311723709106445,
      "step": 13872
    },
    {
      "epoch": 1.8153758076388322,
      "grad_norm": 8.067045211791992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.207495778799057,
      "logits/rejected": -0.39495769143104553,
      "logps/chosen": -426.01953125,
      "logps/rejected": -465.05535888671875,
      "loss": 0.4918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3949631452560425,
      "rewards/margins": 0.9809713959693909,
      "rewards/rejected": -2.375934600830078,
      "step": 13873
    },
    {
      "epoch": 1.815506665576184,
      "grad_norm": 8.747954368591309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10513533651828766,
      "logits/rejected": -0.2971823811531067,
      "logps/chosen": -494.7466735839844,
      "logps/rejected": -462.54425048828125,
      "loss": 0.4041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3398544788360596,
      "rewards/margins": 2.2607483863830566,
      "rewards/rejected": -3.600602626800537,
      "step": 13874
    },
    {
      "epoch": 1.8156375235135356,
      "grad_norm": 6.579504013061523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2103653848171234,
      "logits/rejected": -0.18155112862586975,
      "logps/chosen": -611.133056640625,
      "logps/rejected": -513.3760986328125,
      "loss": 0.2499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1416360139846802,
      "rewards/margins": 1.6490190029144287,
      "rewards/rejected": -2.7906551361083984,
      "step": 13875
    },
    {
      "epoch": 1.8157683814508874,
      "grad_norm": 9.656496047973633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4170672297477722,
      "logits/rejected": -0.4003140330314636,
      "logps/chosen": -485.00701904296875,
      "logps/rejected": -653.02978515625,
      "loss": 0.5458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3744075298309326,
      "rewards/margins": 0.5811962485313416,
      "rewards/rejected": -1.9556037187576294,
      "step": 13876
    },
    {
      "epoch": 1.8158992393882392,
      "grad_norm": 9.554337501525879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39222607016563416,
      "logits/rejected": -0.4174356460571289,
      "logps/chosen": -444.56805419921875,
      "logps/rejected": -369.35211181640625,
      "loss": 0.5143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7420064210891724,
      "rewards/margins": 1.1190754175186157,
      "rewards/rejected": -1.8610817193984985,
      "step": 13877
    },
    {
      "epoch": 1.816030097325591,
      "grad_norm": 32.477264404296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17389100790023804,
      "logits/rejected": -0.04454122856259346,
      "logps/chosen": -780.48486328125,
      "logps/rejected": -603.4061279296875,
      "loss": 0.9796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0013349056243896,
      "rewards/margins": 0.5362639427185059,
      "rewards/rejected": -2.5375988483428955,
      "step": 13878
    },
    {
      "epoch": 1.8161609552629425,
      "grad_norm": 4.469749927520752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5956902503967285,
      "logits/rejected": -0.52149897813797,
      "logps/chosen": -380.8148193359375,
      "logps/rejected": -453.7357482910156,
      "loss": 0.2983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6430098414421082,
      "rewards/margins": 1.710856318473816,
      "rewards/rejected": -2.3538661003112793,
      "step": 13879
    },
    {
      "epoch": 1.8162918132002943,
      "grad_norm": 10.039420127868652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27587616443634033,
      "logits/rejected": -0.22011831402778625,
      "logps/chosen": -383.58551025390625,
      "logps/rejected": -497.8953552246094,
      "loss": 0.3436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5331292152404785,
      "rewards/margins": 2.5383212566375732,
      "rewards/rejected": -4.071450710296631,
      "step": 13880
    },
    {
      "epoch": 1.816422671137646,
      "grad_norm": 12.295199394226074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11361578106880188,
      "logits/rejected": -0.2666304111480713,
      "logps/chosen": -499.81951904296875,
      "logps/rejected": -446.6544494628906,
      "loss": 0.6434,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2435274124145508,
      "rewards/margins": 0.7987728118896484,
      "rewards/rejected": -2.0423004627227783,
      "step": 13881
    },
    {
      "epoch": 1.816553529074998,
      "grad_norm": 9.12242317199707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43377795815467834,
      "logits/rejected": -0.505233645439148,
      "logps/chosen": -454.5406494140625,
      "logps/rejected": -416.3152160644531,
      "loss": 0.5272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4614301919937134,
      "rewards/margins": 0.9928094148635864,
      "rewards/rejected": -2.4542396068573,
      "step": 13882
    },
    {
      "epoch": 1.8166843870123497,
      "grad_norm": 18.644746780395508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31137949228286743,
      "logits/rejected": -0.16495466232299805,
      "logps/chosen": -398.7225341796875,
      "logps/rejected": -549.5755004882812,
      "loss": 0.8887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3864736557006836,
      "rewards/margins": 0.30654120445251465,
      "rewards/rejected": -2.6930153369903564,
      "step": 13883
    },
    {
      "epoch": 1.8168152449497015,
      "grad_norm": 5.975711822509766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10605377703905106,
      "logits/rejected": -0.40652573108673096,
      "logps/chosen": -529.1412963867188,
      "logps/rejected": -479.21014404296875,
      "loss": 0.1867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5003431439399719,
      "rewards/margins": 2.0502662658691406,
      "rewards/rejected": -2.5506093502044678,
      "step": 13884
    },
    {
      "epoch": 1.8169461028870533,
      "grad_norm": 7.030298233032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24700850248336792,
      "logits/rejected": 0.017792820930480957,
      "logps/chosen": -284.69091796875,
      "logps/rejected": -500.5466613769531,
      "loss": 0.2402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3673202395439148,
      "rewards/margins": 1.9576271772384644,
      "rewards/rejected": -2.3249475955963135,
      "step": 13885
    },
    {
      "epoch": 1.817076960824405,
      "grad_norm": 11.899569511413574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19770276546478271,
      "logits/rejected": -0.31166183948516846,
      "logps/chosen": -493.36053466796875,
      "logps/rejected": -478.8442687988281,
      "loss": 0.6105,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.158568263053894,
      "rewards/margins": 0.8119809627532959,
      "rewards/rejected": -1.9705491065979004,
      "step": 13886
    },
    {
      "epoch": 1.8172078187617569,
      "grad_norm": 14.842682838439941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32536837458610535,
      "logits/rejected": -0.3429974615573883,
      "logps/chosen": -548.4135131835938,
      "logps/rejected": -427.21173095703125,
      "loss": 0.511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0090380907058716,
      "rewards/margins": 0.6826059222221375,
      "rewards/rejected": -1.6916439533233643,
      "step": 13887
    },
    {
      "epoch": 1.8173386766991086,
      "grad_norm": 12.635977745056152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05020575225353241,
      "logits/rejected": -0.10125204175710678,
      "logps/chosen": -651.3642578125,
      "logps/rejected": -574.783203125,
      "loss": 0.5181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1286009550094604,
      "rewards/margins": 1.6123418807983398,
      "rewards/rejected": -2.7409427165985107,
      "step": 13888
    },
    {
      "epoch": 1.8174695346364604,
      "grad_norm": 8.248249053955078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.194758802652359,
      "logits/rejected": -0.07838127017021179,
      "logps/chosen": -317.5125427246094,
      "logps/rejected": -287.9173889160156,
      "loss": 0.7611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.194888949394226,
      "rewards/margins": 1.6188815832138062,
      "rewards/rejected": -2.8137705326080322,
      "step": 13889
    },
    {
      "epoch": 1.817600392573812,
      "grad_norm": 9.196300506591797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3785596489906311,
      "logits/rejected": -0.4319581091403961,
      "logps/chosen": -390.22869873046875,
      "logps/rejected": -385.09344482421875,
      "loss": 0.3076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1297309845685959,
      "rewards/margins": 1.7038278579711914,
      "rewards/rejected": -1.8335589170455933,
      "step": 13890
    },
    {
      "epoch": 1.8177312505111638,
      "grad_norm": 15.292816162109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6516066789627075,
      "logits/rejected": -0.8320991396903992,
      "logps/chosen": -472.2366943359375,
      "logps/rejected": -303.30938720703125,
      "loss": 0.886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.606574058532715,
      "rewards/margins": 0.5521734952926636,
      "rewards/rejected": -3.158747434616089,
      "step": 13891
    },
    {
      "epoch": 1.8178621084485156,
      "grad_norm": 13.714624404907227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42955702543258667,
      "logits/rejected": -0.7412673234939575,
      "logps/chosen": -732.6900024414062,
      "logps/rejected": -540.89453125,
      "loss": 0.4929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8842747211456299,
      "rewards/margins": 1.2670249938964844,
      "rewards/rejected": -3.1512999534606934,
      "step": 13892
    },
    {
      "epoch": 1.8179929663858674,
      "grad_norm": 11.997513771057129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3389660120010376,
      "logits/rejected": -0.513707160949707,
      "logps/chosen": -278.04461669921875,
      "logps/rejected": -295.7371826171875,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8004162311553955,
      "rewards/margins": 0.9760036468505859,
      "rewards/rejected": -2.7764198780059814,
      "step": 13893
    },
    {
      "epoch": 1.818123824323219,
      "grad_norm": 12.14599609375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3195743262767792,
      "logits/rejected": -0.1515030413866043,
      "logps/chosen": -374.5976867675781,
      "logps/rejected": -485.3241882324219,
      "loss": 0.6984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7365931272506714,
      "rewards/margins": 0.702608585357666,
      "rewards/rejected": -2.439201831817627,
      "step": 13894
    },
    {
      "epoch": 1.8182546822605707,
      "grad_norm": 8.567488670349121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15965551137924194,
      "logits/rejected": -0.40774330496788025,
      "logps/chosen": -317.9300537109375,
      "logps/rejected": -300.5263671875,
      "loss": 0.3833,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45956724882125854,
      "rewards/margins": 1.9137126207351685,
      "rewards/rejected": -2.3732798099517822,
      "step": 13895
    },
    {
      "epoch": 1.8183855401979225,
      "grad_norm": 15.175703048706055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14016832411289215,
      "logits/rejected": -0.0859946459531784,
      "logps/chosen": -531.1177978515625,
      "logps/rejected": -432.8809814453125,
      "loss": 1.0976,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.159441590309143,
      "rewards/margins": 0.3964622914791107,
      "rewards/rejected": -1.5559037923812866,
      "step": 13896
    },
    {
      "epoch": 1.8185163981352743,
      "grad_norm": 4.356464385986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16053715348243713,
      "logits/rejected": -0.41324007511138916,
      "logps/chosen": -377.2693176269531,
      "logps/rejected": -335.4949951171875,
      "loss": 0.1869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2403786182403564,
      "rewards/margins": 2.569326162338257,
      "rewards/rejected": -3.8097050189971924,
      "step": 13897
    },
    {
      "epoch": 1.8186472560726261,
      "grad_norm": 9.13318920135498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17799176275730133,
      "logits/rejected": -0.3034936189651489,
      "logps/chosen": -574.0689697265625,
      "logps/rejected": -534.5547485351562,
      "loss": 0.4144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19532930850982666,
      "rewards/margins": 2.6276116371154785,
      "rewards/rejected": -2.4322822093963623,
      "step": 13898
    },
    {
      "epoch": 1.818778114009978,
      "grad_norm": 9.058250427246094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36947935819625854,
      "logits/rejected": -0.559904932975769,
      "logps/chosen": -426.19482421875,
      "logps/rejected": -437.2573547363281,
      "loss": 0.4945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8692375421524048,
      "rewards/margins": 0.8411487340927124,
      "rewards/rejected": -1.7103862762451172,
      "step": 13899
    },
    {
      "epoch": 1.8189089719473297,
      "grad_norm": 9.871603965759277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004279688000679016,
      "logits/rejected": -0.0327901765704155,
      "logps/chosen": -411.0412902832031,
      "logps/rejected": -459.2437744140625,
      "loss": 0.4952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2556495666503906,
      "rewards/margins": 0.589015781879425,
      "rewards/rejected": -1.8446654081344604,
      "step": 13900
    },
    {
      "epoch": 1.8190398298846815,
      "grad_norm": 7.479467868804932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2788291871547699,
      "logits/rejected": -0.43797141313552856,
      "logps/chosen": -614.35888671875,
      "logps/rejected": -449.2991027832031,
      "loss": 0.2882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1860333681106567,
      "rewards/margins": 2.2111220359802246,
      "rewards/rejected": -3.39715576171875,
      "step": 13901
    },
    {
      "epoch": 1.8191706878220333,
      "grad_norm": 14.70522403717041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20742006599903107,
      "logits/rejected": 0.087469182908535,
      "logps/chosen": -565.379150390625,
      "logps/rejected": -516.2767944335938,
      "loss": 0.7592,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7238240242004395,
      "rewards/margins": 1.0479718446731567,
      "rewards/rejected": -2.7717959880828857,
      "step": 13902
    },
    {
      "epoch": 1.819301545759385,
      "grad_norm": 10.029451370239258,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12312757968902588,
      "logits/rejected": -0.19372442364692688,
      "logps/chosen": -479.98846435546875,
      "logps/rejected": -537.9482421875,
      "loss": 0.5284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9129505157470703,
      "rewards/margins": 1.599430799484253,
      "rewards/rejected": -2.5123813152313232,
      "step": 13903
    },
    {
      "epoch": 1.8194324036967369,
      "grad_norm": 5.1388349533081055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04433422535657883,
      "logits/rejected": -0.24404728412628174,
      "logps/chosen": -551.8609008789062,
      "logps/rejected": -479.8643493652344,
      "loss": 0.2016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17418213188648224,
      "rewards/margins": 2.1863389015197754,
      "rewards/rejected": -2.0121567249298096,
      "step": 13904
    },
    {
      "epoch": 1.8195632616340887,
      "grad_norm": 9.350425720214844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28472721576690674,
      "logits/rejected": -0.3200857937335968,
      "logps/chosen": -475.03106689453125,
      "logps/rejected": -375.80926513671875,
      "loss": 0.4478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5851247310638428,
      "rewards/margins": 1.1450642347335815,
      "rewards/rejected": -1.7301890850067139,
      "step": 13905
    },
    {
      "epoch": 1.8196941195714402,
      "grad_norm": 7.615041255950928,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4615887999534607,
      "logits/rejected": -0.5023308396339417,
      "logps/chosen": -474.56365966796875,
      "logps/rejected": -408.4173889160156,
      "loss": 0.3417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2254979610443115,
      "rewards/margins": 1.177958607673645,
      "rewards/rejected": -2.403456687927246,
      "step": 13906
    },
    {
      "epoch": 1.819824977508792,
      "grad_norm": 8.780131340026855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2758316993713379,
      "logits/rejected": -0.10940566658973694,
      "logps/chosen": -503.8255920410156,
      "logps/rejected": -464.20758056640625,
      "loss": 0.4949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0017000436782837,
      "rewards/margins": 1.802254557609558,
      "rewards/rejected": -2.803954601287842,
      "step": 13907
    },
    {
      "epoch": 1.8199558354461438,
      "grad_norm": 6.183794021606445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.211706280708313,
      "logits/rejected": -0.25415652990341187,
      "logps/chosen": -443.10125732421875,
      "logps/rejected": -455.0029296875,
      "loss": 0.2281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5253151059150696,
      "rewards/margins": 1.9021965265274048,
      "rewards/rejected": -2.427511692047119,
      "step": 13908
    },
    {
      "epoch": 1.8200866933834954,
      "grad_norm": 7.5401153564453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1201692596077919,
      "logits/rejected": -0.32548895478248596,
      "logps/chosen": -315.2552795410156,
      "logps/rejected": -508.92132568359375,
      "loss": 0.2589,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.038800835609436,
      "rewards/margins": 2.0802226066589355,
      "rewards/rejected": -3.119023084640503,
      "step": 13909
    },
    {
      "epoch": 1.8202175513208472,
      "grad_norm": 12.491561889648438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2518399655818939,
      "logits/rejected": -0.11678286641836166,
      "logps/chosen": -495.1785583496094,
      "logps/rejected": -682.2393798828125,
      "loss": 0.6292,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8365126848220825,
      "rewards/margins": 0.4138409495353699,
      "rewards/rejected": -2.2503535747528076,
      "step": 13910
    },
    {
      "epoch": 1.820348409258199,
      "grad_norm": 7.981207370758057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18828924000263214,
      "logits/rejected": -0.15514284372329712,
      "logps/chosen": -792.1721801757812,
      "logps/rejected": -640.5371704101562,
      "loss": 0.4403,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2639186680316925,
      "rewards/margins": 1.987621784210205,
      "rewards/rejected": -2.251540422439575,
      "step": 13911
    },
    {
      "epoch": 1.8204792671955508,
      "grad_norm": 6.951131343841553,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09357233345508575,
      "logits/rejected": -0.11188414692878723,
      "logps/chosen": -285.5505065917969,
      "logps/rejected": -306.82110595703125,
      "loss": 0.5097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1291241645812988,
      "rewards/margins": 1.079101324081421,
      "rewards/rejected": -2.2082254886627197,
      "step": 13912
    },
    {
      "epoch": 1.8206101251329025,
      "grad_norm": 10.199360847473145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08031358569860458,
      "logits/rejected": -0.18903464078903198,
      "logps/chosen": -456.4254150390625,
      "logps/rejected": -415.2969665527344,
      "loss": 0.6958,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8887958526611328,
      "rewards/margins": 0.3313322067260742,
      "rewards/rejected": -2.220128059387207,
      "step": 13913
    },
    {
      "epoch": 1.8207409830702543,
      "grad_norm": 5.778646469116211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5058690309524536,
      "logits/rejected": -0.4112837612628937,
      "logps/chosen": -506.6849670410156,
      "logps/rejected": -582.8594970703125,
      "loss": 0.1518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6490106582641602,
      "rewards/margins": 2.695798635482788,
      "rewards/rejected": -3.3448092937469482,
      "step": 13914
    },
    {
      "epoch": 1.8208718410076061,
      "grad_norm": 8.534266471862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5465266108512878,
      "logits/rejected": -0.2481059432029724,
      "logps/chosen": -323.7239074707031,
      "logps/rejected": -419.35015869140625,
      "loss": 0.4473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2160282135009766,
      "rewards/margins": 1.0269455909729004,
      "rewards/rejected": -2.242973804473877,
      "step": 13915
    },
    {
      "epoch": 1.821002698944958,
      "grad_norm": 3.3516480922698975,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33478355407714844,
      "logits/rejected": -0.4836362600326538,
      "logps/chosen": -266.4127197265625,
      "logps/rejected": -269.01153564453125,
      "loss": 0.2378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6724768280982971,
      "rewards/margins": 2.31569504737854,
      "rewards/rejected": -2.9881720542907715,
      "step": 13916
    },
    {
      "epoch": 1.8211335568823097,
      "grad_norm": 6.029262542724609,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14366552233695984,
      "logits/rejected": -0.10200469195842743,
      "logps/chosen": -492.1938781738281,
      "logps/rejected": -362.7002868652344,
      "loss": 0.296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6424598693847656,
      "rewards/margins": 1.6368929147720337,
      "rewards/rejected": -2.2793526649475098,
      "step": 13917
    },
    {
      "epoch": 1.8212644148196615,
      "grad_norm": 10.217606544494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26255643367767334,
      "logits/rejected": -0.28728345036506653,
      "logps/chosen": -625.9747314453125,
      "logps/rejected": -457.2406311035156,
      "loss": 0.5146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4598606824874878,
      "rewards/margins": 1.1385177373886108,
      "rewards/rejected": -2.5983786582946777,
      "step": 13918
    },
    {
      "epoch": 1.8213952727570133,
      "grad_norm": 10.80956745147705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12535743415355682,
      "logits/rejected": -0.1774875372648239,
      "logps/chosen": -351.0679626464844,
      "logps/rejected": -440.9916076660156,
      "loss": 0.4925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6891638040542603,
      "rewards/margins": 1.3806267976760864,
      "rewards/rejected": -2.0697903633117676,
      "step": 13919
    },
    {
      "epoch": 1.821526130694365,
      "grad_norm": 12.902434349060059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34163084626197815,
      "logits/rejected": -0.2705089747905731,
      "logps/chosen": -608.5828247070312,
      "logps/rejected": -522.2899780273438,
      "loss": 0.6465,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2274408340454102,
      "rewards/margins": 1.2369370460510254,
      "rewards/rejected": -2.4643781185150146,
      "step": 13920
    },
    {
      "epoch": 1.8216569886317167,
      "grad_norm": 11.469151496887207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06320405006408691,
      "logits/rejected": -0.005773669108748436,
      "logps/chosen": -372.6640319824219,
      "logps/rejected": -387.1163024902344,
      "loss": 0.7088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3286218643188477,
      "rewards/margins": 0.7110888361930847,
      "rewards/rejected": -2.039710521697998,
      "step": 13921
    },
    {
      "epoch": 1.8217878465690684,
      "grad_norm": 6.964097499847412,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010330229997634888,
      "logits/rejected": -0.10903571546077728,
      "logps/chosen": -547.8828735351562,
      "logps/rejected": -506.6958312988281,
      "loss": 0.3177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8642361164093018,
      "rewards/margins": 1.2235281467437744,
      "rewards/rejected": -2.087764263153076,
      "step": 13922
    },
    {
      "epoch": 1.8219187045064202,
      "grad_norm": 7.9860968589782715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04834612086415291,
      "logits/rejected": -0.01539042592048645,
      "logps/chosen": -487.001953125,
      "logps/rejected": -477.03857421875,
      "loss": 0.3542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36257022619247437,
      "rewards/margins": 1.0272668600082397,
      "rewards/rejected": -1.3898370265960693,
      "step": 13923
    },
    {
      "epoch": 1.822049562443772,
      "grad_norm": 8.422632217407227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36281347274780273,
      "logits/rejected": -0.35574403405189514,
      "logps/chosen": -479.2855224609375,
      "logps/rejected": -402.7806091308594,
      "loss": 0.437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.46142062544822693,
      "rewards/margins": 1.2497538328170776,
      "rewards/rejected": -1.711174488067627,
      "step": 13924
    },
    {
      "epoch": 1.8221804203811236,
      "grad_norm": 4.676722049713135,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43743541836738586,
      "logits/rejected": -0.4506155848503113,
      "logps/chosen": -618.9324951171875,
      "logps/rejected": -532.076904296875,
      "loss": 0.2028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35090142488479614,
      "rewards/margins": 1.883727788925171,
      "rewards/rejected": -2.2346291542053223,
      "step": 13925
    },
    {
      "epoch": 1.8223112783184754,
      "grad_norm": 5.72723913192749,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2830267548561096,
      "logits/rejected": -0.28270137310028076,
      "logps/chosen": -389.9535827636719,
      "logps/rejected": -412.3389587402344,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6805689334869385,
      "rewards/margins": 1.7070355415344238,
      "rewards/rejected": -2.3876044750213623,
      "step": 13926
    },
    {
      "epoch": 1.8224421362558272,
      "grad_norm": 13.534041404724121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.017116336151957512,
      "logits/rejected": -0.11625143885612488,
      "logps/chosen": -563.2925415039062,
      "logps/rejected": -547.1510620117188,
      "loss": 0.577,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5924923419952393,
      "rewards/margins": 0.7427608966827393,
      "rewards/rejected": -2.3352532386779785,
      "step": 13927
    },
    {
      "epoch": 1.822572994193179,
      "grad_norm": 12.413307189941406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06889918446540833,
      "logits/rejected": -0.20176424086093903,
      "logps/chosen": -534.7687377929688,
      "logps/rejected": -486.0340270996094,
      "loss": 0.5358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0590510368347168,
      "rewards/margins": 0.9057155847549438,
      "rewards/rejected": -1.964766502380371,
      "step": 13928
    },
    {
      "epoch": 1.8227038521305308,
      "grad_norm": 12.095497131347656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5281630754470825,
      "logits/rejected": -0.4741296172142029,
      "logps/chosen": -496.1463317871094,
      "logps/rejected": -341.1771545410156,
      "loss": 0.6094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9026139974594116,
      "rewards/margins": 0.7033544182777405,
      "rewards/rejected": -2.6059682369232178,
      "step": 13929
    },
    {
      "epoch": 1.8228347100678826,
      "grad_norm": 6.59742546081543,
      "learning_rate": 5e-06,
      "logits/chosen": 0.060366079211235046,
      "logits/rejected": -0.07881145924329758,
      "logps/chosen": -636.1976928710938,
      "logps/rejected": -542.39404296875,
      "loss": 0.3606,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07493627071380615,
      "rewards/margins": 1.8356541395187378,
      "rewards/rejected": -1.7607179880142212,
      "step": 13930
    },
    {
      "epoch": 1.8229655680052343,
      "grad_norm": 4.477747917175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5564557313919067,
      "logits/rejected": -0.33619749546051025,
      "logps/chosen": -403.21295166015625,
      "logps/rejected": -349.7621154785156,
      "loss": 0.1348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03431108593940735,
      "rewards/margins": 2.312288284301758,
      "rewards/rejected": -2.3465993404388428,
      "step": 13931
    },
    {
      "epoch": 1.8230964259425861,
      "grad_norm": 6.448234558105469,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20576956868171692,
      "logits/rejected": -0.20014354586601257,
      "logps/chosen": -409.6385192871094,
      "logps/rejected": -450.2682189941406,
      "loss": 0.3495,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.044044017791748,
      "rewards/margins": 1.6845035552978516,
      "rewards/rejected": -2.7285478115081787,
      "step": 13932
    },
    {
      "epoch": 1.823227283879938,
      "grad_norm": 8.96226978302002,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24581637978553772,
      "logits/rejected": -0.22828905284404755,
      "logps/chosen": -296.55474853515625,
      "logps/rejected": -394.51397705078125,
      "loss": 0.4331,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3108230829238892,
      "rewards/margins": 0.9099829792976379,
      "rewards/rejected": -2.2208058834075928,
      "step": 13933
    },
    {
      "epoch": 1.8233581418172897,
      "grad_norm": 8.508230209350586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38334837555885315,
      "logits/rejected": -0.6781752109527588,
      "logps/chosen": -344.5430908203125,
      "logps/rejected": -428.9842834472656,
      "loss": 0.4911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5176392793655396,
      "rewards/margins": 1.4658523797988892,
      "rewards/rejected": -2.983491897583008,
      "step": 13934
    },
    {
      "epoch": 1.8234889997546415,
      "grad_norm": 6.60898494720459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03489520400762558,
      "logits/rejected": -0.016580894589424133,
      "logps/chosen": -376.6065979003906,
      "logps/rejected": -366.54595947265625,
      "loss": 0.4166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1428654193878174,
      "rewards/margins": 1.014020562171936,
      "rewards/rejected": -2.156886100769043,
      "step": 13935
    },
    {
      "epoch": 1.823619857691993,
      "grad_norm": 16.536853790283203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.014850681647658348,
      "logits/rejected": -0.27259156107902527,
      "logps/chosen": -734.7767333984375,
      "logps/rejected": -487.0492858886719,
      "loss": 0.9825,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4847280979156494,
      "rewards/margins": 0.6723043322563171,
      "rewards/rejected": -2.1570324897766113,
      "step": 13936
    },
    {
      "epoch": 1.8237507156293449,
      "grad_norm": 11.62890911102295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32577842473983765,
      "logits/rejected": -0.4043208658695221,
      "logps/chosen": -593.0095825195312,
      "logps/rejected": -615.006591796875,
      "loss": 0.4877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6190850734710693,
      "rewards/margins": 1.8454715013504028,
      "rewards/rejected": -3.4645564556121826,
      "step": 13937
    },
    {
      "epoch": 1.8238815735666967,
      "grad_norm": 7.4433722496032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3179335594177246,
      "logits/rejected": -0.40385663509368896,
      "logps/chosen": -422.836669921875,
      "logps/rejected": -430.6742248535156,
      "loss": 0.476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.302100658416748,
      "rewards/margins": 0.9946863651275635,
      "rewards/rejected": -2.2967867851257324,
      "step": 13938
    },
    {
      "epoch": 1.8240124315040485,
      "grad_norm": 8.09391975402832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2165634036064148,
      "logits/rejected": -0.4872264564037323,
      "logps/chosen": -296.80694580078125,
      "logps/rejected": -312.45806884765625,
      "loss": 0.2481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3395941257476807,
      "rewards/margins": 2.0487115383148193,
      "rewards/rejected": -3.388305902481079,
      "step": 13939
    },
    {
      "epoch": 1.8241432894414,
      "grad_norm": 15.161215782165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06425077468156815,
      "logits/rejected": -0.25976622104644775,
      "logps/chosen": -538.2208251953125,
      "logps/rejected": -389.4174499511719,
      "loss": 0.7336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6117069125175476,
      "rewards/margins": 1.3844417333602905,
      "rewards/rejected": -1.996148705482483,
      "step": 13940
    },
    {
      "epoch": 1.8242741473787518,
      "grad_norm": 12.704060554504395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19329112768173218,
      "logits/rejected": -0.2743091583251953,
      "logps/chosen": -573.7880249023438,
      "logps/rejected": -424.3447265625,
      "loss": 0.9722,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1435481309890747,
      "rewards/margins": 0.5963321924209595,
      "rewards/rejected": -1.7398803234100342,
      "step": 13941
    },
    {
      "epoch": 1.8244050053161036,
      "grad_norm": 12.03431224822998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4688555598258972,
      "logits/rejected": -0.6527853608131409,
      "logps/chosen": -321.27362060546875,
      "logps/rejected": -436.1504821777344,
      "loss": 0.6657,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5322184562683105,
      "rewards/margins": 0.8611562848091125,
      "rewards/rejected": -2.3933746814727783,
      "step": 13942
    },
    {
      "epoch": 1.8245358632534554,
      "grad_norm": 10.041419982910156,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10704705119132996,
      "logits/rejected": -0.227331280708313,
      "logps/chosen": -605.434326171875,
      "logps/rejected": -514.9993286132812,
      "loss": 0.4572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7778992056846619,
      "rewards/margins": 1.7157214879989624,
      "rewards/rejected": -2.4936208724975586,
      "step": 13943
    },
    {
      "epoch": 1.8246667211908072,
      "grad_norm": 5.689135551452637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07148028910160065,
      "logits/rejected": -0.16937294602394104,
      "logps/chosen": -522.5613403320312,
      "logps/rejected": -542.458984375,
      "loss": 0.3246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1617190837860107,
      "rewards/margins": 1.1265655755996704,
      "rewards/rejected": -2.2882847785949707,
      "step": 13944
    },
    {
      "epoch": 1.824797579128159,
      "grad_norm": 10.237664222717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21517324447631836,
      "logits/rejected": -0.5007971525192261,
      "logps/chosen": -450.1310119628906,
      "logps/rejected": -394.9207763671875,
      "loss": 0.43,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2952173948287964,
      "rewards/margins": 1.4501675367355347,
      "rewards/rejected": -2.745384693145752,
      "step": 13945
    },
    {
      "epoch": 1.8249284370655108,
      "grad_norm": 10.103490829467773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2386995255947113,
      "logits/rejected": -0.26541945338249207,
      "logps/chosen": -429.9903869628906,
      "logps/rejected": -449.0469970703125,
      "loss": 0.6405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7966781854629517,
      "rewards/margins": 1.0211254358291626,
      "rewards/rejected": -1.8178036212921143,
      "step": 13946
    },
    {
      "epoch": 1.8250592950028626,
      "grad_norm": 8.646673202514648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15344049036502838,
      "logits/rejected": -0.4385443329811096,
      "logps/chosen": -521.1026000976562,
      "logps/rejected": -478.6744079589844,
      "loss": 0.5241,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4502416253089905,
      "rewards/margins": 0.9212961196899414,
      "rewards/rejected": -1.3715378046035767,
      "step": 13947
    },
    {
      "epoch": 1.8251901529402144,
      "grad_norm": 10.194112777709961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4596186578273773,
      "logits/rejected": -0.542256772518158,
      "logps/chosen": -407.30108642578125,
      "logps/rejected": -440.0693359375,
      "loss": 0.5764,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0184128284454346,
      "rewards/margins": 0.9562925100326538,
      "rewards/rejected": -1.9747052192687988,
      "step": 13948
    },
    {
      "epoch": 1.8253210108775662,
      "grad_norm": 6.847842693328857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6088994145393372,
      "logits/rejected": -0.45965826511383057,
      "logps/chosen": -344.04876708984375,
      "logps/rejected": -393.90032958984375,
      "loss": 0.2785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.21334969997406,
      "rewards/margins": 1.193581223487854,
      "rewards/rejected": -2.406930923461914,
      "step": 13949
    },
    {
      "epoch": 1.825451868814918,
      "grad_norm": 8.959346771240234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11687710136175156,
      "logits/rejected": -0.171190544962883,
      "logps/chosen": -612.9131469726562,
      "logps/rejected": -528.21142578125,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.723514199256897,
      "rewards/margins": 2.6078672409057617,
      "rewards/rejected": -3.331381320953369,
      "step": 13950
    },
    {
      "epoch": 1.8255827267522695,
      "grad_norm": 6.346065998077393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11475963145494461,
      "logits/rejected": -0.35932013392448425,
      "logps/chosen": -488.27197265625,
      "logps/rejected": -415.9557800292969,
      "loss": 0.2992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6024490594863892,
      "rewards/margins": 1.3922898769378662,
      "rewards/rejected": -1.994739055633545,
      "step": 13951
    },
    {
      "epoch": 1.8257135846896213,
      "grad_norm": 9.212363243103027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2774021029472351,
      "logits/rejected": -0.3375726044178009,
      "logps/chosen": -512.4957885742188,
      "logps/rejected": -616.5376586914062,
      "loss": 0.4399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.269904613494873,
      "rewards/margins": 1.1418919563293457,
      "rewards/rejected": -2.4117965698242188,
      "step": 13952
    },
    {
      "epoch": 1.825844442626973,
      "grad_norm": 12.967874526977539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3439970314502716,
      "logits/rejected": -0.254097044467926,
      "logps/chosen": -514.5213012695312,
      "logps/rejected": -513.7677001953125,
      "loss": 0.6737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.95762699842453,
      "rewards/margins": 0.8072350025177002,
      "rewards/rejected": -1.764861822128296,
      "step": 13953
    },
    {
      "epoch": 1.8259753005643249,
      "grad_norm": 12.492020606994629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24072889983654022,
      "logits/rejected": -0.177792027592659,
      "logps/chosen": -355.8402404785156,
      "logps/rejected": -401.1375732421875,
      "loss": 0.8819,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3933786153793335,
      "rewards/margins": 0.01896781474351883,
      "rewards/rejected": -1.412346601486206,
      "step": 13954
    },
    {
      "epoch": 1.8261061585016765,
      "grad_norm": 4.163114070892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11673404276371002,
      "logits/rejected": -0.24366775155067444,
      "logps/chosen": -419.7295837402344,
      "logps/rejected": -403.8604736328125,
      "loss": 0.1746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5062219500541687,
      "rewards/margins": 2.039844512939453,
      "rewards/rejected": -2.5460662841796875,
      "step": 13955
    },
    {
      "epoch": 1.8262370164390282,
      "grad_norm": 7.176529884338379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07286909967660904,
      "logits/rejected": -0.2702255845069885,
      "logps/chosen": -491.3262634277344,
      "logps/rejected": -602.5233154296875,
      "loss": 0.2291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7953466773033142,
      "rewards/margins": 2.164435386657715,
      "rewards/rejected": -2.959782361984253,
      "step": 13956
    },
    {
      "epoch": 1.82636787437638,
      "grad_norm": 3.8970775604248047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07199174910783768,
      "logits/rejected": -0.20257733762264252,
      "logps/chosen": -449.41802978515625,
      "logps/rejected": -513.04296875,
      "loss": 0.1938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29781895875930786,
      "rewards/margins": 2.281414031982422,
      "rewards/rejected": -2.579232931137085,
      "step": 13957
    },
    {
      "epoch": 1.8264987323137318,
      "grad_norm": 4.3604021072387695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020892959088087082,
      "logits/rejected": 0.1289217472076416,
      "logps/chosen": -701.7551879882812,
      "logps/rejected": -515.5325317382812,
      "loss": 0.1598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36542394757270813,
      "rewards/margins": 2.4008407592773438,
      "rewards/rejected": -2.035416841506958,
      "step": 13958
    },
    {
      "epoch": 1.8266295902510836,
      "grad_norm": 7.22498893737793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3741995394229889,
      "logits/rejected": -0.25366178154945374,
      "logps/chosen": -693.239013671875,
      "logps/rejected": -595.7003784179688,
      "loss": 0.3106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1500675678253174,
      "rewards/margins": 1.4085124731063843,
      "rewards/rejected": -2.558580160140991,
      "step": 13959
    },
    {
      "epoch": 1.8267604481884354,
      "grad_norm": 5.8570146560668945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3025605082511902,
      "logits/rejected": -0.25913113355636597,
      "logps/chosen": -532.8577270507812,
      "logps/rejected": -436.0270080566406,
      "loss": 0.2647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7034233808517456,
      "rewards/margins": 1.7822885513305664,
      "rewards/rejected": -3.4857115745544434,
      "step": 13960
    },
    {
      "epoch": 1.8268913061257872,
      "grad_norm": 8.956094741821289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2709830403327942,
      "logits/rejected": -0.266042560338974,
      "logps/chosen": -435.2920227050781,
      "logps/rejected": -474.2879638671875,
      "loss": 0.4938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0157232284545898,
      "rewards/margins": 1.1447932720184326,
      "rewards/rejected": -2.1605165004730225,
      "step": 13961
    },
    {
      "epoch": 1.827022164063139,
      "grad_norm": 6.856356620788574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11566958576440811,
      "logits/rejected": -0.42312315106391907,
      "logps/chosen": -470.4423828125,
      "logps/rejected": -555.0113525390625,
      "loss": 0.1907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9153500199317932,
      "rewards/margins": 2.3925681114196777,
      "rewards/rejected": -3.307917594909668,
      "step": 13962
    },
    {
      "epoch": 1.8271530220004908,
      "grad_norm": 6.4388933181762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0942651778459549,
      "logits/rejected": -0.06196878105401993,
      "logps/chosen": -453.5220642089844,
      "logps/rejected": -489.8717956542969,
      "loss": 0.2137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41082555055618286,
      "rewards/margins": 2.79646635055542,
      "rewards/rejected": -3.207291841506958,
      "step": 13963
    },
    {
      "epoch": 1.8272838799378426,
      "grad_norm": 8.429986000061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30682218074798584,
      "logits/rejected": -0.45777758955955505,
      "logps/chosen": -543.0211181640625,
      "logps/rejected": -341.13177490234375,
      "loss": 0.4531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8940449953079224,
      "rewards/margins": 2.1363987922668457,
      "rewards/rejected": -4.0304436683654785,
      "step": 13964
    },
    {
      "epoch": 1.8274147378751944,
      "grad_norm": 10.189780235290527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34761276841163635,
      "logits/rejected": -0.18032428622245789,
      "logps/chosen": -369.5037536621094,
      "logps/rejected": -381.56243896484375,
      "loss": 0.6695,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4715018272399902,
      "rewards/margins": 0.9714832901954651,
      "rewards/rejected": -2.4429855346679688,
      "step": 13965
    },
    {
      "epoch": 1.8275455958125462,
      "grad_norm": 15.675531387329102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12978577613830566,
      "logits/rejected": -0.3664245307445526,
      "logps/chosen": -633.0933837890625,
      "logps/rejected": -502.7543029785156,
      "loss": 0.62,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9993501901626587,
      "rewards/margins": 1.296130657196045,
      "rewards/rejected": -2.295480728149414,
      "step": 13966
    },
    {
      "epoch": 1.8276764537498977,
      "grad_norm": 9.800353050231934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18578128516674042,
      "logits/rejected": -0.4435989260673523,
      "logps/chosen": -537.9506225585938,
      "logps/rejected": -530.246337890625,
      "loss": 0.3543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9519704580307007,
      "rewards/margins": 1.1904239654541016,
      "rewards/rejected": -2.142394542694092,
      "step": 13967
    },
    {
      "epoch": 1.8278073116872495,
      "grad_norm": 10.316397666931152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09633509814739227,
      "logits/rejected": -0.27149584889411926,
      "logps/chosen": -417.0233459472656,
      "logps/rejected": -293.3450927734375,
      "loss": 0.5289,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2828421592712402,
      "rewards/margins": 0.6395922899246216,
      "rewards/rejected": -1.9224345684051514,
      "step": 13968
    },
    {
      "epoch": 1.8279381696246013,
      "grad_norm": 11.4864501953125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04298624396324158,
      "logits/rejected": 0.16221901774406433,
      "logps/chosen": -439.7779541015625,
      "logps/rejected": -660.4130249023438,
      "loss": 0.5428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5094791650772095,
      "rewards/margins": 0.6105010509490967,
      "rewards/rejected": -1.1199803352355957,
      "step": 13969
    },
    {
      "epoch": 1.828069027561953,
      "grad_norm": 11.177212715148926,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14521275460720062,
      "logits/rejected": 0.021446075290441513,
      "logps/chosen": -408.4977111816406,
      "logps/rejected": -481.00311279296875,
      "loss": 0.6871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.004333734512329,
      "rewards/margins": 0.8094373345375061,
      "rewards/rejected": -1.81377112865448,
      "step": 13970
    },
    {
      "epoch": 1.8281998854993047,
      "grad_norm": 4.405956268310547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24917691946029663,
      "logits/rejected": -0.44168180227279663,
      "logps/chosen": -341.50836181640625,
      "logps/rejected": -348.19268798828125,
      "loss": 0.3077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.399929404258728,
      "rewards/margins": 1.43251371383667,
      "rewards/rejected": -2.8324432373046875,
      "step": 13971
    },
    {
      "epoch": 1.8283307434366565,
      "grad_norm": 6.121216773986816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3741356432437897,
      "logits/rejected": -0.446286141872406,
      "logps/chosen": -472.2312927246094,
      "logps/rejected": -447.7018127441406,
      "loss": 0.3143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3512816429138184,
      "rewards/margins": 1.2023811340332031,
      "rewards/rejected": -2.5536627769470215,
      "step": 13972
    },
    {
      "epoch": 1.8284616013740083,
      "grad_norm": 5.483182430267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5403056144714355,
      "logits/rejected": -0.5424843430519104,
      "logps/chosen": -354.6134338378906,
      "logps/rejected": -683.36962890625,
      "loss": 0.2953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3004668951034546,
      "rewards/margins": 1.632457971572876,
      "rewards/rejected": -2.932924747467041,
      "step": 13973
    },
    {
      "epoch": 1.82859245931136,
      "grad_norm": 3.7919089794158936,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12265772372484207,
      "logits/rejected": -0.34280142188072205,
      "logps/chosen": -313.2983703613281,
      "logps/rejected": -433.6518249511719,
      "loss": 0.1533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3982667922973633,
      "rewards/margins": 2.245471715927124,
      "rewards/rejected": -3.6437389850616455,
      "step": 13974
    },
    {
      "epoch": 1.8287233172487118,
      "grad_norm": 7.455471992492676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16118432581424713,
      "logits/rejected": -0.16611658036708832,
      "logps/chosen": -526.5641479492188,
      "logps/rejected": -424.23516845703125,
      "loss": 0.2924,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3690848350524902,
      "rewards/margins": 1.636854887008667,
      "rewards/rejected": -3.0059397220611572,
      "step": 13975
    },
    {
      "epoch": 1.8288541751860636,
      "grad_norm": 10.08254337310791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5539181232452393,
      "logits/rejected": -0.2785629332065582,
      "logps/chosen": -299.0954895019531,
      "logps/rejected": -321.6334228515625,
      "loss": 0.5495,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1474015712738037,
      "rewards/margins": 0.8016838431358337,
      "rewards/rejected": -1.9490853548049927,
      "step": 13976
    },
    {
      "epoch": 1.8289850331234154,
      "grad_norm": 6.177802562713623,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10917414724826813,
      "logits/rejected": -0.14101536571979523,
      "logps/chosen": -414.1365966796875,
      "logps/rejected": -511.3356018066406,
      "loss": 0.3326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0857322216033936,
      "rewards/margins": 2.180572271347046,
      "rewards/rejected": -3.2663047313690186,
      "step": 13977
    },
    {
      "epoch": 1.8291158910607672,
      "grad_norm": 8.829041481018066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2340376377105713,
      "logits/rejected": -0.36542898416519165,
      "logps/chosen": -491.2523498535156,
      "logps/rejected": -458.041259765625,
      "loss": 0.5512,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6940441131591797,
      "rewards/margins": 0.9752664566040039,
      "rewards/rejected": -1.6693105697631836,
      "step": 13978
    },
    {
      "epoch": 1.829246748998119,
      "grad_norm": 13.351286888122559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4482356905937195,
      "logits/rejected": -0.4457187056541443,
      "logps/chosen": -518.1244506835938,
      "logps/rejected": -498.9041442871094,
      "loss": 0.5718,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1005909442901611,
      "rewards/margins": 2.3219733238220215,
      "rewards/rejected": -3.4225640296936035,
      "step": 13979
    },
    {
      "epoch": 1.8293776069354708,
      "grad_norm": 8.36312484741211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2786564528942108,
      "logits/rejected": -0.35898274183273315,
      "logps/chosen": -366.3309326171875,
      "logps/rejected": -403.87603759765625,
      "loss": 0.594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0462052822113037,
      "rewards/margins": 0.4236595332622528,
      "rewards/rejected": -1.469864845275879,
      "step": 13980
    },
    {
      "epoch": 1.8295084648728226,
      "grad_norm": 2.3117356300354004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4958922266960144,
      "logits/rejected": -0.6131071448326111,
      "logps/chosen": -431.0757141113281,
      "logps/rejected": -394.99456787109375,
      "loss": 0.1156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.126763105392456,
      "rewards/margins": 2.816255569458008,
      "rewards/rejected": -3.943018674850464,
      "step": 13981
    },
    {
      "epoch": 1.8296393228101742,
      "grad_norm": 6.505128860473633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3341560959815979,
      "logits/rejected": -0.20322611927986145,
      "logps/chosen": -333.0201721191406,
      "logps/rejected": -299.7574462890625,
      "loss": 0.4559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1058839559555054,
      "rewards/margins": 1.0670946836471558,
      "rewards/rejected": -2.172978162765503,
      "step": 13982
    },
    {
      "epoch": 1.829770180747526,
      "grad_norm": 5.585634708404541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025080490857362747,
      "logits/rejected": -0.3753473460674286,
      "logps/chosen": -611.3693237304688,
      "logps/rejected": -426.74810791015625,
      "loss": 0.192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7834978103637695,
      "rewards/margins": 2.044057607650757,
      "rewards/rejected": -2.8275551795959473,
      "step": 13983
    },
    {
      "epoch": 1.8299010386848777,
      "grad_norm": 14.675686836242676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3820106089115143,
      "logits/rejected": -0.08959238231182098,
      "logps/chosen": -342.69989013671875,
      "logps/rejected": -460.26220703125,
      "loss": 0.8601,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.528762936592102,
      "rewards/margins": 0.42199093103408813,
      "rewards/rejected": -1.9507538080215454,
      "step": 13984
    },
    {
      "epoch": 1.8300318966222295,
      "grad_norm": 13.532155990600586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4051530659198761,
      "logits/rejected": -0.26594841480255127,
      "logps/chosen": -558.1555786132812,
      "logps/rejected": -651.4951782226562,
      "loss": 0.5921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5271610021591187,
      "rewards/margins": 0.7333294749259949,
      "rewards/rejected": -2.2604904174804688,
      "step": 13985
    },
    {
      "epoch": 1.830162754559581,
      "grad_norm": 4.324077606201172,
      "learning_rate": 5e-06,
      "logits/chosen": 0.22271834313869476,
      "logits/rejected": 0.0343787744641304,
      "logps/chosen": -505.45819091796875,
      "logps/rejected": -509.8856506347656,
      "loss": 0.1777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4871550798416138,
      "rewards/margins": 2.037330389022827,
      "rewards/rejected": -3.5244855880737305,
      "step": 13986
    },
    {
      "epoch": 1.830293612496933,
      "grad_norm": 12.220276832580566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19548998773097992,
      "logits/rejected": -0.3436184227466583,
      "logps/chosen": -551.2684936523438,
      "logps/rejected": -382.91729736328125,
      "loss": 0.84,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1804605722427368,
      "rewards/margins": 1.1184418201446533,
      "rewards/rejected": -2.2989025115966797,
      "step": 13987
    },
    {
      "epoch": 1.8304244704342847,
      "grad_norm": 12.838669776916504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2443460375070572,
      "logits/rejected": -0.17791911959648132,
      "logps/chosen": -445.32354736328125,
      "logps/rejected": -429.6828918457031,
      "loss": 0.6534,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1382919549942017,
      "rewards/margins": 0.44591236114501953,
      "rewards/rejected": -1.5842044353485107,
      "step": 13988
    },
    {
      "epoch": 1.8305553283716365,
      "grad_norm": 10.963859558105469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1081668958067894,
      "logits/rejected": -0.238241046667099,
      "logps/chosen": -524.8968505859375,
      "logps/rejected": -394.2259521484375,
      "loss": 0.6161,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8956339359283447,
      "rewards/margins": 0.9055370688438416,
      "rewards/rejected": -2.801171064376831,
      "step": 13989
    },
    {
      "epoch": 1.8306861863089883,
      "grad_norm": 5.719403266906738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.044291283935308456,
      "logits/rejected": -0.062308549880981445,
      "logps/chosen": -387.1764831542969,
      "logps/rejected": -381.2398986816406,
      "loss": 0.1597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5132442116737366,
      "rewards/margins": 2.2807235717773438,
      "rewards/rejected": -2.7939674854278564,
      "step": 13990
    },
    {
      "epoch": 1.83081704424634,
      "grad_norm": 11.416120529174805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27791208028793335,
      "logits/rejected": -0.514863908290863,
      "logps/chosen": -519.015869140625,
      "logps/rejected": -375.91943359375,
      "loss": 0.5692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.579897165298462,
      "rewards/margins": 1.4053010940551758,
      "rewards/rejected": -2.9851982593536377,
      "step": 13991
    },
    {
      "epoch": 1.8309479021836919,
      "grad_norm": 8.286096572875977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26096057891845703,
      "logits/rejected": -0.47812649607658386,
      "logps/chosen": -431.1998596191406,
      "logps/rejected": -541.555908203125,
      "loss": 0.3332,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.493865966796875,
      "rewards/margins": 1.1584339141845703,
      "rewards/rejected": -2.6523001194000244,
      "step": 13992
    },
    {
      "epoch": 1.8310787601210436,
      "grad_norm": 6.7023420333862305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2622080445289612,
      "logits/rejected": -0.49270933866500854,
      "logps/chosen": -511.5846252441406,
      "logps/rejected": -694.4852294921875,
      "loss": 0.291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0213158130645752,
      "rewards/margins": 1.701707363128662,
      "rewards/rejected": -2.723022937774658,
      "step": 13993
    },
    {
      "epoch": 1.8312096180583954,
      "grad_norm": 11.299691200256348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.502687394618988,
      "logits/rejected": -0.4893740117549896,
      "logps/chosen": -490.47369384765625,
      "logps/rejected": -371.5088806152344,
      "loss": 0.4188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1092538833618164,
      "rewards/margins": 1.5051604509353638,
      "rewards/rejected": -2.6144144535064697,
      "step": 13994
    },
    {
      "epoch": 1.8313404759957472,
      "grad_norm": 5.374054431915283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.351428359746933,
      "logits/rejected": -0.4730166792869568,
      "logps/chosen": -649.2501831054688,
      "logps/rejected": -556.1262817382812,
      "loss": 0.1936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7585249543190002,
      "rewards/margins": 2.388015031814575,
      "rewards/rejected": -3.1465401649475098,
      "step": 13995
    },
    {
      "epoch": 1.831471333933099,
      "grad_norm": 1.3358322381973267,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30943042039871216,
      "logits/rejected": -0.3306056261062622,
      "logps/chosen": -255.97572326660156,
      "logps/rejected": -273.39959716796875,
      "loss": 0.0709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2621265649795532,
      "rewards/margins": 3.274440288543701,
      "rewards/rejected": -3.536566734313965,
      "step": 13996
    },
    {
      "epoch": 1.8316021918704506,
      "grad_norm": 7.670201778411865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1250365972518921,
      "logits/rejected": -0.30393943190574646,
      "logps/chosen": -593.9124755859375,
      "logps/rejected": -416.4548034667969,
      "loss": 0.3492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8244733810424805,
      "rewards/margins": 2.0818891525268555,
      "rewards/rejected": -2.906362771987915,
      "step": 13997
    },
    {
      "epoch": 1.8317330498078024,
      "grad_norm": 10.600439071655273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31323671340942383,
      "logits/rejected": -0.1465359628200531,
      "logps/chosen": -497.0386047363281,
      "logps/rejected": -513.7197265625,
      "loss": 0.4475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6278165578842163,
      "rewards/margins": 2.104640483856201,
      "rewards/rejected": -2.732457399368286,
      "step": 13998
    },
    {
      "epoch": 1.8318639077451542,
      "grad_norm": 10.407245635986328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022055163979530334,
      "logits/rejected": -0.38546156883239746,
      "logps/chosen": -437.4466857910156,
      "logps/rejected": -347.2425537109375,
      "loss": 0.549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.373434066772461,
      "rewards/margins": 0.6978897452354431,
      "rewards/rejected": -2.071323871612549,
      "step": 13999
    },
    {
      "epoch": 1.831994765682506,
      "grad_norm": 6.561121463775635,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2496614158153534,
      "logits/rejected": -0.27468371391296387,
      "logps/chosen": -418.66973876953125,
      "logps/rejected": -431.7171325683594,
      "loss": 0.4356,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0244550704956055,
      "rewards/margins": 1.29462730884552,
      "rewards/rejected": -2.319082498550415,
      "step": 14000
    },
    {
      "epoch": 1.8321256236198575,
      "grad_norm": 9.088525772094727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43450987339019775,
      "logits/rejected": -0.5041046738624573,
      "logps/chosen": -464.529541015625,
      "logps/rejected": -379.4320068359375,
      "loss": 0.4681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3366645574569702,
      "rewards/margins": 1.0862226486206055,
      "rewards/rejected": -2.422887086868286,
      "step": 14001
    },
    {
      "epoch": 1.8322564815572093,
      "grad_norm": 11.770971298217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1918800175189972,
      "logits/rejected": 0.11950083076953888,
      "logps/chosen": -555.6370849609375,
      "logps/rejected": -707.648193359375,
      "loss": 0.5044,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7513263821601868,
      "rewards/margins": 0.9150236248970032,
      "rewards/rejected": -1.66635000705719,
      "step": 14002
    },
    {
      "epoch": 1.8323873394945611,
      "grad_norm": 3.3821773529052734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.169602170586586,
      "logits/rejected": -0.10914038121700287,
      "logps/chosen": -471.6536865234375,
      "logps/rejected": -471.2875061035156,
      "loss": 0.1058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25753021240234375,
      "rewards/margins": 3.4598774909973145,
      "rewards/rejected": -3.2023472785949707,
      "step": 14003
    },
    {
      "epoch": 1.832518197431913,
      "grad_norm": 5.355742454528809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18669964373111725,
      "logits/rejected": -0.1822614222764969,
      "logps/chosen": -451.26898193359375,
      "logps/rejected": -400.8970031738281,
      "loss": 0.3062,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34692588448524475,
      "rewards/margins": 1.5729894638061523,
      "rewards/rejected": -1.9199155569076538,
      "step": 14004
    },
    {
      "epoch": 1.8326490553692647,
      "grad_norm": 13.454118728637695,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15603342652320862,
      "logits/rejected": 0.26947879791259766,
      "logps/chosen": -590.9740600585938,
      "logps/rejected": -426.68646240234375,
      "loss": 0.6762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7098147869110107,
      "rewards/margins": 1.675551414489746,
      "rewards/rejected": -3.3853659629821777,
      "step": 14005
    },
    {
      "epoch": 1.8327799133066165,
      "grad_norm": 9.189132690429688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2963411808013916,
      "logits/rejected": -0.24699389934539795,
      "logps/chosen": -539.577392578125,
      "logps/rejected": -560.34326171875,
      "loss": 0.4842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4006032943725586,
      "rewards/margins": 0.6670499444007874,
      "rewards/rejected": -3.0676534175872803,
      "step": 14006
    },
    {
      "epoch": 1.8329107712439683,
      "grad_norm": 10.610615730285645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4084295630455017,
      "logits/rejected": -0.3272777199745178,
      "logps/chosen": -445.968017578125,
      "logps/rejected": -457.7489929199219,
      "loss": 0.5761,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2424135208129883,
      "rewards/margins": 1.5984630584716797,
      "rewards/rejected": -2.840876579284668,
      "step": 14007
    },
    {
      "epoch": 1.83304162918132,
      "grad_norm": 9.359187126159668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41792070865631104,
      "logits/rejected": -0.6671900749206543,
      "logps/chosen": -544.0201416015625,
      "logps/rejected": -511.52667236328125,
      "loss": 0.2918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.230337142944336,
      "rewards/margins": 2.5980477333068848,
      "rewards/rejected": -3.8283846378326416,
      "step": 14008
    },
    {
      "epoch": 1.8331724871186719,
      "grad_norm": 11.158806800842285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09574204683303833,
      "logits/rejected": -0.11146007478237152,
      "logps/chosen": -447.6845397949219,
      "logps/rejected": -472.9917297363281,
      "loss": 0.7625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3483760356903076,
      "rewards/margins": 0.5407499074935913,
      "rewards/rejected": -1.8891260623931885,
      "step": 14009
    },
    {
      "epoch": 1.8333033450560237,
      "grad_norm": 10.682652473449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18900929391384125,
      "logits/rejected": -0.2192533016204834,
      "logps/chosen": -482.33099365234375,
      "logps/rejected": -462.35565185546875,
      "loss": 0.5999,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5751449465751648,
      "rewards/margins": 0.40805625915527344,
      "rewards/rejected": -0.983201265335083,
      "step": 14010
    },
    {
      "epoch": 1.8334342029933755,
      "grad_norm": 15.12360668182373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21365158259868622,
      "logits/rejected": -0.33652007579803467,
      "logps/chosen": -605.9674682617188,
      "logps/rejected": -426.3813781738281,
      "loss": 0.8349,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.002760171890259,
      "rewards/margins": 0.1665058135986328,
      "rewards/rejected": -2.1692662239074707,
      "step": 14011
    },
    {
      "epoch": 1.8335650609307272,
      "grad_norm": 13.830609321594238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09744894504547119,
      "logits/rejected": -0.12078169733285904,
      "logps/chosen": -370.31695556640625,
      "logps/rejected": -450.5264892578125,
      "loss": 0.8993,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8669047951698303,
      "rewards/margins": 0.3228679299354553,
      "rewards/rejected": -1.1897728443145752,
      "step": 14012
    },
    {
      "epoch": 1.8336959188680788,
      "grad_norm": 12.524694442749023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33441585302352905,
      "logits/rejected": -0.286843478679657,
      "logps/chosen": -342.7706298828125,
      "logps/rejected": -395.70611572265625,
      "loss": 0.7893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0312538146972656,
      "rewards/margins": 0.2995557188987732,
      "rewards/rejected": -1.3308095932006836,
      "step": 14013
    },
    {
      "epoch": 1.8338267768054306,
      "grad_norm": 8.25537395477295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.021596373990178108,
      "logits/rejected": -0.11429621279239655,
      "logps/chosen": -496.8141784667969,
      "logps/rejected": -389.34637451171875,
      "loss": 0.5221,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.281846821308136,
      "rewards/margins": 0.9544700384140015,
      "rewards/rejected": -1.2363168001174927,
      "step": 14014
    },
    {
      "epoch": 1.8339576347427824,
      "grad_norm": 7.429635047912598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4020712077617645,
      "logits/rejected": -0.16741597652435303,
      "logps/chosen": -412.365478515625,
      "logps/rejected": -542.1923828125,
      "loss": 0.3804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6020492315292358,
      "rewards/margins": 1.591292381286621,
      "rewards/rejected": -2.1933414936065674,
      "step": 14015
    },
    {
      "epoch": 1.834088492680134,
      "grad_norm": 3.053898811340332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2028190940618515,
      "logits/rejected": -0.31608617305755615,
      "logps/chosen": -518.3795166015625,
      "logps/rejected": -381.14434814453125,
      "loss": 0.1178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9082747101783752,
      "rewards/margins": 3.085984468460083,
      "rewards/rejected": -3.9942593574523926,
      "step": 14016
    },
    {
      "epoch": 1.8342193506174858,
      "grad_norm": 11.61569881439209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21651996672153473,
      "logits/rejected": -0.4086538851261139,
      "logps/chosen": -423.8426818847656,
      "logps/rejected": -367.65478515625,
      "loss": 0.5355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4452937841415405,
      "rewards/margins": 1.8868006467819214,
      "rewards/rejected": -3.332094669342041,
      "step": 14017
    },
    {
      "epoch": 1.8343502085548375,
      "grad_norm": 9.043981552124023,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00983024388551712,
      "logits/rejected": 0.11042329668998718,
      "logps/chosen": -517.564208984375,
      "logps/rejected": -554.054443359375,
      "loss": 0.4422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5680742263793945,
      "rewards/margins": 1.4616628885269165,
      "rewards/rejected": -2.0297369956970215,
      "step": 14018
    },
    {
      "epoch": 1.8344810664921893,
      "grad_norm": 4.603702068328857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16753223538398743,
      "logits/rejected": -0.3426375687122345,
      "logps/chosen": -523.4227294921875,
      "logps/rejected": -356.7421569824219,
      "loss": 0.2173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7642606496810913,
      "rewards/margins": 2.4248125553131104,
      "rewards/rejected": -3.189073085784912,
      "step": 14019
    },
    {
      "epoch": 1.8346119244295411,
      "grad_norm": 6.875271320343018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20898592472076416,
      "logits/rejected": -0.5061826705932617,
      "logps/chosen": -344.2040710449219,
      "logps/rejected": -337.7087097167969,
      "loss": 0.2682,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24283218383789062,
      "rewards/margins": 1.8270909786224365,
      "rewards/rejected": -2.069923162460327,
      "step": 14020
    },
    {
      "epoch": 1.834742782366893,
      "grad_norm": 12.392901420593262,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030603813007473946,
      "logits/rejected": -0.22197803854942322,
      "logps/chosen": -446.5220642089844,
      "logps/rejected": -364.67449951171875,
      "loss": 0.7915,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5246713161468506,
      "rewards/margins": -0.042519018054008484,
      "rewards/rejected": -0.48215237259864807,
      "step": 14021
    },
    {
      "epoch": 1.8348736403042447,
      "grad_norm": 6.766476631164551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19100187718868256,
      "logits/rejected": -0.13937640190124512,
      "logps/chosen": -330.5599670410156,
      "logps/rejected": -473.2445983886719,
      "loss": 0.3756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1723483800888062,
      "rewards/margins": 2.20937180519104,
      "rewards/rejected": -3.381720542907715,
      "step": 14022
    },
    {
      "epoch": 1.8350044982415965,
      "grad_norm": 6.699954032897949,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3349095582962036,
      "logits/rejected": -0.47136053442955017,
      "logps/chosen": -361.0354919433594,
      "logps/rejected": -278.396484375,
      "loss": 0.3704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6713842153549194,
      "rewards/margins": 1.820820689201355,
      "rewards/rejected": -2.4922049045562744,
      "step": 14023
    },
    {
      "epoch": 1.8351353561789483,
      "grad_norm": 8.499170303344727,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3357594609260559,
      "logits/rejected": -0.22065380215644836,
      "logps/chosen": -467.78155517578125,
      "logps/rejected": -378.5643310546875,
      "loss": 0.3535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.20211660861969,
      "rewards/margins": 1.557881474494934,
      "rewards/rejected": -2.759997844696045,
      "step": 14024
    },
    {
      "epoch": 1.8352662141163,
      "grad_norm": 9.51345157623291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0955938994884491,
      "logits/rejected": -0.17159317433834076,
      "logps/chosen": -442.7098083496094,
      "logps/rejected": -578.9703369140625,
      "loss": 0.5329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6901166439056396,
      "rewards/margins": 0.8101012110710144,
      "rewards/rejected": -1.5002177953720093,
      "step": 14025
    },
    {
      "epoch": 1.8353970720536519,
      "grad_norm": 10.691829681396484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09662016481161118,
      "logits/rejected": -0.24199944734573364,
      "logps/chosen": -463.0589904785156,
      "logps/rejected": -640.60986328125,
      "loss": 0.3343,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8393764495849609,
      "rewards/margins": 1.6138484477996826,
      "rewards/rejected": -2.4532248973846436,
      "step": 14026
    },
    {
      "epoch": 1.8355279299910037,
      "grad_norm": 13.466470718383789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19762098789215088,
      "logits/rejected": -0.10582974553108215,
      "logps/chosen": -555.063232421875,
      "logps/rejected": -658.0610961914062,
      "loss": 0.6008,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4266420602798462,
      "rewards/margins": 0.5126979947090149,
      "rewards/rejected": -1.9393399953842163,
      "step": 14027
    },
    {
      "epoch": 1.8356587879283552,
      "grad_norm": 5.283024311065674,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19739380478858948,
      "logits/rejected": -0.4202943444252014,
      "logps/chosen": -496.4347229003906,
      "logps/rejected": -345.9342956542969,
      "loss": 0.2885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7241017818450928,
      "rewards/margins": 1.5163315534591675,
      "rewards/rejected": -2.24043345451355,
      "step": 14028
    },
    {
      "epoch": 1.835789645865707,
      "grad_norm": 7.015437126159668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31514665484428406,
      "logits/rejected": -0.4594748020172119,
      "logps/chosen": -421.633544921875,
      "logps/rejected": -395.34100341796875,
      "loss": 0.3246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8682727217674255,
      "rewards/margins": 1.776464819908142,
      "rewards/rejected": -2.6447372436523438,
      "step": 14029
    },
    {
      "epoch": 1.8359205038030588,
      "grad_norm": 8.77660083770752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03568079695105553,
      "logits/rejected": -0.005805959925055504,
      "logps/chosen": -761.4463500976562,
      "logps/rejected": -636.785400390625,
      "loss": 0.4418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07337947934865952,
      "rewards/margins": 1.4988977909088135,
      "rewards/rejected": -1.42551851272583,
      "step": 14030
    },
    {
      "epoch": 1.8360513617404106,
      "grad_norm": 11.548157691955566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25283902883529663,
      "logits/rejected": 0.027933979406952858,
      "logps/chosen": -408.68768310546875,
      "logps/rejected": -536.1255493164062,
      "loss": 0.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5241326093673706,
      "rewards/margins": 1.145728588104248,
      "rewards/rejected": -2.669861316680908,
      "step": 14031
    },
    {
      "epoch": 1.8361822196777622,
      "grad_norm": 6.286362171173096,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10206402838230133,
      "logits/rejected": -0.16436609625816345,
      "logps/chosen": -402.1249084472656,
      "logps/rejected": -367.9642028808594,
      "loss": 0.397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0904096364974976,
      "rewards/margins": 1.6884342432022095,
      "rewards/rejected": -2.778843641281128,
      "step": 14032
    },
    {
      "epoch": 1.836313077615114,
      "grad_norm": 8.736260414123535,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16878584027290344,
      "logits/rejected": 0.040819235146045685,
      "logps/chosen": -270.54290771484375,
      "logps/rejected": -368.40899658203125,
      "loss": 0.5783,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7966244220733643,
      "rewards/margins": 1.0912092924118042,
      "rewards/rejected": -1.8878334760665894,
      "step": 14033
    },
    {
      "epoch": 1.8364439355524658,
      "grad_norm": 8.742981910705566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2833214998245239,
      "logits/rejected": -0.21812336146831512,
      "logps/chosen": -492.7452392578125,
      "logps/rejected": -431.9498291015625,
      "loss": 0.4428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2806146442890167,
      "rewards/margins": 1.05647611618042,
      "rewards/rejected": -0.7758614420890808,
      "step": 14034
    },
    {
      "epoch": 1.8365747934898176,
      "grad_norm": 8.586652755737305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3085401654243469,
      "logits/rejected": -0.18841677904129028,
      "logps/chosen": -494.5054931640625,
      "logps/rejected": -561.6375732421875,
      "loss": 0.5823,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4624757766723633,
      "rewards/margins": 0.9296370148658752,
      "rewards/rejected": -2.392112970352173,
      "step": 14035
    },
    {
      "epoch": 1.8367056514271694,
      "grad_norm": 10.339336395263672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21537074446678162,
      "logits/rejected": -0.13992181420326233,
      "logps/chosen": -321.3501892089844,
      "logps/rejected": -325.2511291503906,
      "loss": 0.532,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0348870754241943,
      "rewards/margins": 0.770055890083313,
      "rewards/rejected": -1.8049429655075073,
      "step": 14036
    },
    {
      "epoch": 1.8368365093645211,
      "grad_norm": 9.549957275390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04079165682196617,
      "logits/rejected": -0.11274325102567673,
      "logps/chosen": -534.5210571289062,
      "logps/rejected": -440.7010803222656,
      "loss": 0.4005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29813557863235474,
      "rewards/margins": 1.2185600996017456,
      "rewards/rejected": -1.5166957378387451,
      "step": 14037
    },
    {
      "epoch": 1.836967367301873,
      "grad_norm": 6.890247821807861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2888188362121582,
      "logits/rejected": -0.5355725288391113,
      "logps/chosen": -724.6624145507812,
      "logps/rejected": -522.27685546875,
      "loss": 0.3532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10852698981761932,
      "rewards/margins": 1.4007182121276855,
      "rewards/rejected": -1.5092452764511108,
      "step": 14038
    },
    {
      "epoch": 1.8370982252392247,
      "grad_norm": 6.397885799407959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11279211193323135,
      "logits/rejected": -0.23772403597831726,
      "logps/chosen": -456.8228759765625,
      "logps/rejected": -388.9214172363281,
      "loss": 0.3396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33744797110557556,
      "rewards/margins": 1.6053341627120972,
      "rewards/rejected": -1.9427820444107056,
      "step": 14039
    },
    {
      "epoch": 1.8372290831765765,
      "grad_norm": 9.66642951965332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5661447644233704,
      "logits/rejected": -0.5193809866905212,
      "logps/chosen": -354.3973388671875,
      "logps/rejected": -413.81378173828125,
      "loss": 0.5276,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4516037702560425,
      "rewards/margins": 1.2718931436538696,
      "rewards/rejected": -2.723496913909912,
      "step": 14040
    },
    {
      "epoch": 1.8373599411139283,
      "grad_norm": 5.371596813201904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08439952880144119,
      "logits/rejected": -0.29683464765548706,
      "logps/chosen": -532.5247802734375,
      "logps/rejected": -456.3607482910156,
      "loss": 0.303,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26813963055610657,
      "rewards/margins": 2.296966552734375,
      "rewards/rejected": -2.0288267135620117,
      "step": 14041
    },
    {
      "epoch": 1.83749079905128,
      "grad_norm": 10.537619590759277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4042114317417145,
      "logits/rejected": -0.45603322982788086,
      "logps/chosen": -407.997314453125,
      "logps/rejected": -510.57293701171875,
      "loss": 0.4782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3533976078033447,
      "rewards/margins": 1.262639045715332,
      "rewards/rejected": -2.616036891937256,
      "step": 14042
    },
    {
      "epoch": 1.8376216569886317,
      "grad_norm": 7.407217025756836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14630021154880524,
      "logits/rejected": -0.2550845146179199,
      "logps/chosen": -674.0933837890625,
      "logps/rejected": -524.0189819335938,
      "loss": 0.2225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7631416916847229,
      "rewards/margins": 1.8469302654266357,
      "rewards/rejected": -2.610071897506714,
      "step": 14043
    },
    {
      "epoch": 1.8377525149259835,
      "grad_norm": 7.261802673339844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26629969477653503,
      "logits/rejected": -0.16544124484062195,
      "logps/chosen": -352.85968017578125,
      "logps/rejected": -507.59368896484375,
      "loss": 0.389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8750120401382446,
      "rewards/margins": 2.180999517440796,
      "rewards/rejected": -3.05601167678833,
      "step": 14044
    },
    {
      "epoch": 1.8378833728633353,
      "grad_norm": 7.016733646392822,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19626396894454956,
      "logits/rejected": -0.25872257351875305,
      "logps/chosen": -510.00030517578125,
      "logps/rejected": -456.5582275390625,
      "loss": 0.3635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.839846670627594,
      "rewards/margins": 1.7217319011688232,
      "rewards/rejected": -2.5615782737731934,
      "step": 14045
    },
    {
      "epoch": 1.838014230800687,
      "grad_norm": 9.48198127746582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2139762043952942,
      "logits/rejected": -0.2238413393497467,
      "logps/chosen": -599.0601806640625,
      "logps/rejected": -486.4605712890625,
      "loss": 0.5035,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18168964982032776,
      "rewards/margins": 1.0171170234680176,
      "rewards/rejected": -1.1988065242767334,
      "step": 14046
    },
    {
      "epoch": 1.8381450887380386,
      "grad_norm": 4.3202056884765625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18289180099964142,
      "logits/rejected": -0.08227076381444931,
      "logps/chosen": -651.9500732421875,
      "logps/rejected": -627.1337890625,
      "loss": 0.1505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07854422926902771,
      "rewards/margins": 2.3041298389434814,
      "rewards/rejected": -2.382673978805542,
      "step": 14047
    },
    {
      "epoch": 1.8382759466753904,
      "grad_norm": 8.928086280822754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2104950249195099,
      "logits/rejected": -0.2073468267917633,
      "logps/chosen": -370.4075012207031,
      "logps/rejected": -401.48193359375,
      "loss": 0.4432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1867273449897766,
      "rewards/margins": 1.2468055486679077,
      "rewards/rejected": -1.4335328340530396,
      "step": 14048
    },
    {
      "epoch": 1.8384068046127422,
      "grad_norm": 11.625271797180176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0052501484751701355,
      "logits/rejected": -0.017407454550266266,
      "logps/chosen": -454.3077697753906,
      "logps/rejected": -433.6204528808594,
      "loss": 0.7468,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8350511193275452,
      "rewards/margins": 0.9352566003799438,
      "rewards/rejected": -1.7703078985214233,
      "step": 14049
    },
    {
      "epoch": 1.838537662550094,
      "grad_norm": 12.958212852478027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25238892436027527,
      "logits/rejected": -0.19326151907444,
      "logps/chosen": -451.57012939453125,
      "logps/rejected": -555.9757690429688,
      "loss": 0.7161,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.344457983970642,
      "rewards/margins": 0.3175586462020874,
      "rewards/rejected": -1.662016749382019,
      "step": 14050
    },
    {
      "epoch": 1.8386685204874458,
      "grad_norm": 7.269633769989014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14117243885993958,
      "logits/rejected": -0.2761363089084625,
      "logps/chosen": -466.1981201171875,
      "logps/rejected": -558.1992797851562,
      "loss": 0.2912,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.613734245300293,
      "rewards/margins": 2.4892470836639404,
      "rewards/rejected": -3.1029810905456543,
      "step": 14051
    },
    {
      "epoch": 1.8387993784247976,
      "grad_norm": 6.627199649810791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.422715961933136,
      "logits/rejected": -0.4090198576450348,
      "logps/chosen": -496.571533203125,
      "logps/rejected": -497.7427978515625,
      "loss": 0.3184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7346043586730957,
      "rewards/margins": 1.2847976684570312,
      "rewards/rejected": -3.019402027130127,
      "step": 14052
    },
    {
      "epoch": 1.8389302363621494,
      "grad_norm": 5.796688079833984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32794255018234253,
      "logits/rejected": -0.2282344102859497,
      "logps/chosen": -458.34954833984375,
      "logps/rejected": -575.9464111328125,
      "loss": 0.1861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8906072378158569,
      "rewards/margins": 2.253523826599121,
      "rewards/rejected": -3.1441311836242676,
      "step": 14053
    },
    {
      "epoch": 1.8390610942995012,
      "grad_norm": 4.48196268081665,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06811625510454178,
      "logits/rejected": -0.13351914286613464,
      "logps/chosen": -426.79290771484375,
      "logps/rejected": -370.8829040527344,
      "loss": 0.1681,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19823893904685974,
      "rewards/margins": 2.8012073040008545,
      "rewards/rejected": -2.602968215942383,
      "step": 14054
    },
    {
      "epoch": 1.839191952236853,
      "grad_norm": 9.30666732788086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42301344871520996,
      "logits/rejected": -0.42422744631767273,
      "logps/chosen": -337.9906005859375,
      "logps/rejected": -405.1428527832031,
      "loss": 0.4183,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5320521593093872,
      "rewards/margins": 1.3156630992889404,
      "rewards/rejected": -2.847714900970459,
      "step": 14055
    },
    {
      "epoch": 1.8393228101742047,
      "grad_norm": 12.820418357849121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34629949927330017,
      "logits/rejected": -0.41101017594337463,
      "logps/chosen": -529.4371337890625,
      "logps/rejected": -442.8248291015625,
      "loss": 0.6302,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1815006732940674,
      "rewards/margins": 0.933664083480835,
      "rewards/rejected": -2.1151649951934814,
      "step": 14056
    },
    {
      "epoch": 1.8394536681115565,
      "grad_norm": 9.258051872253418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2618054747581482,
      "logits/rejected": -0.27562448382377625,
      "logps/chosen": -508.7018127441406,
      "logps/rejected": -536.5559692382812,
      "loss": 0.4532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6967577934265137,
      "rewards/margins": 1.302161455154419,
      "rewards/rejected": -2.9989194869995117,
      "step": 14057
    },
    {
      "epoch": 1.8395845260489083,
      "grad_norm": 9.038375854492188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19256535172462463,
      "logits/rejected": -0.20759060978889465,
      "logps/chosen": -349.1768493652344,
      "logps/rejected": -268.92578125,
      "loss": 0.5149,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.053120493888855,
      "rewards/margins": 0.8955948352813721,
      "rewards/rejected": -1.9487152099609375,
      "step": 14058
    },
    {
      "epoch": 1.8397153839862599,
      "grad_norm": 8.138727188110352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2912094295024872,
      "logits/rejected": -0.3737334609031677,
      "logps/chosen": -412.909912109375,
      "logps/rejected": -458.114013671875,
      "loss": 0.3836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.357061743736267,
      "rewards/margins": 1.6147055625915527,
      "rewards/rejected": -2.9717674255371094,
      "step": 14059
    },
    {
      "epoch": 1.8398462419236117,
      "grad_norm": 7.038708209991455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2619645595550537,
      "logits/rejected": -0.026110969483852386,
      "logps/chosen": -361.712890625,
      "logps/rejected": -337.19805908203125,
      "loss": 0.2836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31213289499282837,
      "rewards/margins": 2.4019947052001953,
      "rewards/rejected": -2.0898618698120117,
      "step": 14060
    },
    {
      "epoch": 1.8399770998609635,
      "grad_norm": 10.112217903137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19226416945457458,
      "logits/rejected": -0.5183600783348083,
      "logps/chosen": -766.5584106445312,
      "logps/rejected": -514.0784912109375,
      "loss": 0.4593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2484604120254517,
      "rewards/margins": 1.463767647743225,
      "rewards/rejected": -2.7122278213500977,
      "step": 14061
    },
    {
      "epoch": 1.840107957798315,
      "grad_norm": 10.321893692016602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.219105064868927,
      "logits/rejected": -0.03925254940986633,
      "logps/chosen": -387.5406494140625,
      "logps/rejected": -300.8311462402344,
      "loss": 0.5454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9158936738967896,
      "rewards/margins": 0.8534135818481445,
      "rewards/rejected": -1.7693073749542236,
      "step": 14062
    },
    {
      "epoch": 1.8402388157356668,
      "grad_norm": 7.428383827209473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.049665436148643494,
      "logits/rejected": -0.12342887371778488,
      "logps/chosen": -449.0599365234375,
      "logps/rejected": -461.7398986816406,
      "loss": 0.311,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0917940139770508,
      "rewards/margins": 1.9428261518478394,
      "rewards/rejected": -3.0346202850341797,
      "step": 14063
    },
    {
      "epoch": 1.8403696736730186,
      "grad_norm": 5.570794582366943,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15759648382663727,
      "logits/rejected": 0.14291533827781677,
      "logps/chosen": -444.4502868652344,
      "logps/rejected": -541.2166748046875,
      "loss": 0.273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9352042078971863,
      "rewards/margins": 1.311387062072754,
      "rewards/rejected": -2.246591091156006,
      "step": 14064
    },
    {
      "epoch": 1.8405005316103704,
      "grad_norm": 8.510204315185547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5899044275283813,
      "logits/rejected": -0.6074897050857544,
      "logps/chosen": -350.2423400878906,
      "logps/rejected": -407.34625244140625,
      "loss": 0.3294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2255959510803223,
      "rewards/margins": 2.3976778984069824,
      "rewards/rejected": -3.6232738494873047,
      "step": 14065
    },
    {
      "epoch": 1.8406313895477222,
      "grad_norm": 9.832232475280762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.367087721824646,
      "logits/rejected": -0.3457443118095398,
      "logps/chosen": -426.1816101074219,
      "logps/rejected": -389.319580078125,
      "loss": 0.805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0741055011749268,
      "rewards/margins": 0.33358174562454224,
      "rewards/rejected": -1.4076871871948242,
      "step": 14066
    },
    {
      "epoch": 1.840762247485074,
      "grad_norm": 5.951701641082764,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04899560660123825,
      "logits/rejected": -0.08261819183826447,
      "logps/chosen": -474.8548278808594,
      "logps/rejected": -498.01080322265625,
      "loss": 0.2801,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8381708860397339,
      "rewards/margins": 2.23760724067688,
      "rewards/rejected": -3.0757782459259033,
      "step": 14067
    },
    {
      "epoch": 1.8408931054224258,
      "grad_norm": 4.196286678314209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3397690951824188,
      "logits/rejected": -0.5717198848724365,
      "logps/chosen": -421.2506103515625,
      "logps/rejected": -444.9207763671875,
      "loss": 0.1441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7528297901153564,
      "rewards/margins": 2.583993434906006,
      "rewards/rejected": -3.336822986602783,
      "step": 14068
    },
    {
      "epoch": 1.8410239633597776,
      "grad_norm": 17.511314392089844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4296320974826813,
      "logits/rejected": -0.4111793041229248,
      "logps/chosen": -506.791259765625,
      "logps/rejected": -490.74041748046875,
      "loss": 0.9604,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.3009347915649414,
      "rewards/margins": 0.017568230628967285,
      "rewards/rejected": -2.3185031414031982,
      "step": 14069
    },
    {
      "epoch": 1.8411548212971294,
      "grad_norm": 12.259087562561035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3727022111415863,
      "logits/rejected": -0.4574970006942749,
      "logps/chosen": -574.4501342773438,
      "logps/rejected": -400.7423095703125,
      "loss": 0.4305,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6764166355133057,
      "rewards/margins": 1.5051130056381226,
      "rewards/rejected": -2.1815297603607178,
      "step": 14070
    },
    {
      "epoch": 1.8412856792344812,
      "grad_norm": 6.072625160217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37136679887771606,
      "logits/rejected": -0.5631505250930786,
      "logps/chosen": -591.4708862304688,
      "logps/rejected": -444.5679626464844,
      "loss": 0.2176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.150068998336792,
      "rewards/margins": 1.9120527505874634,
      "rewards/rejected": -3.062121629714966,
      "step": 14071
    },
    {
      "epoch": 1.841416537171833,
      "grad_norm": 5.737176418304443,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1307046115398407,
      "logits/rejected": -0.30771008133888245,
      "logps/chosen": -642.7301025390625,
      "logps/rejected": -602.4625244140625,
      "loss": 0.2656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8287956714630127,
      "rewards/margins": 1.4169975519180298,
      "rewards/rejected": -2.245793104171753,
      "step": 14072
    },
    {
      "epoch": 1.8415473951091847,
      "grad_norm": 9.538151741027832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30437585711479187,
      "logits/rejected": -0.37496691942214966,
      "logps/chosen": -691.7050170898438,
      "logps/rejected": -519.6351928710938,
      "loss": 0.5055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.579698920249939,
      "rewards/margins": 0.7115787267684937,
      "rewards/rejected": -2.2912776470184326,
      "step": 14073
    },
    {
      "epoch": 1.8416782530465363,
      "grad_norm": 5.35970401763916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.553567111492157,
      "logits/rejected": -0.4182491898536682,
      "logps/chosen": -427.4072570800781,
      "logps/rejected": -374.6943359375,
      "loss": 0.1964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0061146020889282,
      "rewards/margins": 2.074470043182373,
      "rewards/rejected": -3.08058500289917,
      "step": 14074
    },
    {
      "epoch": 1.841809110983888,
      "grad_norm": 13.567398071289062,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05369919538497925,
      "logits/rejected": 0.01679603010416031,
      "logps/chosen": -588.153076171875,
      "logps/rejected": -509.65850830078125,
      "loss": 0.7119,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4632190465927124,
      "rewards/margins": 0.9156122207641602,
      "rewards/rejected": -2.378831386566162,
      "step": 14075
    },
    {
      "epoch": 1.84193996892124,
      "grad_norm": 6.644155025482178,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5514316558837891,
      "logits/rejected": -0.3187267780303955,
      "logps/chosen": -425.45770263671875,
      "logps/rejected": -439.91168212890625,
      "loss": 0.3543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8455349206924438,
      "rewards/margins": 1.5602840185165405,
      "rewards/rejected": -2.4058189392089844,
      "step": 14076
    },
    {
      "epoch": 1.8420708268585917,
      "grad_norm": 6.630067348480225,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22772924602031708,
      "logits/rejected": -0.3319534957408905,
      "logps/chosen": -456.0923767089844,
      "logps/rejected": -338.1259765625,
      "loss": 0.3955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1573388576507568,
      "rewards/margins": 1.4153555631637573,
      "rewards/rejected": -2.5726945400238037,
      "step": 14077
    },
    {
      "epoch": 1.8422016847959433,
      "grad_norm": 4.789866924285889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35676467418670654,
      "logits/rejected": -0.35695359110832214,
      "logps/chosen": -449.9940490722656,
      "logps/rejected": -465.7217102050781,
      "loss": 0.211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3679487705230713,
      "rewards/margins": 1.9435012340545654,
      "rewards/rejected": -3.3114500045776367,
      "step": 14078
    },
    {
      "epoch": 1.842332542733295,
      "grad_norm": 12.283894538879395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5052608251571655,
      "logits/rejected": -0.4047260582447052,
      "logps/chosen": -346.0693054199219,
      "logps/rejected": -289.0499267578125,
      "loss": 0.6648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1828932762145996,
      "rewards/margins": 0.7986312508583069,
      "rewards/rejected": -1.9815245866775513,
      "step": 14079
    },
    {
      "epoch": 1.8424634006706468,
      "grad_norm": 12.135344505310059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4175720810890198,
      "logits/rejected": -0.23784202337265015,
      "logps/chosen": -451.9237976074219,
      "logps/rejected": -512.450927734375,
      "loss": 0.7015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1833860874176025,
      "rewards/margins": 0.5361337661743164,
      "rewards/rejected": -1.719519853591919,
      "step": 14080
    },
    {
      "epoch": 1.8425942586079986,
      "grad_norm": 8.268342018127441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47212275862693787,
      "logits/rejected": -0.43490660190582275,
      "logps/chosen": -448.9241943359375,
      "logps/rejected": -487.360595703125,
      "loss": 0.402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2790616750717163,
      "rewards/margins": 1.683109164237976,
      "rewards/rejected": -2.9621708393096924,
      "step": 14081
    },
    {
      "epoch": 1.8427251165453504,
      "grad_norm": 6.661996841430664,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02754707634449005,
      "logits/rejected": -0.26802539825439453,
      "logps/chosen": -330.6086120605469,
      "logps/rejected": -369.49639892578125,
      "loss": 0.3173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5577986240386963,
      "rewards/margins": 1.8254178762435913,
      "rewards/rejected": -3.383216619491577,
      "step": 14082
    },
    {
      "epoch": 1.8428559744827022,
      "grad_norm": 7.758239269256592,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15999308228492737,
      "logits/rejected": -0.3045858144760132,
      "logps/chosen": -420.88323974609375,
      "logps/rejected": -391.46087646484375,
      "loss": 0.4867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2304433584213257,
      "rewards/margins": 1.0549566745758057,
      "rewards/rejected": -2.285400152206421,
      "step": 14083
    },
    {
      "epoch": 1.842986832420054,
      "grad_norm": 11.105100631713867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3982597291469574,
      "logits/rejected": -0.300972044467926,
      "logps/chosen": -633.7486572265625,
      "logps/rejected": -620.5375366210938,
      "loss": 0.6392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1167432069778442,
      "rewards/margins": 1.0342066287994385,
      "rewards/rejected": -2.1509499549865723,
      "step": 14084
    },
    {
      "epoch": 1.8431176903574058,
      "grad_norm": 11.69038200378418,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08932358026504517,
      "logits/rejected": -0.08788764476776123,
      "logps/chosen": -507.0285949707031,
      "logps/rejected": -497.96392822265625,
      "loss": 0.6111,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4144648313522339,
      "rewards/margins": 1.1846609115600586,
      "rewards/rejected": -2.599125862121582,
      "step": 14085
    },
    {
      "epoch": 1.8432485482947576,
      "grad_norm": 10.012991905212402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5491343140602112,
      "logits/rejected": -0.3839091658592224,
      "logps/chosen": -457.7334289550781,
      "logps/rejected": -444.707275390625,
      "loss": 0.4314,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6413458585739136,
      "rewards/margins": 1.3671371936798096,
      "rewards/rejected": -3.0084829330444336,
      "step": 14086
    },
    {
      "epoch": 1.8433794062321094,
      "grad_norm": 7.712673664093018,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2166871428489685,
      "logits/rejected": -0.39905282855033875,
      "logps/chosen": -510.8705749511719,
      "logps/rejected": -793.3348388671875,
      "loss": 0.2756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6696075201034546,
      "rewards/margins": 2.9810259342193604,
      "rewards/rejected": -3.6506335735321045,
      "step": 14087
    },
    {
      "epoch": 1.8435102641694612,
      "grad_norm": 14.422738075256348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33535823225975037,
      "logits/rejected": -0.4051407277584076,
      "logps/chosen": -494.42572021484375,
      "logps/rejected": -354.11383056640625,
      "loss": 0.9871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7300580739974976,
      "rewards/margins": 0.7687443494796753,
      "rewards/rejected": -2.4988021850585938,
      "step": 14088
    },
    {
      "epoch": 1.8436411221068127,
      "grad_norm": 3.7315142154693604,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38742101192474365,
      "logits/rejected": -0.2551160454750061,
      "logps/chosen": -476.841552734375,
      "logps/rejected": -515.8719482421875,
      "loss": 0.3034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9593795537948608,
      "rewards/margins": 1.4389872550964355,
      "rewards/rejected": -2.398366928100586,
      "step": 14089
    },
    {
      "epoch": 1.8437719800441645,
      "grad_norm": 8.82727336883545,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32963740825653076,
      "logits/rejected": -0.27306315302848816,
      "logps/chosen": -552.1401977539062,
      "logps/rejected": -506.4227600097656,
      "loss": 0.4414,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1677532196044922,
      "rewards/margins": 1.2546865940093994,
      "rewards/rejected": -2.4224395751953125,
      "step": 14090
    },
    {
      "epoch": 1.8439028379815163,
      "grad_norm": 11.366005897521973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32729750871658325,
      "logits/rejected": -0.2203584760427475,
      "logps/chosen": -238.131103515625,
      "logps/rejected": -429.7453308105469,
      "loss": 0.383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7856183648109436,
      "rewards/margins": 1.7428028583526611,
      "rewards/rejected": -2.52842116355896,
      "step": 14091
    },
    {
      "epoch": 1.8440336959188681,
      "grad_norm": 17.917322158813477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4964006245136261,
      "logits/rejected": -0.27499857544898987,
      "logps/chosen": -382.4438171386719,
      "logps/rejected": -330.5734558105469,
      "loss": 0.8698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.661419153213501,
      "rewards/margins": 0.31630751490592957,
      "rewards/rejected": -0.9777267575263977,
      "step": 14092
    },
    {
      "epoch": 1.8441645538562197,
      "grad_norm": 7.190001964569092,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40514594316482544,
      "logits/rejected": -0.3273860216140747,
      "logps/chosen": -384.8534240722656,
      "logps/rejected": -390.9127197265625,
      "loss": 0.4061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2801854610443115,
      "rewards/margins": 1.0449644327163696,
      "rewards/rejected": -2.3251497745513916,
      "step": 14093
    },
    {
      "epoch": 1.8442954117935715,
      "grad_norm": 12.775097846984863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27445274591445923,
      "logits/rejected": -0.380666047334671,
      "logps/chosen": -646.3093872070312,
      "logps/rejected": -527.8593139648438,
      "loss": 0.674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2389951944351196,
      "rewards/margins": 1.8477821350097656,
      "rewards/rejected": -3.086777448654175,
      "step": 14094
    },
    {
      "epoch": 1.8444262697309233,
      "grad_norm": 11.586065292358398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30344605445861816,
      "logits/rejected": -0.343181848526001,
      "logps/chosen": -513.9481201171875,
      "logps/rejected": -335.2761535644531,
      "loss": 0.5888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6976847648620605,
      "rewards/margins": 1.3251513242721558,
      "rewards/rejected": -3.022836208343506,
      "step": 14095
    },
    {
      "epoch": 1.844557127668275,
      "grad_norm": 12.7439546585083,
      "learning_rate": 5e-06,
      "logits/chosen": -0.516430139541626,
      "logits/rejected": -0.4628307819366455,
      "logps/chosen": -523.3067626953125,
      "logps/rejected": -549.3790283203125,
      "loss": 0.5724,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4280290603637695,
      "rewards/margins": 1.9170722961425781,
      "rewards/rejected": -3.3451013565063477,
      "step": 14096
    },
    {
      "epoch": 1.8446879856056269,
      "grad_norm": 8.41623306274414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3261556625366211,
      "logits/rejected": -0.490975022315979,
      "logps/chosen": -281.34735107421875,
      "logps/rejected": -343.0412902832031,
      "loss": 0.3975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3346831798553467,
      "rewards/margins": 2.111945867538452,
      "rewards/rejected": -3.446629047393799,
      "step": 14097
    },
    {
      "epoch": 1.8448188435429786,
      "grad_norm": 8.51027774810791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3796296715736389,
      "logits/rejected": -0.33739277720451355,
      "logps/chosen": -500.908935546875,
      "logps/rejected": -380.31463623046875,
      "loss": 0.4358,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5194343328475952,
      "rewards/margins": 1.0831462144851685,
      "rewards/rejected": -1.6025804281234741,
      "step": 14098
    },
    {
      "epoch": 1.8449497014803304,
      "grad_norm": 6.279449462890625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3528327941894531,
      "logits/rejected": -0.34561991691589355,
      "logps/chosen": -394.0509033203125,
      "logps/rejected": -346.3526611328125,
      "loss": 0.3267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7321557998657227,
      "rewards/margins": 1.6909105777740479,
      "rewards/rejected": -2.4230661392211914,
      "step": 14099
    },
    {
      "epoch": 1.8450805594176822,
      "grad_norm": 6.855419635772705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.622361421585083,
      "logits/rejected": -0.5093213319778442,
      "logps/chosen": -526.222412109375,
      "logps/rejected": -420.375732421875,
      "loss": 0.2335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7169268131256104,
      "rewards/margins": 2.2605347633361816,
      "rewards/rejected": -2.977461576461792,
      "step": 14100
    },
    {
      "epoch": 1.845211417355034,
      "grad_norm": 11.728325843811035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32635724544525146,
      "logits/rejected": -0.4058331847190857,
      "logps/chosen": -403.8009338378906,
      "logps/rejected": -360.1260070800781,
      "loss": 0.5707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2970901727676392,
      "rewards/margins": 0.6959589719772339,
      "rewards/rejected": -1.9930492639541626,
      "step": 14101
    },
    {
      "epoch": 1.8453422752923858,
      "grad_norm": 2.9997899532318115,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28497734665870667,
      "logits/rejected": -0.2023548036813736,
      "logps/chosen": -370.8693542480469,
      "logps/rejected": -327.6727294921875,
      "loss": 0.1219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3506778478622437,
      "rewards/margins": 2.8283047676086426,
      "rewards/rejected": -4.178982734680176,
      "step": 14102
    },
    {
      "epoch": 1.8454731332297376,
      "grad_norm": 6.7016987800598145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34112945199012756,
      "logits/rejected": -0.40780285000801086,
      "logps/chosen": -338.5582580566406,
      "logps/rejected": -356.082763671875,
      "loss": 0.2535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.250186562538147,
      "rewards/margins": 1.977238416671753,
      "rewards/rejected": -3.2274248600006104,
      "step": 14103
    },
    {
      "epoch": 1.8456039911670892,
      "grad_norm": 6.237057209014893,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01546054519712925,
      "logits/rejected": -0.38167324662208557,
      "logps/chosen": -623.7476806640625,
      "logps/rejected": -503.79644775390625,
      "loss": 0.21,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1055667400360107,
      "rewards/margins": 2.7726433277130127,
      "rewards/rejected": -3.8782100677490234,
      "step": 14104
    },
    {
      "epoch": 1.845734849104441,
      "grad_norm": 8.827248573303223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3104724586009979,
      "logits/rejected": -0.19119063019752502,
      "logps/chosen": -415.70623779296875,
      "logps/rejected": -332.000732421875,
      "loss": 0.3878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0536972284317017,
      "rewards/margins": 1.2220958471298218,
      "rewards/rejected": -2.2757930755615234,
      "step": 14105
    },
    {
      "epoch": 1.8458657070417928,
      "grad_norm": 10.803298950195312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08644793927669525,
      "logits/rejected": -0.015287891030311584,
      "logps/chosen": -500.8629150390625,
      "logps/rejected": -482.5615234375,
      "loss": 0.5758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7534522414207458,
      "rewards/margins": 1.3369929790496826,
      "rewards/rejected": -2.0904452800750732,
      "step": 14106
    },
    {
      "epoch": 1.8459965649791445,
      "grad_norm": 9.84963607788086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09585953503847122,
      "logits/rejected": 0.02077271044254303,
      "logps/chosen": -429.0762023925781,
      "logps/rejected": -361.9835205078125,
      "loss": 0.4636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5304529666900635,
      "rewards/margins": 1.493985891342163,
      "rewards/rejected": -2.0244386196136475,
      "step": 14107
    },
    {
      "epoch": 1.8461274229164961,
      "grad_norm": 7.0805253982543945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11993643641471863,
      "logits/rejected": -0.16175271570682526,
      "logps/chosen": -767.3216552734375,
      "logps/rejected": -422.5481262207031,
      "loss": 0.2507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7542861700057983,
      "rewards/margins": 2.055455446243286,
      "rewards/rejected": -2.809741735458374,
      "step": 14108
    },
    {
      "epoch": 1.846258280853848,
      "grad_norm": 7.6198201179504395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20630250871181488,
      "logits/rejected": -0.2042355090379715,
      "logps/chosen": -407.4725646972656,
      "logps/rejected": -475.9197082519531,
      "loss": 0.4738,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5196501016616821,
      "rewards/margins": 0.9793035984039307,
      "rewards/rejected": -2.4989538192749023,
      "step": 14109
    },
    {
      "epoch": 1.8463891387911997,
      "grad_norm": 12.851293563842773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19557809829711914,
      "logits/rejected": -0.19844302535057068,
      "logps/chosen": -526.33349609375,
      "logps/rejected": -471.960205078125,
      "loss": 0.6772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8630353212356567,
      "rewards/margins": 1.2802448272705078,
      "rewards/rejected": -2.143280267715454,
      "step": 14110
    },
    {
      "epoch": 1.8465199967285515,
      "grad_norm": 7.921909809112549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36877521872520447,
      "logits/rejected": -0.30595654249191284,
      "logps/chosen": -398.0911560058594,
      "logps/rejected": -433.02716064453125,
      "loss": 0.457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.702954888343811,
      "rewards/margins": 1.1329339742660522,
      "rewards/rejected": -2.8358891010284424,
      "step": 14111
    },
    {
      "epoch": 1.8466508546659033,
      "grad_norm": 11.342230796813965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23111745715141296,
      "logits/rejected": -0.3194432854652405,
      "logps/chosen": -435.44757080078125,
      "logps/rejected": -361.93829345703125,
      "loss": 0.7813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1677764654159546,
      "rewards/margins": 0.5213606357574463,
      "rewards/rejected": -1.6891371011734009,
      "step": 14112
    },
    {
      "epoch": 1.846781712603255,
      "grad_norm": 13.384904861450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45879003405570984,
      "logits/rejected": -0.4627123177051544,
      "logps/chosen": -290.45654296875,
      "logps/rejected": -438.57220458984375,
      "loss": 0.7182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1985273361206055,
      "rewards/margins": 0.7017914056777954,
      "rewards/rejected": -1.9003188610076904,
      "step": 14113
    },
    {
      "epoch": 1.8469125705406069,
      "grad_norm": 5.577004432678223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1573311686515808,
      "logits/rejected": -0.24301449954509735,
      "logps/chosen": -402.0017395019531,
      "logps/rejected": -427.9331970214844,
      "loss": 0.3112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6367229223251343,
      "rewards/margins": 1.2445852756500244,
      "rewards/rejected": -1.8813083171844482,
      "step": 14114
    },
    {
      "epoch": 1.8470434284779587,
      "grad_norm": 7.392096042633057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4369674324989319,
      "logits/rejected": -0.3816973567008972,
      "logps/chosen": -479.4990234375,
      "logps/rejected": -607.2850952148438,
      "loss": 0.2902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0856552124023438,
      "rewards/margins": 1.707511067390442,
      "rewards/rejected": -2.7931666374206543,
      "step": 14115
    },
    {
      "epoch": 1.8471742864153105,
      "grad_norm": 9.701972961425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3272683620452881,
      "logits/rejected": -0.5471162796020508,
      "logps/chosen": -420.01055908203125,
      "logps/rejected": -380.1690368652344,
      "loss": 0.4998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9473892450332642,
      "rewards/margins": 1.0589691400527954,
      "rewards/rejected": -2.0063583850860596,
      "step": 14116
    },
    {
      "epoch": 1.8473051443526622,
      "grad_norm": 5.995124340057373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6576715111732483,
      "logits/rejected": -0.7321621775627136,
      "logps/chosen": -483.95166015625,
      "logps/rejected": -509.5292053222656,
      "loss": 0.3019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4212232530117035,
      "rewards/margins": 2.1820058822631836,
      "rewards/rejected": -2.60322904586792,
      "step": 14117
    },
    {
      "epoch": 1.847436002290014,
      "grad_norm": 9.585755348205566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48723024129867554,
      "logits/rejected": -0.6301475763320923,
      "logps/chosen": -575.7589111328125,
      "logps/rejected": -653.049560546875,
      "loss": 0.5384,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8181421160697937,
      "rewards/margins": 1.1942167282104492,
      "rewards/rejected": -2.0123589038848877,
      "step": 14118
    },
    {
      "epoch": 1.8475668602273658,
      "grad_norm": 9.36400032043457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1528051793575287,
      "logits/rejected": -0.4174523949623108,
      "logps/chosen": -355.92474365234375,
      "logps/rejected": -473.29473876953125,
      "loss": 0.4175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5459599494934082,
      "rewards/margins": 1.7546730041503906,
      "rewards/rejected": -3.300632953643799,
      "step": 14119
    },
    {
      "epoch": 1.8476977181647174,
      "grad_norm": 8.500242233276367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3693777322769165,
      "logits/rejected": -0.5273520350456238,
      "logps/chosen": -501.06640625,
      "logps/rejected": -380.6286315917969,
      "loss": 0.3354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8517144918441772,
      "rewards/margins": 1.2762902975082397,
      "rewards/rejected": -3.128004789352417,
      "step": 14120
    },
    {
      "epoch": 1.8478285761020692,
      "grad_norm": 6.382613658905029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052360761910676956,
      "logits/rejected": -0.020687513053417206,
      "logps/chosen": -346.35931396484375,
      "logps/rejected": -501.38720703125,
      "loss": 0.36,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1110867261886597,
      "rewards/margins": 1.308658242225647,
      "rewards/rejected": -2.4197449684143066,
      "step": 14121
    },
    {
      "epoch": 1.847959434039421,
      "grad_norm": 11.382740020751953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2349894642829895,
      "logits/rejected": -0.3174973726272583,
      "logps/chosen": -546.1417236328125,
      "logps/rejected": -456.4861755371094,
      "loss": 0.3262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6750970482826233,
      "rewards/margins": 2.6185755729675293,
      "rewards/rejected": -3.293672561645508,
      "step": 14122
    },
    {
      "epoch": 1.8480902919767728,
      "grad_norm": 8.203699111938477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1386251449584961,
      "logits/rejected": -0.29502761363983154,
      "logps/chosen": -567.29443359375,
      "logps/rejected": -540.8681030273438,
      "loss": 0.3002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8217312097549438,
      "rewards/margins": 2.011660099029541,
      "rewards/rejected": -2.8333911895751953,
      "step": 14123
    },
    {
      "epoch": 1.8482211499141243,
      "grad_norm": 7.06229305267334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17822439968585968,
      "logits/rejected": -0.34282416105270386,
      "logps/chosen": -438.1839294433594,
      "logps/rejected": -525.1876220703125,
      "loss": 0.2771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15008889138698578,
      "rewards/margins": 2.0531132221221924,
      "rewards/rejected": -2.20320200920105,
      "step": 14124
    },
    {
      "epoch": 1.8483520078514761,
      "grad_norm": 7.015843391418457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25413793325424194,
      "logits/rejected": -0.23140235245227814,
      "logps/chosen": -637.34765625,
      "logps/rejected": -460.63836669921875,
      "loss": 0.2737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10722596198320389,
      "rewards/margins": 2.4098119735717773,
      "rewards/rejected": -2.302586317062378,
      "step": 14125
    },
    {
      "epoch": 1.848482865788828,
      "grad_norm": 5.905178546905518,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3209100365638733,
      "logits/rejected": -0.3398510813713074,
      "logps/chosen": -442.465087890625,
      "logps/rejected": -407.2903137207031,
      "loss": 0.2104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0560389757156372,
      "rewards/margins": 1.6006306409835815,
      "rewards/rejected": -2.6566696166992188,
      "step": 14126
    },
    {
      "epoch": 1.8486137237261797,
      "grad_norm": 23.41537857055664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3888806998729706,
      "logits/rejected": -0.36441415548324585,
      "logps/chosen": -558.7388305664062,
      "logps/rejected": -436.79718017578125,
      "loss": 1.1799,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.0257861614227295,
      "rewards/margins": -0.08966231346130371,
      "rewards/rejected": -1.9361238479614258,
      "step": 14127
    },
    {
      "epoch": 1.8487445816635315,
      "grad_norm": 11.932879447937012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1695873737335205,
      "logits/rejected": -0.187453031539917,
      "logps/chosen": -565.9733276367188,
      "logps/rejected": -578.5728149414062,
      "loss": 0.8326,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6201785802841187,
      "rewards/margins": 0.9255765080451965,
      "rewards/rejected": -1.5457550287246704,
      "step": 14128
    },
    {
      "epoch": 1.8488754396008833,
      "grad_norm": 9.99384593963623,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2297687828540802,
      "logits/rejected": -0.46127790212631226,
      "logps/chosen": -417.9469299316406,
      "logps/rejected": -612.1087646484375,
      "loss": 0.431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.199784755706787,
      "rewards/margins": 1.4045385122299194,
      "rewards/rejected": -2.604323148727417,
      "step": 14129
    },
    {
      "epoch": 1.849006297538235,
      "grad_norm": 7.848666667938232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17553164064884186,
      "logits/rejected": -0.3426835834980011,
      "logps/chosen": -424.25531005859375,
      "logps/rejected": -433.13128662109375,
      "loss": 0.309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1162142753601074,
      "rewards/margins": 1.7224966287612915,
      "rewards/rejected": -2.8387107849121094,
      "step": 14130
    },
    {
      "epoch": 1.8491371554755869,
      "grad_norm": 13.558114051818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12300244718790054,
      "logits/rejected": -0.3343183100223541,
      "logps/chosen": -402.48773193359375,
      "logps/rejected": -428.5411376953125,
      "loss": 0.8018,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9120888710021973,
      "rewards/margins": 0.5851808190345764,
      "rewards/rejected": -2.497269630432129,
      "step": 14131
    },
    {
      "epoch": 1.8492680134129387,
      "grad_norm": 13.20540714263916,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4413110613822937,
      "logits/rejected": -0.48457032442092896,
      "logps/chosen": -397.99285888671875,
      "logps/rejected": -386.74554443359375,
      "loss": 0.9837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8013572692871094,
      "rewards/margins": -0.15528950095176697,
      "rewards/rejected": -1.6460678577423096,
      "step": 14132
    },
    {
      "epoch": 1.8493988713502905,
      "grad_norm": 10.377815246582031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3651069104671478,
      "logits/rejected": -0.3685632646083832,
      "logps/chosen": -508.80706787109375,
      "logps/rejected": -409.9600830078125,
      "loss": 0.4554,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7963021993637085,
      "rewards/margins": 1.2882041931152344,
      "rewards/rejected": -3.0845067501068115,
      "step": 14133
    },
    {
      "epoch": 1.8495297292876423,
      "grad_norm": 9.706507682800293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6134739518165588,
      "logits/rejected": -0.3949489891529083,
      "logps/chosen": -376.4666442871094,
      "logps/rejected": -593.2108764648438,
      "loss": 0.3976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.981619119644165,
      "rewards/margins": 1.36714768409729,
      "rewards/rejected": -3.348766565322876,
      "step": 14134
    },
    {
      "epoch": 1.8496605872249938,
      "grad_norm": 4.558936595916748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3215844929218292,
      "logits/rejected": -0.18494971096515656,
      "logps/chosen": -359.2563781738281,
      "logps/rejected": -367.9787902832031,
      "loss": 0.2079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6578462719917297,
      "rewards/margins": 2.1857259273529053,
      "rewards/rejected": -2.843571901321411,
      "step": 14135
    },
    {
      "epoch": 1.8497914451623456,
      "grad_norm": 7.751189231872559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5323447585105896,
      "logits/rejected": -0.7015228867530823,
      "logps/chosen": -390.6335144042969,
      "logps/rejected": -462.63555908203125,
      "loss": 0.2499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5294065475463867,
      "rewards/margins": 2.4364633560180664,
      "rewards/rejected": -3.965869903564453,
      "step": 14136
    },
    {
      "epoch": 1.8499223030996974,
      "grad_norm": 9.05870532989502,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18657587468624115,
      "logits/rejected": -0.5568583011627197,
      "logps/chosen": -677.7174072265625,
      "logps/rejected": -904.7661743164062,
      "loss": 0.2024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7826083898544312,
      "rewards/margins": 2.651691436767578,
      "rewards/rejected": -3.434300184249878,
      "step": 14137
    },
    {
      "epoch": 1.8500531610370492,
      "grad_norm": 13.29404067993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3473619818687439,
      "logits/rejected": -0.40188488364219666,
      "logps/chosen": -489.7486267089844,
      "logps/rejected": -481.0855407714844,
      "loss": 0.7227,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.9059066772460938,
      "rewards/margins": 0.7619994282722473,
      "rewards/rejected": -2.6679060459136963,
      "step": 14138
    },
    {
      "epoch": 1.8501840189744008,
      "grad_norm": 9.487062454223633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33289146423339844,
      "logits/rejected": -0.5236599445343018,
      "logps/chosen": -488.7829284667969,
      "logps/rejected": -375.79034423828125,
      "loss": 0.5871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.081143856048584,
      "rewards/margins": 0.5106656551361084,
      "rewards/rejected": -1.5918093919754028,
      "step": 14139
    },
    {
      "epoch": 1.8503148769117526,
      "grad_norm": 11.21608829498291,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02894887700676918,
      "logits/rejected": -0.18914885818958282,
      "logps/chosen": -404.67291259765625,
      "logps/rejected": -420.7521057128906,
      "loss": 0.5833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6887619495391846,
      "rewards/margins": 0.8232292532920837,
      "rewards/rejected": -2.511991024017334,
      "step": 14140
    },
    {
      "epoch": 1.8504457348491044,
      "grad_norm": 10.123595237731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49141111969947815,
      "logits/rejected": -0.5512845516204834,
      "logps/chosen": -545.7130126953125,
      "logps/rejected": -534.7943115234375,
      "loss": 0.3674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8951599597930908,
      "rewards/margins": 2.1670684814453125,
      "rewards/rejected": -3.0622284412384033,
      "step": 14141
    },
    {
      "epoch": 1.8505765927864561,
      "grad_norm": 13.465972900390625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7342984080314636,
      "logits/rejected": -0.39103424549102783,
      "logps/chosen": -438.5810852050781,
      "logps/rejected": -487.3664245605469,
      "loss": 0.4944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5693068504333496,
      "rewards/margins": 1.530074119567871,
      "rewards/rejected": -3.0993809700012207,
      "step": 14142
    },
    {
      "epoch": 1.850707450723808,
      "grad_norm": 9.920503616333008,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0007826834917068481,
      "logits/rejected": -0.18432019650936127,
      "logps/chosen": -542.365966796875,
      "logps/rejected": -509.41748046875,
      "loss": 0.5197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0683075189590454,
      "rewards/margins": 0.6726794242858887,
      "rewards/rejected": -1.7409868240356445,
      "step": 14143
    },
    {
      "epoch": 1.8508383086611597,
      "grad_norm": 8.882534980773926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5741177797317505,
      "logits/rejected": -0.46005183458328247,
      "logps/chosen": -442.0191650390625,
      "logps/rejected": -524.4824829101562,
      "loss": 0.4176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7774273157119751,
      "rewards/margins": 1.017006278038025,
      "rewards/rejected": -1.7944334745407104,
      "step": 14144
    },
    {
      "epoch": 1.8509691665985115,
      "grad_norm": 7.753755569458008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32247525453567505,
      "logits/rejected": -0.3519635796546936,
      "logps/chosen": -507.9651794433594,
      "logps/rejected": -450.36700439453125,
      "loss": 0.2384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8744731545448303,
      "rewards/margins": 1.841378092765808,
      "rewards/rejected": -2.715851068496704,
      "step": 14145
    },
    {
      "epoch": 1.8511000245358633,
      "grad_norm": 7.77501916885376,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1699381321668625,
      "logits/rejected": -0.35898828506469727,
      "logps/chosen": -607.1226196289062,
      "logps/rejected": -516.1559448242188,
      "loss": 0.2813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9130833148956299,
      "rewards/margins": 1.9175485372543335,
      "rewards/rejected": -2.8306314945220947,
      "step": 14146
    },
    {
      "epoch": 1.851230882473215,
      "grad_norm": 12.280611038208008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4156113266944885,
      "logits/rejected": -0.2945166826248169,
      "logps/chosen": -340.866455078125,
      "logps/rejected": -471.4993896484375,
      "loss": 0.6839,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1378464698791504,
      "rewards/margins": 0.2627474069595337,
      "rewards/rejected": -1.4005939960479736,
      "step": 14147
    },
    {
      "epoch": 1.851361740410567,
      "grad_norm": 23.772794723510742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2224033623933792,
      "logits/rejected": -0.29338276386260986,
      "logps/chosen": -429.896484375,
      "logps/rejected": -391.1174011230469,
      "loss": 1.4571,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.354893207550049,
      "rewards/margins": 0.12144407629966736,
      "rewards/rejected": -3.476337432861328,
      "step": 14148
    },
    {
      "epoch": 1.8514925983479187,
      "grad_norm": 10.975776672363281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4624265134334564,
      "logits/rejected": -0.5564247369766235,
      "logps/chosen": -402.4132080078125,
      "logps/rejected": -334.3285827636719,
      "loss": 0.5726,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.883856177330017,
      "rewards/margins": 0.5599824786186218,
      "rewards/rejected": -2.443838596343994,
      "step": 14149
    },
    {
      "epoch": 1.8516234562852703,
      "grad_norm": 11.62687873840332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18456436693668365,
      "logits/rejected": -0.2618919909000397,
      "logps/chosen": -463.1280822753906,
      "logps/rejected": -638.9121704101562,
      "loss": 0.4108,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0772897005081177,
      "rewards/margins": 2.045997142791748,
      "rewards/rejected": -3.123286485671997,
      "step": 14150
    },
    {
      "epoch": 1.851754314222622,
      "grad_norm": 5.2846598625183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2691756784915924,
      "logits/rejected": -0.27391862869262695,
      "logps/chosen": -476.5982360839844,
      "logps/rejected": -460.95867919921875,
      "loss": 0.2575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9603094458580017,
      "rewards/margins": 1.5669138431549072,
      "rewards/rejected": -2.5272233486175537,
      "step": 14151
    },
    {
      "epoch": 1.8518851721599738,
      "grad_norm": 4.877342224121094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47541147470474243,
      "logits/rejected": -0.5396785736083984,
      "logps/chosen": -474.789306640625,
      "logps/rejected": -488.78472900390625,
      "loss": 0.1628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6123016476631165,
      "rewards/margins": 2.5580010414123535,
      "rewards/rejected": -3.170302391052246,
      "step": 14152
    },
    {
      "epoch": 1.8520160300973256,
      "grad_norm": 7.39076042175293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47585636377334595,
      "logits/rejected": -0.6206755638122559,
      "logps/chosen": -417.21295166015625,
      "logps/rejected": -538.5927124023438,
      "loss": 0.3206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9966022372245789,
      "rewards/margins": 1.5292611122131348,
      "rewards/rejected": -2.5258634090423584,
      "step": 14153
    },
    {
      "epoch": 1.8521468880346772,
      "grad_norm": 9.192208290100098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43253982067108154,
      "logits/rejected": -0.3023005425930023,
      "logps/chosen": -522.54443359375,
      "logps/rejected": -642.4150390625,
      "loss": 0.3467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.502125859260559,
      "rewards/margins": 1.5096235275268555,
      "rewards/rejected": -3.011749505996704,
      "step": 14154
    },
    {
      "epoch": 1.852277745972029,
      "grad_norm": 7.91402006149292,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2714230418205261,
      "logits/rejected": -0.5098168849945068,
      "logps/chosen": -393.5277099609375,
      "logps/rejected": -500.29888916015625,
      "loss": 0.3991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.49765896797180176,
      "rewards/margins": 2.607365369796753,
      "rewards/rejected": -3.1050240993499756,
      "step": 14155
    },
    {
      "epoch": 1.8524086039093808,
      "grad_norm": 13.622076988220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2976744472980499,
      "logits/rejected": -0.2669925093650818,
      "logps/chosen": -411.50469970703125,
      "logps/rejected": -371.5550537109375,
      "loss": 0.6325,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.933955430984497,
      "rewards/margins": 0.8403452634811401,
      "rewards/rejected": -2.7743005752563477,
      "step": 14156
    },
    {
      "epoch": 1.8525394618467326,
      "grad_norm": 6.384225368499756,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13677199184894562,
      "logits/rejected": -0.09318539500236511,
      "logps/chosen": -432.75958251953125,
      "logps/rejected": -441.85040283203125,
      "loss": 0.3047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6169527173042297,
      "rewards/margins": 1.3167083263397217,
      "rewards/rejected": -1.9336609840393066,
      "step": 14157
    },
    {
      "epoch": 1.8526703197840844,
      "grad_norm": 5.869616985321045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19132819771766663,
      "logits/rejected": -0.1863182932138443,
      "logps/chosen": -531.3635864257812,
      "logps/rejected": -404.0411682128906,
      "loss": 0.282,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.135424017906189,
      "rewards/margins": 2.0982587337493896,
      "rewards/rejected": -3.233682870864868,
      "step": 14158
    },
    {
      "epoch": 1.8528011777214362,
      "grad_norm": 16.30185317993164,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06169019639492035,
      "logits/rejected": 0.1130552738904953,
      "logps/chosen": -638.150146484375,
      "logps/rejected": -545.11474609375,
      "loss": 0.6264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9701449871063232,
      "rewards/margins": 0.862006425857544,
      "rewards/rejected": -3.8321516513824463,
      "step": 14159
    },
    {
      "epoch": 1.852932035658788,
      "grad_norm": 9.589756965637207,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03477667272090912,
      "logits/rejected": 0.06732302159070969,
      "logps/chosen": -476.3083190917969,
      "logps/rejected": -570.6787719726562,
      "loss": 0.4077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8537637591362,
      "rewards/margins": 1.1447595357894897,
      "rewards/rejected": -1.998523235321045,
      "step": 14160
    },
    {
      "epoch": 1.8530628935961397,
      "grad_norm": 15.277098655700684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2973811626434326,
      "logits/rejected": -0.20804427564144135,
      "logps/chosen": -364.3385925292969,
      "logps/rejected": -382.47698974609375,
      "loss": 1.0681,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6341274976730347,
      "rewards/margins": 0.542912483215332,
      "rewards/rejected": -2.1770403385162354,
      "step": 14161
    },
    {
      "epoch": 1.8531937515334915,
      "grad_norm": 10.082307815551758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39059990644454956,
      "logits/rejected": -0.3940064311027527,
      "logps/chosen": -506.61517333984375,
      "logps/rejected": -350.4554443359375,
      "loss": 0.553,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0865154266357422,
      "rewards/margins": 1.2941703796386719,
      "rewards/rejected": -2.380685806274414,
      "step": 14162
    },
    {
      "epoch": 1.8533246094708433,
      "grad_norm": 4.608775615692139,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3005702793598175,
      "logits/rejected": -0.6057443022727966,
      "logps/chosen": -321.9134216308594,
      "logps/rejected": -319.2212219238281,
      "loss": 0.2826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0810036659240723,
      "rewards/margins": 2.3766303062438965,
      "rewards/rejected": -3.4576337337493896,
      "step": 14163
    },
    {
      "epoch": 1.8534554674081951,
      "grad_norm": 6.063028812408447,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2577337622642517,
      "logits/rejected": -0.27666550874710083,
      "logps/chosen": -554.5032348632812,
      "logps/rejected": -485.28570556640625,
      "loss": 0.2608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5506902933120728,
      "rewards/margins": 1.6593177318572998,
      "rewards/rejected": -3.210008144378662,
      "step": 14164
    },
    {
      "epoch": 1.853586325345547,
      "grad_norm": 10.468206405639648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1304618865251541,
      "logits/rejected": -0.19072692096233368,
      "logps/chosen": -505.4561767578125,
      "logps/rejected": -564.4649047851562,
      "loss": 0.5123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9733472466468811,
      "rewards/margins": 1.2903451919555664,
      "rewards/rejected": -2.263692617416382,
      "step": 14165
    },
    {
      "epoch": 1.8537171832828985,
      "grad_norm": 8.860429763793945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5481326580047607,
      "logits/rejected": -0.8069782257080078,
      "logps/chosen": -462.2113952636719,
      "logps/rejected": -423.65460205078125,
      "loss": 0.3694,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0308712720870972,
      "rewards/margins": 1.3783479928970337,
      "rewards/rejected": -2.409219264984131,
      "step": 14166
    },
    {
      "epoch": 1.8538480412202503,
      "grad_norm": 8.113541603088379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2806602716445923,
      "logits/rejected": -0.33224815130233765,
      "logps/chosen": -419.4807434082031,
      "logps/rejected": -521.0655517578125,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1679188013076782,
      "rewards/margins": 1.4746999740600586,
      "rewards/rejected": -2.6426186561584473,
      "step": 14167
    },
    {
      "epoch": 1.853978899157602,
      "grad_norm": 6.96136999130249,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31717947125434875,
      "logits/rejected": -0.5009912848472595,
      "logps/chosen": -659.8812866210938,
      "logps/rejected": -429.171630859375,
      "loss": 0.4251,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7061612606048584,
      "rewards/margins": 1.1888798475265503,
      "rewards/rejected": -1.8950411081314087,
      "step": 14168
    },
    {
      "epoch": 1.8541097570949536,
      "grad_norm": 13.356742858886719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4239595830440521,
      "logits/rejected": -0.6070693731307983,
      "logps/chosen": -340.4858703613281,
      "logps/rejected": -359.08551025390625,
      "loss": 0.9772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2736798524856567,
      "rewards/margins": 0.6229153871536255,
      "rewards/rejected": -1.8965952396392822,
      "step": 14169
    },
    {
      "epoch": 1.8542406150323054,
      "grad_norm": 7.806358337402344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16531774401664734,
      "logits/rejected": -0.33165109157562256,
      "logps/chosen": -618.8844604492188,
      "logps/rejected": -608.2351684570312,
      "loss": 0.43,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6416529417037964,
      "rewards/margins": 0.9398092031478882,
      "rewards/rejected": -2.5814621448516846,
      "step": 14170
    },
    {
      "epoch": 1.8543714729696572,
      "grad_norm": 5.305861473083496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1376115083694458,
      "logits/rejected": -0.03758621960878372,
      "logps/chosen": -357.8728332519531,
      "logps/rejected": -503.28228759765625,
      "loss": 0.1929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0442849397659302,
      "rewards/margins": 2.5331623554229736,
      "rewards/rejected": -3.5774474143981934,
      "step": 14171
    },
    {
      "epoch": 1.854502330907009,
      "grad_norm": 3.8519792556762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6219847202301025,
      "logits/rejected": -0.49477076530456543,
      "logps/chosen": -378.8990783691406,
      "logps/rejected": -288.6164855957031,
      "loss": 0.2526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6995777487754822,
      "rewards/margins": 1.8757472038269043,
      "rewards/rejected": -2.5753250122070312,
      "step": 14172
    },
    {
      "epoch": 1.8546331888443608,
      "grad_norm": 9.914835929870605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09614966809749603,
      "logits/rejected": -0.46468377113342285,
      "logps/chosen": -521.8178100585938,
      "logps/rejected": -383.9246520996094,
      "loss": 0.4529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7858670949935913,
      "rewards/margins": 1.4345046281814575,
      "rewards/rejected": -3.2203714847564697,
      "step": 14173
    },
    {
      "epoch": 1.8547640467817126,
      "grad_norm": 5.637514591217041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2827649414539337,
      "logits/rejected": -0.37156790494918823,
      "logps/chosen": -235.82774353027344,
      "logps/rejected": -242.4835205078125,
      "loss": 0.2733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1602760553359985,
      "rewards/margins": 1.7394328117370605,
      "rewards/rejected": -2.8997089862823486,
      "step": 14174
    },
    {
      "epoch": 1.8548949047190644,
      "grad_norm": 12.331050872802734,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0036154529079794884,
      "logits/rejected": 0.14722903072834015,
      "logps/chosen": -400.8022155761719,
      "logps/rejected": -538.7947387695312,
      "loss": 0.5593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5136969089508057,
      "rewards/margins": 1.0838029384613037,
      "rewards/rejected": -2.5974998474121094,
      "step": 14175
    },
    {
      "epoch": 1.8550257626564162,
      "grad_norm": 6.381165981292725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17738032341003418,
      "logits/rejected": -0.24318821728229523,
      "logps/chosen": -551.7850952148438,
      "logps/rejected": -475.8328552246094,
      "loss": 0.2504,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8967405557632446,
      "rewards/margins": 1.6899640560150146,
      "rewards/rejected": -3.5867044925689697,
      "step": 14176
    },
    {
      "epoch": 1.855156620593768,
      "grad_norm": 6.719316005706787,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5337265729904175,
      "logits/rejected": -0.4349459707736969,
      "logps/chosen": -599.5438232421875,
      "logps/rejected": -569.7003173828125,
      "loss": 0.3065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.982239842414856,
      "rewards/margins": 1.60383141040802,
      "rewards/rejected": -2.586071252822876,
      "step": 14177
    },
    {
      "epoch": 1.8552874785311197,
      "grad_norm": 8.849539756774902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19606786966323853,
      "logits/rejected": -0.2919464111328125,
      "logps/chosen": -454.1176452636719,
      "logps/rejected": -588.0218505859375,
      "loss": 0.5107,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0495753288269043,
      "rewards/margins": 1.507164478302002,
      "rewards/rejected": -2.5567400455474854,
      "step": 14178
    },
    {
      "epoch": 1.8554183364684715,
      "grad_norm": 3.959507703781128,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5068526864051819,
      "logits/rejected": -0.39978229999542236,
      "logps/chosen": -469.8629150390625,
      "logps/rejected": -445.61199951171875,
      "loss": 0.1268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8476024270057678,
      "rewards/margins": 2.607724666595459,
      "rewards/rejected": -3.455327272415161,
      "step": 14179
    },
    {
      "epoch": 1.8555491944058233,
      "grad_norm": 5.254383087158203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.11428628116846085,
      "logits/rejected": -0.027777640148997307,
      "logps/chosen": -707.2491455078125,
      "logps/rejected": -626.6796875,
      "loss": 0.2342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09456334263086319,
      "rewards/margins": 1.6917680501937866,
      "rewards/rejected": -1.7863315343856812,
      "step": 14180
    },
    {
      "epoch": 1.855680052343175,
      "grad_norm": 15.087478637695312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2891994118690491,
      "logits/rejected": -0.5945296883583069,
      "logps/chosen": -444.91412353515625,
      "logps/rejected": -413.4271240234375,
      "loss": 0.7142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.60605788230896,
      "rewards/margins": 0.5358133316040039,
      "rewards/rejected": -3.141871213912964,
      "step": 14181
    },
    {
      "epoch": 1.8558109102805267,
      "grad_norm": 6.031747341156006,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4871952533721924,
      "logits/rejected": -0.6487936973571777,
      "logps/chosen": -465.1314697265625,
      "logps/rejected": -424.7225646972656,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5178009271621704,
      "rewards/margins": 1.8167905807495117,
      "rewards/rejected": -3.3345913887023926,
      "step": 14182
    },
    {
      "epoch": 1.8559417682178785,
      "grad_norm": 8.876751899719238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5485097169876099,
      "logits/rejected": -0.4435542821884155,
      "logps/chosen": -465.4606628417969,
      "logps/rejected": -488.251708984375,
      "loss": 0.4269,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6445376873016357,
      "rewards/margins": 2.6981985569000244,
      "rewards/rejected": -3.34273624420166,
      "step": 14183
    },
    {
      "epoch": 1.8560726261552303,
      "grad_norm": 7.169814586639404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2904774248600006,
      "logits/rejected": -0.2030838280916214,
      "logps/chosen": -525.7146606445312,
      "logps/rejected": -455.1624450683594,
      "loss": 0.2998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5939432382583618,
      "rewards/margins": 1.497727394104004,
      "rewards/rejected": -2.0916707515716553,
      "step": 14184
    },
    {
      "epoch": 1.8562034840925818,
      "grad_norm": 8.81181526184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14851760864257812,
      "logits/rejected": -0.06550423055887222,
      "logps/chosen": -546.0052490234375,
      "logps/rejected": -490.9720153808594,
      "loss": 0.4,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.325933814048767,
      "rewards/margins": 1.0139862298965454,
      "rewards/rejected": -2.3399202823638916,
      "step": 14185
    },
    {
      "epoch": 1.8563343420299336,
      "grad_norm": 7.4703240394592285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5203907489776611,
      "logits/rejected": -0.44904619455337524,
      "logps/chosen": -292.54864501953125,
      "logps/rejected": -408.76202392578125,
      "loss": 0.3181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0781731605529785,
      "rewards/margins": 1.618725061416626,
      "rewards/rejected": -3.6968979835510254,
      "step": 14186
    },
    {
      "epoch": 1.8564651999672854,
      "grad_norm": 8.043203353881836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1911875456571579,
      "logits/rejected": -0.18392649292945862,
      "logps/chosen": -587.4168701171875,
      "logps/rejected": -465.6625061035156,
      "loss": 0.4328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0642579793930054,
      "rewards/margins": 0.8809624910354614,
      "rewards/rejected": -1.9452204704284668,
      "step": 14187
    },
    {
      "epoch": 1.8565960579046372,
      "grad_norm": 12.447750091552734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09673565626144409,
      "logits/rejected": 0.1416187733411789,
      "logps/chosen": -471.0822448730469,
      "logps/rejected": -435.8428649902344,
      "loss": 0.7638,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.767471969127655,
      "rewards/margins": 0.5380350351333618,
      "rewards/rejected": -1.305506944656372,
      "step": 14188
    },
    {
      "epoch": 1.856726915841989,
      "grad_norm": 5.095576763153076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08590412884950638,
      "logits/rejected": -0.3494001626968384,
      "logps/chosen": -567.798828125,
      "logps/rejected": -494.69549560546875,
      "loss": 0.2158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3365482091903687,
      "rewards/margins": 1.954759955406189,
      "rewards/rejected": -3.2913081645965576,
      "step": 14189
    },
    {
      "epoch": 1.8568577737793408,
      "grad_norm": 12.29495906829834,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09323777258396149,
      "logits/rejected": -0.11276568472385406,
      "logps/chosen": -486.9594421386719,
      "logps/rejected": -439.0842590332031,
      "loss": 0.6214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.889472484588623,
      "rewards/margins": 1.029106855392456,
      "rewards/rejected": -2.918579339981079,
      "step": 14190
    },
    {
      "epoch": 1.8569886317166926,
      "grad_norm": 7.826006889343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34493786096572876,
      "logits/rejected": -0.15336734056472778,
      "logps/chosen": -306.11285400390625,
      "logps/rejected": -391.96209716796875,
      "loss": 0.3922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5213669538497925,
      "rewards/margins": 1.9315265417099,
      "rewards/rejected": -3.4528932571411133,
      "step": 14191
    },
    {
      "epoch": 1.8571194896540444,
      "grad_norm": 8.452045440673828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1915310025215149,
      "logits/rejected": -0.3834625780582428,
      "logps/chosen": -512.5950317382812,
      "logps/rejected": -642.6566772460938,
      "loss": 0.3691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.804051160812378,
      "rewards/margins": 1.7881581783294678,
      "rewards/rejected": -3.5922091007232666,
      "step": 14192
    },
    {
      "epoch": 1.8572503475913962,
      "grad_norm": 6.639313220977783,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2568194270133972,
      "logits/rejected": -0.14551961421966553,
      "logps/chosen": -341.408447265625,
      "logps/rejected": -417.3503112792969,
      "loss": 0.3709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4905034303665161,
      "rewards/margins": 1.1026897430419922,
      "rewards/rejected": -2.5931930541992188,
      "step": 14193
    },
    {
      "epoch": 1.857381205528748,
      "grad_norm": 17.3067684173584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32820817828178406,
      "logits/rejected": -0.43579694628715515,
      "logps/chosen": -737.1595458984375,
      "logps/rejected": -504.95831298828125,
      "loss": 0.4272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0715025663375854,
      "rewards/margins": 1.5357682704925537,
      "rewards/rejected": -2.6072707176208496,
      "step": 14194
    },
    {
      "epoch": 1.8575120634660998,
      "grad_norm": 6.024627208709717,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19337870180606842,
      "logits/rejected": -0.44659289717674255,
      "logps/chosen": -330.1142883300781,
      "logps/rejected": -396.7070617675781,
      "loss": 0.2772,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5461384057998657,
      "rewards/margins": 2.487063407897949,
      "rewards/rejected": -4.033202171325684,
      "step": 14195
    },
    {
      "epoch": 1.8576429214034513,
      "grad_norm": 15.065298080444336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5252729654312134,
      "logits/rejected": -0.36028969287872314,
      "logps/chosen": -399.1751403808594,
      "logps/rejected": -340.44464111328125,
      "loss": 0.6483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8779234886169434,
      "rewards/margins": 1.0116674900054932,
      "rewards/rejected": -2.8895909786224365,
      "step": 14196
    },
    {
      "epoch": 1.8577737793408031,
      "grad_norm": 9.275247573852539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7061071395874023,
      "logits/rejected": -0.7081387042999268,
      "logps/chosen": -487.59918212890625,
      "logps/rejected": -334.1173095703125,
      "loss": 0.3605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6181858777999878,
      "rewards/margins": 1.8134629726409912,
      "rewards/rejected": -3.4316489696502686,
      "step": 14197
    },
    {
      "epoch": 1.857904637278155,
      "grad_norm": 5.10898494720459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6015800833702087,
      "logits/rejected": -0.3801608681678772,
      "logps/chosen": -255.87623596191406,
      "logps/rejected": -330.7948913574219,
      "loss": 0.2971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1255117654800415,
      "rewards/margins": 2.0300800800323486,
      "rewards/rejected": -3.1555919647216797,
      "step": 14198
    },
    {
      "epoch": 1.8580354952155067,
      "grad_norm": 9.174274444580078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030567383393645287,
      "logits/rejected": -0.28789961338043213,
      "logps/chosen": -474.5630187988281,
      "logps/rejected": -479.248046875,
      "loss": 0.3517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.70917546749115,
      "rewards/margins": 1.9000933170318604,
      "rewards/rejected": -3.6092689037323,
      "step": 14199
    },
    {
      "epoch": 1.8581663531528583,
      "grad_norm": 13.136030197143555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2532629072666168,
      "logits/rejected": -0.6116798520088196,
      "logps/chosen": -531.4562377929688,
      "logps/rejected": -409.8321533203125,
      "loss": 0.7685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.120667815208435,
      "rewards/margins": 0.9364018440246582,
      "rewards/rejected": -2.057070016860962,
      "step": 14200
    },
    {
      "epoch": 1.85829721109021,
      "grad_norm": 12.263983726501465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26714617013931274,
      "logits/rejected": -0.19863413274288177,
      "logps/chosen": -499.7137756347656,
      "logps/rejected": -461.87359619140625,
      "loss": 0.4711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2381346225738525,
      "rewards/margins": 1.0567353963851929,
      "rewards/rejected": -2.294870138168335,
      "step": 14201
    },
    {
      "epoch": 1.8584280690275619,
      "grad_norm": 5.475272178649902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21686677634716034,
      "logits/rejected": -0.34224843978881836,
      "logps/chosen": -297.52276611328125,
      "logps/rejected": -302.4905090332031,
      "loss": 0.2349,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7187128067016602,
      "rewards/margins": 2.720991611480713,
      "rewards/rejected": -3.439704418182373,
      "step": 14202
    },
    {
      "epoch": 1.8585589269649136,
      "grad_norm": 15.243131637573242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2953767478466034,
      "logits/rejected": -0.35146987438201904,
      "logps/chosen": -508.05902099609375,
      "logps/rejected": -521.4824829101562,
      "loss": 0.9571,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.0550308227539062,
      "rewards/margins": 0.08983123302459717,
      "rewards/rejected": -2.144861936569214,
      "step": 14203
    },
    {
      "epoch": 1.8586897849022654,
      "grad_norm": 5.996728420257568,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4294632077217102,
      "logits/rejected": -0.32244011759757996,
      "logps/chosen": -378.51458740234375,
      "logps/rejected": -428.8520202636719,
      "loss": 0.2661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8605838418006897,
      "rewards/margins": 2.171016216278076,
      "rewards/rejected": -3.0316002368927,
      "step": 14204
    },
    {
      "epoch": 1.8588206428396172,
      "grad_norm": 12.059453964233398,
      "learning_rate": 5e-06,
      "logits/chosen": 0.037183359265327454,
      "logits/rejected": -0.0943872332572937,
      "logps/chosen": -531.5867919921875,
      "logps/rejected": -546.39453125,
      "loss": 0.619,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0042872428894043,
      "rewards/margins": 0.7897297143936157,
      "rewards/rejected": -2.7940165996551514,
      "step": 14205
    },
    {
      "epoch": 1.858951500776969,
      "grad_norm": 7.278506755828857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30630752444267273,
      "logits/rejected": -0.2995240092277527,
      "logps/chosen": -509.7707824707031,
      "logps/rejected": -673.440673828125,
      "loss": 0.3109,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9979649186134338,
      "rewards/margins": 1.3343353271484375,
      "rewards/rejected": -2.3322999477386475,
      "step": 14206
    },
    {
      "epoch": 1.8590823587143208,
      "grad_norm": 10.407628059387207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21291646361351013,
      "logits/rejected": -0.4093386232852936,
      "logps/chosen": -353.62066650390625,
      "logps/rejected": -415.3931884765625,
      "loss": 0.5733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0407581329345703,
      "rewards/margins": 1.2460341453552246,
      "rewards/rejected": -3.286792039871216,
      "step": 14207
    },
    {
      "epoch": 1.8592132166516726,
      "grad_norm": 15.739758491516113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5161255598068237,
      "logits/rejected": -0.3403945565223694,
      "logps/chosen": -352.7231140136719,
      "logps/rejected": -447.76763916015625,
      "loss": 0.7192,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3169283866882324,
      "rewards/margins": 0.4578372538089752,
      "rewards/rejected": -2.7747654914855957,
      "step": 14208
    },
    {
      "epoch": 1.8593440745890244,
      "grad_norm": 7.405043125152588,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17623203992843628,
      "logits/rejected": -0.2652778923511505,
      "logps/chosen": -468.3615417480469,
      "logps/rejected": -461.4271240234375,
      "loss": 0.3073,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1467479467391968,
      "rewards/margins": 1.4859542846679688,
      "rewards/rejected": -2.632702350616455,
      "step": 14209
    },
    {
      "epoch": 1.8594749325263762,
      "grad_norm": 11.407519340515137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30050408840179443,
      "logits/rejected": -0.33144357800483704,
      "logps/chosen": -484.0655517578125,
      "logps/rejected": -383.461669921875,
      "loss": 0.4048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.54512357711792,
      "rewards/margins": 1.611247181892395,
      "rewards/rejected": -3.1563708782196045,
      "step": 14210
    },
    {
      "epoch": 1.859605790463728,
      "grad_norm": 10.873167037963867,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21590089797973633,
      "logits/rejected": -0.2321167290210724,
      "logps/chosen": -488.3218994140625,
      "logps/rejected": -474.03338623046875,
      "loss": 0.6102,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4101202487945557,
      "rewards/margins": 0.9228236079216003,
      "rewards/rejected": -2.3329436779022217,
      "step": 14211
    },
    {
      "epoch": 1.8597366484010796,
      "grad_norm": 8.176809310913086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.007615514099597931,
      "logits/rejected": -0.01906682550907135,
      "logps/chosen": -677.4229125976562,
      "logps/rejected": -527.9871826171875,
      "loss": 0.4928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28095152974128723,
      "rewards/margins": 1.727739930152893,
      "rewards/rejected": -2.0086913108825684,
      "step": 14212
    },
    {
      "epoch": 1.8598675063384313,
      "grad_norm": 18.726341247558594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2720274031162262,
      "logits/rejected": -0.010863956063985825,
      "logps/chosen": -694.1010131835938,
      "logps/rejected": -537.8256225585938,
      "loss": 0.7873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.359100341796875,
      "rewards/margins": 1.0182983875274658,
      "rewards/rejected": -2.377398729324341,
      "step": 14213
    },
    {
      "epoch": 1.8599983642757831,
      "grad_norm": 11.320812225341797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3383656144142151,
      "logits/rejected": -0.5072512626647949,
      "logps/chosen": -255.8595428466797,
      "logps/rejected": -312.72515869140625,
      "loss": 0.9459,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6135393381118774,
      "rewards/margins": -0.04360602796077728,
      "rewards/rejected": -1.569933295249939,
      "step": 14214
    },
    {
      "epoch": 1.8601292222131347,
      "grad_norm": 10.436841011047363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03379557654261589,
      "logits/rejected": -0.05933818221092224,
      "logps/chosen": -458.5008850097656,
      "logps/rejected": -540.4346313476562,
      "loss": 0.4794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3156614303588867,
      "rewards/margins": 2.193608522415161,
      "rewards/rejected": -3.509269952774048,
      "step": 14215
    },
    {
      "epoch": 1.8602600801504865,
      "grad_norm": 4.564116477966309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37569060921669006,
      "logits/rejected": -0.5637277960777283,
      "logps/chosen": -326.6180419921875,
      "logps/rejected": -420.5823669433594,
      "loss": 0.2105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4507906138896942,
      "rewards/margins": 2.8093783855438232,
      "rewards/rejected": -3.26016902923584,
      "step": 14216
    },
    {
      "epoch": 1.8603909380878383,
      "grad_norm": 9.608860969543457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2603693902492523,
      "logits/rejected": -0.308724045753479,
      "logps/chosen": -410.3970642089844,
      "logps/rejected": -426.4703063964844,
      "loss": 0.5869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2139747142791748,
      "rewards/margins": 0.8486694693565369,
      "rewards/rejected": -2.0626442432403564,
      "step": 14217
    },
    {
      "epoch": 1.86052179602519,
      "grad_norm": 7.760824680328369,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08558512479066849,
      "logits/rejected": -0.0007311664521694183,
      "logps/chosen": -474.65509033203125,
      "logps/rejected": -498.552490234375,
      "loss": 0.2969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5833897590637207,
      "rewards/margins": 1.5261032581329346,
      "rewards/rejected": -3.1094930171966553,
      "step": 14218
    },
    {
      "epoch": 1.8606526539625419,
      "grad_norm": 10.326549530029297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1864464282989502,
      "logits/rejected": -0.31675735116004944,
      "logps/chosen": -517.29541015625,
      "logps/rejected": -485.5884704589844,
      "loss": 0.559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8069359660148621,
      "rewards/margins": 0.8457403182983398,
      "rewards/rejected": -1.6526763439178467,
      "step": 14219
    },
    {
      "epoch": 1.8607835118998937,
      "grad_norm": 7.473623752593994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4739227294921875,
      "logits/rejected": -0.3339517116546631,
      "logps/chosen": -461.7405090332031,
      "logps/rejected": -461.4871520996094,
      "loss": 0.3335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.604460597038269,
      "rewards/margins": 1.1124005317687988,
      "rewards/rejected": -2.716860771179199,
      "step": 14220
    },
    {
      "epoch": 1.8609143698372455,
      "grad_norm": 6.173833847045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5231544971466064,
      "logits/rejected": -0.15503668785095215,
      "logps/chosen": -485.5251159667969,
      "logps/rejected": -584.635009765625,
      "loss": 0.3063,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3223297595977783,
      "rewards/margins": 2.4230148792266846,
      "rewards/rejected": -3.745344638824463,
      "step": 14221
    },
    {
      "epoch": 1.8610452277745972,
      "grad_norm": 2.9773831367492676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07147806137800217,
      "logits/rejected": -0.20717540383338928,
      "logps/chosen": -362.5246887207031,
      "logps/rejected": -390.0931091308594,
      "loss": 0.1069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9834530353546143,
      "rewards/margins": 3.0192959308624268,
      "rewards/rejected": -4.002748489379883,
      "step": 14222
    },
    {
      "epoch": 1.861176085711949,
      "grad_norm": 7.202653884887695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1455564945936203,
      "logits/rejected": -0.2868213355541229,
      "logps/chosen": -277.1805114746094,
      "logps/rejected": -225.77426147460938,
      "loss": 0.4734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5378491878509521,
      "rewards/margins": 0.9256430268287659,
      "rewards/rejected": -2.4634921550750732,
      "step": 14223
    },
    {
      "epoch": 1.8613069436493008,
      "grad_norm": 17.38724136352539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41837066411972046,
      "logits/rejected": -0.4119192659854889,
      "logps/chosen": -463.36993408203125,
      "logps/rejected": -493.22564697265625,
      "loss": 0.6239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2051793336868286,
      "rewards/margins": 1.5962746143341064,
      "rewards/rejected": -2.8014538288116455,
      "step": 14224
    },
    {
      "epoch": 1.8614378015866526,
      "grad_norm": 6.815432071685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24905717372894287,
      "logits/rejected": -0.3096822500228882,
      "logps/chosen": -545.0065307617188,
      "logps/rejected": -505.37445068359375,
      "loss": 0.2519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2036601305007935,
      "rewards/margins": 1.6208373308181763,
      "rewards/rejected": -2.8244974613189697,
      "step": 14225
    },
    {
      "epoch": 1.8615686595240044,
      "grad_norm": 6.649152755737305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.655007541179657,
      "logits/rejected": -0.6907585859298706,
      "logps/chosen": -753.8361206054688,
      "logps/rejected": -688.5810546875,
      "loss": 0.2732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9441717863082886,
      "rewards/margins": 1.792150616645813,
      "rewards/rejected": -2.7363221645355225,
      "step": 14226
    },
    {
      "epoch": 1.861699517461356,
      "grad_norm": 5.337549686431885,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3124842941761017,
      "logits/rejected": -0.40328121185302734,
      "logps/chosen": -334.754150390625,
      "logps/rejected": -313.53887939453125,
      "loss": 0.3054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9707438349723816,
      "rewards/margins": 1.346251130104065,
      "rewards/rejected": -2.3169949054718018,
      "step": 14227
    },
    {
      "epoch": 1.8618303753987078,
      "grad_norm": 6.104965686798096,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2725391685962677,
      "logits/rejected": -0.3007201850414276,
      "logps/chosen": -699.7415161132812,
      "logps/rejected": -695.8470458984375,
      "loss": 0.1384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4505071640014648,
      "rewards/margins": 2.7456419467926025,
      "rewards/rejected": -4.196148872375488,
      "step": 14228
    },
    {
      "epoch": 1.8619612333360596,
      "grad_norm": 7.9311394691467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20322179794311523,
      "logits/rejected": -0.2875503897666931,
      "logps/chosen": -590.4535522460938,
      "logps/rejected": -590.9900512695312,
      "loss": 0.2775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8705341219902039,
      "rewards/margins": 2.6383206844329834,
      "rewards/rejected": -3.508854866027832,
      "step": 14229
    },
    {
      "epoch": 1.8620920912734114,
      "grad_norm": 9.972892761230469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.031336501240730286,
      "logits/rejected": -0.11607786267995834,
      "logps/chosen": -426.00689697265625,
      "logps/rejected": -420.7434387207031,
      "loss": 0.4881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9852641820907593,
      "rewards/margins": 0.6480823755264282,
      "rewards/rejected": -1.6333465576171875,
      "step": 14230
    },
    {
      "epoch": 1.862222949210763,
      "grad_norm": 13.096403121948242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006607621908187866,
      "logits/rejected": -0.04781723767518997,
      "logps/chosen": -485.2144775390625,
      "logps/rejected": -437.6741943359375,
      "loss": 0.718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1686202883720398,
      "rewards/margins": 0.8673916459083557,
      "rewards/rejected": -1.0360119342803955,
      "step": 14231
    },
    {
      "epoch": 1.8623538071481147,
      "grad_norm": 16.791433334350586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03960496932268143,
      "logits/rejected": 0.010419853031635284,
      "logps/chosen": -467.7463073730469,
      "logps/rejected": -399.9814147949219,
      "loss": 0.9795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.021909236907959,
      "rewards/margins": 0.13887694478034973,
      "rewards/rejected": -1.1607860326766968,
      "step": 14232
    },
    {
      "epoch": 1.8624846650854665,
      "grad_norm": 6.998781681060791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46705928444862366,
      "logits/rejected": -0.3265191316604614,
      "logps/chosen": -370.4756164550781,
      "logps/rejected": -343.6439208984375,
      "loss": 0.2516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6361162662506104,
      "rewards/margins": 2.0498239994049072,
      "rewards/rejected": -2.6859400272369385,
      "step": 14233
    },
    {
      "epoch": 1.8626155230228183,
      "grad_norm": 11.628118515014648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052974555641412735,
      "logits/rejected": -0.23607227206230164,
      "logps/chosen": -378.52667236328125,
      "logps/rejected": -392.078857421875,
      "loss": 0.3708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9222742319107056,
      "rewards/margins": 1.5969951152801514,
      "rewards/rejected": -3.5192694664001465,
      "step": 14234
    },
    {
      "epoch": 1.86274638096017,
      "grad_norm": 6.809370040893555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32856079936027527,
      "logits/rejected": -0.30153417587280273,
      "logps/chosen": -335.5512390136719,
      "logps/rejected": -404.5411376953125,
      "loss": 0.2773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40052321553230286,
      "rewards/margins": 1.4666616916656494,
      "rewards/rejected": -1.8671848773956299,
      "step": 14235
    },
    {
      "epoch": 1.8628772388975219,
      "grad_norm": 18.24057388305664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1906806230545044,
      "logits/rejected": -0.14239250123500824,
      "logps/chosen": -460.8147277832031,
      "logps/rejected": -319.77301025390625,
      "loss": 0.9942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.1526176929473877,
      "rewards/margins": 0.1950642168521881,
      "rewards/rejected": -2.347681760787964,
      "step": 14236
    },
    {
      "epoch": 1.8630080968348737,
      "grad_norm": 5.6577067375183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1000826507806778,
      "logits/rejected": -0.27304741740226746,
      "logps/chosen": -421.0820617675781,
      "logps/rejected": -388.8194580078125,
      "loss": 0.4302,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9641339778900146,
      "rewards/margins": 1.3317466974258423,
      "rewards/rejected": -2.2958807945251465,
      "step": 14237
    },
    {
      "epoch": 1.8631389547722255,
      "grad_norm": 9.159181594848633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11149074882268906,
      "logits/rejected": -0.012867996469140053,
      "logps/chosen": -495.1444091796875,
      "logps/rejected": -527.9661865234375,
      "loss": 0.3361,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7449470162391663,
      "rewards/margins": 1.5357754230499268,
      "rewards/rejected": -2.2807228565216064,
      "step": 14238
    },
    {
      "epoch": 1.8632698127095773,
      "grad_norm": 11.870928764343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37060779333114624,
      "logits/rejected": -0.31952717900276184,
      "logps/chosen": -497.8695373535156,
      "logps/rejected": -404.7987365722656,
      "loss": 0.5675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1020539999008179,
      "rewards/margins": 0.4575769305229187,
      "rewards/rejected": -1.5596308708190918,
      "step": 14239
    },
    {
      "epoch": 1.863400670646929,
      "grad_norm": 9.650958061218262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17127463221549988,
      "logits/rejected": -0.2103399783372879,
      "logps/chosen": -293.5731201171875,
      "logps/rejected": -328.0030212402344,
      "loss": 0.4763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1460142135620117,
      "rewards/margins": 1.0270709991455078,
      "rewards/rejected": -2.1730852127075195,
      "step": 14240
    },
    {
      "epoch": 1.8635315285842808,
      "grad_norm": 5.594003677368164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3730877935886383,
      "logits/rejected": -0.3422969579696655,
      "logps/chosen": -399.9404602050781,
      "logps/rejected": -380.9819641113281,
      "loss": 0.2522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9856418967247009,
      "rewards/margins": 1.6283317804336548,
      "rewards/rejected": -2.61397385597229,
      "step": 14241
    },
    {
      "epoch": 1.8636623865216324,
      "grad_norm": 9.8829345703125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5939076542854309,
      "logits/rejected": -0.3777623772621155,
      "logps/chosen": -381.2707824707031,
      "logps/rejected": -617.4733276367188,
      "loss": 0.4094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9409444332122803,
      "rewards/margins": 1.258092999458313,
      "rewards/rejected": -2.199037551879883,
      "step": 14242
    },
    {
      "epoch": 1.8637932444589842,
      "grad_norm": 7.23844051361084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18567059934139252,
      "logits/rejected": -0.21098840236663818,
      "logps/chosen": -703.1766967773438,
      "logps/rejected": -477.8982849121094,
      "loss": 0.3416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9180485606193542,
      "rewards/margins": 1.7454510927200317,
      "rewards/rejected": -2.6634998321533203,
      "step": 14243
    },
    {
      "epoch": 1.863924102396336,
      "grad_norm": 5.3993988037109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26364925503730774,
      "logits/rejected": -0.6247084736824036,
      "logps/chosen": -405.2630920410156,
      "logps/rejected": -473.7518310546875,
      "loss": 0.2823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2876838445663452,
      "rewards/margins": 1.7797765731811523,
      "rewards/rejected": -3.067460775375366,
      "step": 14244
    },
    {
      "epoch": 1.8640549603336878,
      "grad_norm": 8.57790756225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30732491612434387,
      "logits/rejected": -0.22285905480384827,
      "logps/chosen": -441.43536376953125,
      "logps/rejected": -459.6279602050781,
      "loss": 0.4149,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.060543417930603,
      "rewards/margins": 1.2983570098876953,
      "rewards/rejected": -2.358900547027588,
      "step": 14245
    },
    {
      "epoch": 1.8641858182710394,
      "grad_norm": 5.881515026092529,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34075599908828735,
      "logits/rejected": -0.3044748902320862,
      "logps/chosen": -627.6893310546875,
      "logps/rejected": -437.5479431152344,
      "loss": 0.2246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7020754218101501,
      "rewards/margins": 2.117067337036133,
      "rewards/rejected": -2.8191428184509277,
      "step": 14246
    },
    {
      "epoch": 1.8643166762083911,
      "grad_norm": 11.766328811645508,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04421476274728775,
      "logits/rejected": -0.02513321302831173,
      "logps/chosen": -513.925048828125,
      "logps/rejected": -513.8760986328125,
      "loss": 0.5191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9607653617858887,
      "rewards/margins": 1.3959286212921143,
      "rewards/rejected": -3.356693983078003,
      "step": 14247
    },
    {
      "epoch": 1.864447534145743,
      "grad_norm": 9.144222259521484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7636750340461731,
      "logits/rejected": -0.5420918464660645,
      "logps/chosen": -422.64715576171875,
      "logps/rejected": -587.0801391601562,
      "loss": 0.3966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.470834255218506,
      "rewards/margins": 1.4258782863616943,
      "rewards/rejected": -3.8967125415802,
      "step": 14248
    },
    {
      "epoch": 1.8645783920830947,
      "grad_norm": 7.608189105987549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.091802679002285,
      "logits/rejected": -0.16549861431121826,
      "logps/chosen": -527.830810546875,
      "logps/rejected": -494.7563781738281,
      "loss": 0.3154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.483043909072876,
      "rewards/margins": 1.6626038551330566,
      "rewards/rejected": -3.1456477642059326,
      "step": 14249
    },
    {
      "epoch": 1.8647092500204465,
      "grad_norm": 13.641626358032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3642699420452118,
      "logits/rejected": -0.1823861449956894,
      "logps/chosen": -525.2950439453125,
      "logps/rejected": -648.1984252929688,
      "loss": 0.8354,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0032739639282227,
      "rewards/margins": -0.1062127947807312,
      "rewards/rejected": -0.8970612287521362,
      "step": 14250
    },
    {
      "epoch": 1.8648401079577983,
      "grad_norm": 9.5784330368042,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23558607697486877,
      "logits/rejected": -0.25766414403915405,
      "logps/chosen": -460.5994567871094,
      "logps/rejected": -459.7292175292969,
      "loss": 0.5473,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3375070095062256,
      "rewards/margins": 0.9942550659179688,
      "rewards/rejected": -2.3317620754241943,
      "step": 14251
    },
    {
      "epoch": 1.86497096589515,
      "grad_norm": 8.104348182678223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09947355091571808,
      "logits/rejected": 0.016760222613811493,
      "logps/chosen": -529.744873046875,
      "logps/rejected": -472.5205993652344,
      "loss": 0.4331,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3700979948043823,
      "rewards/margins": 1.1312388181686401,
      "rewards/rejected": -2.5013365745544434,
      "step": 14252
    },
    {
      "epoch": 1.865101823832502,
      "grad_norm": 6.611140727996826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3720543384552002,
      "logits/rejected": -0.34044352173805237,
      "logps/chosen": -377.9759826660156,
      "logps/rejected": -486.1121520996094,
      "loss": 0.2908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2336727380752563,
      "rewards/margins": 1.9022798538208008,
      "rewards/rejected": -3.1359527111053467,
      "step": 14253
    },
    {
      "epoch": 1.8652326817698537,
      "grad_norm": 5.956179618835449,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21416454017162323,
      "logits/rejected": -0.15988971292972565,
      "logps/chosen": -322.2643737792969,
      "logps/rejected": -340.56793212890625,
      "loss": 0.3361,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2602728605270386,
      "rewards/margins": 1.377603530883789,
      "rewards/rejected": -2.637876510620117,
      "step": 14254
    },
    {
      "epoch": 1.8653635397072055,
      "grad_norm": 4.551324367523193,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13325028121471405,
      "logits/rejected": -0.2711292505264282,
      "logps/chosen": -268.64849853515625,
      "logps/rejected": -273.06378173828125,
      "loss": 0.1798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6709045171737671,
      "rewards/margins": 2.3042964935302734,
      "rewards/rejected": -2.97520112991333,
      "step": 14255
    },
    {
      "epoch": 1.8654943976445573,
      "grad_norm": 8.323224067687988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28321585059165955,
      "logits/rejected": -0.21605803072452545,
      "logps/chosen": -532.1038208007812,
      "logps/rejected": -538.2100830078125,
      "loss": 0.2582,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6550413370132446,
      "rewards/margins": 1.828385591506958,
      "rewards/rejected": -2.483427047729492,
      "step": 14256
    },
    {
      "epoch": 1.8656252555819088,
      "grad_norm": 10.425159454345703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46724140644073486,
      "logits/rejected": -0.45138901472091675,
      "logps/chosen": -357.64691162109375,
      "logps/rejected": -381.0030517578125,
      "loss": 0.6104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7885550260543823,
      "rewards/margins": 0.8413194417953491,
      "rewards/rejected": -2.6298742294311523,
      "step": 14257
    },
    {
      "epoch": 1.8657561135192606,
      "grad_norm": 3.9360909461975098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2749180197715759,
      "logits/rejected": -0.19327715039253235,
      "logps/chosen": -377.9991455078125,
      "logps/rejected": -491.75482177734375,
      "loss": 0.2295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1706593036651611,
      "rewards/margins": 1.9756898880004883,
      "rewards/rejected": -3.1463494300842285,
      "step": 14258
    },
    {
      "epoch": 1.8658869714566124,
      "grad_norm": 9.229777336120605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40254056453704834,
      "logits/rejected": -0.39071768522262573,
      "logps/chosen": -314.5707092285156,
      "logps/rejected": -361.8964538574219,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1613820791244507,
      "rewards/margins": 1.1687419414520264,
      "rewards/rejected": -2.3301241397857666,
      "step": 14259
    },
    {
      "epoch": 1.8660178293939642,
      "grad_norm": 6.4268083572387695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21304145455360413,
      "logits/rejected": -0.29192858934402466,
      "logps/chosen": -523.497314453125,
      "logps/rejected": -617.853515625,
      "loss": 0.2471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1394035816192627,
      "rewards/margins": 1.808741807937622,
      "rewards/rejected": -2.9481453895568848,
      "step": 14260
    },
    {
      "epoch": 1.8661486873313158,
      "grad_norm": 5.805533409118652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08091463148593903,
      "logits/rejected": -0.11479617655277252,
      "logps/chosen": -542.8607788085938,
      "logps/rejected": -455.78656005859375,
      "loss": 0.2475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1266210675239563,
      "rewards/margins": 2.2226037979125977,
      "rewards/rejected": -2.095982551574707,
      "step": 14261
    },
    {
      "epoch": 1.8662795452686676,
      "grad_norm": 9.508282661437988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1449558138847351,
      "logits/rejected": -0.29650241136550903,
      "logps/chosen": -559.587158203125,
      "logps/rejected": -592.372802734375,
      "loss": 0.4024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7594807147979736,
      "rewards/margins": 1.907180905342102,
      "rewards/rejected": -2.6666617393493652,
      "step": 14262
    },
    {
      "epoch": 1.8664104032060194,
      "grad_norm": 7.530950546264648,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2300817370414734,
      "logits/rejected": -0.13299335539340973,
      "logps/chosen": -581.92822265625,
      "logps/rejected": -656.50048828125,
      "loss": 0.3219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7258034944534302,
      "rewards/margins": 1.911295771598816,
      "rewards/rejected": -3.637099027633667,
      "step": 14263
    },
    {
      "epoch": 1.8665412611433712,
      "grad_norm": 7.627362251281738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34461188316345215,
      "logits/rejected": -0.13569538295269012,
      "logps/chosen": -435.8662109375,
      "logps/rejected": -402.4014587402344,
      "loss": 0.3659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5149751901626587,
      "rewards/margins": 1.2377843856811523,
      "rewards/rejected": -1.7527596950531006,
      "step": 14264
    },
    {
      "epoch": 1.866672119080723,
      "grad_norm": 5.457470417022705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5601494312286377,
      "logits/rejected": -0.21692396700382233,
      "logps/chosen": -327.6192321777344,
      "logps/rejected": -412.4819641113281,
      "loss": 0.2622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1577084064483643,
      "rewards/margins": 1.6061549186706543,
      "rewards/rejected": -2.7638633251190186,
      "step": 14265
    },
    {
      "epoch": 1.8668029770180747,
      "grad_norm": 6.497296333312988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4234042167663574,
      "logits/rejected": -0.5318433046340942,
      "logps/chosen": -466.823974609375,
      "logps/rejected": -413.55401611328125,
      "loss": 0.2836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6468988060951233,
      "rewards/margins": 1.9109971523284912,
      "rewards/rejected": -2.557896137237549,
      "step": 14266
    },
    {
      "epoch": 1.8669338349554265,
      "grad_norm": 7.44778299331665,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05056125670671463,
      "logits/rejected": -0.1837000846862793,
      "logps/chosen": -477.8340148925781,
      "logps/rejected": -518.4020385742188,
      "loss": 0.2541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19916477799415588,
      "rewards/margins": 2.687746047973633,
      "rewards/rejected": -2.4885811805725098,
      "step": 14267
    },
    {
      "epoch": 1.8670646928927783,
      "grad_norm": 11.638143539428711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34113970398902893,
      "logits/rejected": -0.34863996505737305,
      "logps/chosen": -581.9421997070312,
      "logps/rejected": -467.51629638671875,
      "loss": 0.5387,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9820120334625244,
      "rewards/margins": 1.1450858116149902,
      "rewards/rejected": -3.1270978450775146,
      "step": 14268
    },
    {
      "epoch": 1.8671955508301301,
      "grad_norm": 6.91970157623291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25203725695610046,
      "logits/rejected": -0.3878386914730072,
      "logps/chosen": -454.3513488769531,
      "logps/rejected": -427.4920654296875,
      "loss": 0.2853,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1062990427017212,
      "rewards/margins": 1.629915714263916,
      "rewards/rejected": -2.7362148761749268,
      "step": 14269
    },
    {
      "epoch": 1.867326408767482,
      "grad_norm": 7.035587787628174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15490327775478363,
      "logits/rejected": -0.17036205530166626,
      "logps/chosen": -382.5650939941406,
      "logps/rejected": -416.7471008300781,
      "loss": 0.5015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3213024139404297,
      "rewards/margins": 0.8933133482933044,
      "rewards/rejected": -2.2146155834198,
      "step": 14270
    },
    {
      "epoch": 1.8674572667048337,
      "grad_norm": 8.4929780960083,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18034237623214722,
      "logits/rejected": -0.3973890542984009,
      "logps/chosen": -526.941650390625,
      "logps/rejected": -438.82940673828125,
      "loss": 0.5645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.405643105506897,
      "rewards/margins": 1.2374918460845947,
      "rewards/rejected": -2.6431350708007812,
      "step": 14271
    },
    {
      "epoch": 1.8675881246421855,
      "grad_norm": 6.400145530700684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.011079231277108192,
      "logits/rejected": -0.016387583687901497,
      "logps/chosen": -532.799072265625,
      "logps/rejected": -563.9497680664062,
      "loss": 0.25,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.625348687171936,
      "rewards/margins": 2.079613208770752,
      "rewards/rejected": -2.7049617767333984,
      "step": 14272
    },
    {
      "epoch": 1.867718982579537,
      "grad_norm": 8.949797630310059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25384658575057983,
      "logits/rejected": -0.07750572264194489,
      "logps/chosen": -453.9707336425781,
      "logps/rejected": -406.4854431152344,
      "loss": 0.4271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8157722353935242,
      "rewards/margins": 1.6726588010787964,
      "rewards/rejected": -2.488430976867676,
      "step": 14273
    },
    {
      "epoch": 1.8678498405168888,
      "grad_norm": 11.042462348937988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1751716434955597,
      "logits/rejected": -0.20642079412937164,
      "logps/chosen": -585.130615234375,
      "logps/rejected": -487.4848327636719,
      "loss": 0.4602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9257305860519409,
      "rewards/margins": 1.1012104749679565,
      "rewards/rejected": -2.0269410610198975,
      "step": 14274
    },
    {
      "epoch": 1.8679806984542406,
      "grad_norm": 12.160901069641113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.261984646320343,
      "logits/rejected": -0.3838810324668884,
      "logps/chosen": -601.7631225585938,
      "logps/rejected": -671.123046875,
      "loss": 0.6292,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8215078115463257,
      "rewards/margins": 1.1906275749206543,
      "rewards/rejected": -3.0121355056762695,
      "step": 14275
    },
    {
      "epoch": 1.8681115563915924,
      "grad_norm": 11.235299110412598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35873886942863464,
      "logits/rejected": -0.26497453451156616,
      "logps/chosen": -526.80322265625,
      "logps/rejected": -556.5776977539062,
      "loss": 0.7396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9556993246078491,
      "rewards/margins": 0.7776435613632202,
      "rewards/rejected": -1.7333428859710693,
      "step": 14276
    },
    {
      "epoch": 1.868242414328944,
      "grad_norm": 8.687651634216309,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054155923426151276,
      "logits/rejected": -0.28504979610443115,
      "logps/chosen": -534.1038208007812,
      "logps/rejected": -733.183349609375,
      "loss": 0.3624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9563589096069336,
      "rewards/margins": 1.722057580947876,
      "rewards/rejected": -2.6784164905548096,
      "step": 14277
    },
    {
      "epoch": 1.8683732722662958,
      "grad_norm": 7.884696960449219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2291661500930786,
      "logits/rejected": -0.08434171974658966,
      "logps/chosen": -470.9613037109375,
      "logps/rejected": -474.93609619140625,
      "loss": 0.369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9074228405952454,
      "rewards/margins": 1.4841994047164917,
      "rewards/rejected": -2.391622304916382,
      "step": 14278
    },
    {
      "epoch": 1.8685041302036476,
      "grad_norm": 9.263075828552246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4275408983230591,
      "logits/rejected": -0.3990412950515747,
      "logps/chosen": -440.7678527832031,
      "logps/rejected": -470.2005310058594,
      "loss": 0.6573,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.49371567368507385,
      "rewards/margins": 0.7803147435188293,
      "rewards/rejected": -1.2740306854248047,
      "step": 14279
    },
    {
      "epoch": 1.8686349881409994,
      "grad_norm": 7.916388034820557,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02923990972340107,
      "logits/rejected": -0.20024138689041138,
      "logps/chosen": -576.2496337890625,
      "logps/rejected": -430.04974365234375,
      "loss": 0.4189,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1550318002700806,
      "rewards/margins": 0.9989497661590576,
      "rewards/rejected": -2.1539816856384277,
      "step": 14280
    },
    {
      "epoch": 1.8687658460783512,
      "grad_norm": 12.418893814086914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39310336112976074,
      "logits/rejected": -0.3547934889793396,
      "logps/chosen": -559.3671875,
      "logps/rejected": -537.88818359375,
      "loss": 0.5856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8465947508811951,
      "rewards/margins": 1.279664397239685,
      "rewards/rejected": -2.1262590885162354,
      "step": 14281
    },
    {
      "epoch": 1.868896704015703,
      "grad_norm": 5.618628978729248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0582287460565567,
      "logits/rejected": 0.02708294242620468,
      "logps/chosen": -457.6838684082031,
      "logps/rejected": -509.60711669921875,
      "loss": 0.2417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08139188587665558,
      "rewards/margins": 2.245814800262451,
      "rewards/rejected": -2.32720685005188,
      "step": 14282
    },
    {
      "epoch": 1.8690275619530548,
      "grad_norm": 5.149449825286865,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23379889130592346,
      "logits/rejected": -0.11395987123250961,
      "logps/chosen": -653.71337890625,
      "logps/rejected": -422.3240966796875,
      "loss": 0.2477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.367183804512024,
      "rewards/margins": 2.015888214111328,
      "rewards/rejected": -3.3830718994140625,
      "step": 14283
    },
    {
      "epoch": 1.8691584198904065,
      "grad_norm": 5.662147045135498,
      "learning_rate": 5e-06,
      "logits/chosen": -0.055320046842098236,
      "logits/rejected": -0.09111326932907104,
      "logps/chosen": -382.3182373046875,
      "logps/rejected": -476.6548156738281,
      "loss": 0.179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08548997342586517,
      "rewards/margins": 2.0433454513549805,
      "rewards/rejected": -2.128835439682007,
      "step": 14284
    },
    {
      "epoch": 1.8692892778277583,
      "grad_norm": 9.695280075073242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1652657389640808,
      "logits/rejected": -0.2281908541917801,
      "logps/chosen": -613.5626831054688,
      "logps/rejected": -615.44189453125,
      "loss": 0.5423,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9629770517349243,
      "rewards/margins": 0.6094132661819458,
      "rewards/rejected": -2.57239031791687,
      "step": 14285
    },
    {
      "epoch": 1.8694201357651101,
      "grad_norm": 6.509032249450684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08221357315778732,
      "logits/rejected": -0.0932028517127037,
      "logps/chosen": -405.767822265625,
      "logps/rejected": -451.69091796875,
      "loss": 0.3047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.143535614013672,
      "rewards/margins": 1.1584609746932983,
      "rewards/rejected": -3.3019967079162598,
      "step": 14286
    },
    {
      "epoch": 1.869550993702462,
      "grad_norm": 7.63041353225708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3537978529930115,
      "logits/rejected": -0.5087792277336121,
      "logps/chosen": -542.146484375,
      "logps/rejected": -416.05291748046875,
      "loss": 0.4681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.493329644203186,
      "rewards/margins": 1.2908570766448975,
      "rewards/rejected": -2.784186601638794,
      "step": 14287
    },
    {
      "epoch": 1.8696818516398135,
      "grad_norm": 9.69917106628418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12764543294906616,
      "logits/rejected": -0.06622204184532166,
      "logps/chosen": -321.8091735839844,
      "logps/rejected": -363.6248779296875,
      "loss": 0.5065,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.102159857749939,
      "rewards/margins": 1.3948285579681396,
      "rewards/rejected": -2.496988534927368,
      "step": 14288
    },
    {
      "epoch": 1.8698127095771653,
      "grad_norm": 8.307744026184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4912738800048828,
      "logits/rejected": -0.3607214689254761,
      "logps/chosen": -351.6859436035156,
      "logps/rejected": -464.53167724609375,
      "loss": 0.3976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4406431913375854,
      "rewards/margins": 1.0710467100143433,
      "rewards/rejected": -2.511690139770508,
      "step": 14289
    },
    {
      "epoch": 1.869943567514517,
      "grad_norm": 5.235691070556641,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3197755813598633,
      "logits/rejected": -0.25995054841041565,
      "logps/chosen": -413.2229309082031,
      "logps/rejected": -388.6777648925781,
      "loss": 0.2122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7104790210723877,
      "rewards/margins": 2.192631959915161,
      "rewards/rejected": -2.903111219406128,
      "step": 14290
    },
    {
      "epoch": 1.8700744254518689,
      "grad_norm": 9.190926551818848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29796209931373596,
      "logits/rejected": -0.3651522994041443,
      "logps/chosen": -471.1988830566406,
      "logps/rejected": -548.9520874023438,
      "loss": 0.4651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0976214408874512,
      "rewards/margins": 1.6888419389724731,
      "rewards/rejected": -2.7864632606506348,
      "step": 14291
    },
    {
      "epoch": 1.8702052833892204,
      "grad_norm": 8.337514877319336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14683881402015686,
      "logits/rejected": -0.1699552983045578,
      "logps/chosen": -697.2515258789062,
      "logps/rejected": -505.2833557128906,
      "loss": 0.3912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3053695857524872,
      "rewards/margins": 1.4995687007904053,
      "rewards/rejected": -1.8049381971359253,
      "step": 14292
    },
    {
      "epoch": 1.8703361413265722,
      "grad_norm": 8.329264640808105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21288244426250458,
      "logits/rejected": -0.42711758613586426,
      "logps/chosen": -495.634765625,
      "logps/rejected": -523.1533203125,
      "loss": 0.3304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7184739112854004,
      "rewards/margins": 1.3235466480255127,
      "rewards/rejected": -3.042020797729492,
      "step": 14293
    },
    {
      "epoch": 1.870466999263924,
      "grad_norm": 7.942152500152588,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0005403552204370499,
      "logits/rejected": -0.16259731352329254,
      "logps/chosen": -513.8981323242188,
      "logps/rejected": -320.4735412597656,
      "loss": 0.3954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6995648741722107,
      "rewards/margins": 1.7531101703643799,
      "rewards/rejected": -2.4526755809783936,
      "step": 14294
    },
    {
      "epoch": 1.8705978572012758,
      "grad_norm": 10.1598539352417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.022212184965610504,
      "logits/rejected": -0.3163977861404419,
      "logps/chosen": -456.4617919921875,
      "logps/rejected": -503.3008117675781,
      "loss": 0.5801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.245572805404663,
      "rewards/margins": 0.777338445186615,
      "rewards/rejected": -2.022911310195923,
      "step": 14295
    },
    {
      "epoch": 1.8707287151386276,
      "grad_norm": 8.112000465393066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19033825397491455,
      "logits/rejected": -0.1629726141691208,
      "logps/chosen": -572.2763061523438,
      "logps/rejected": -548.3291625976562,
      "loss": 0.2858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8993655443191528,
      "rewards/margins": 1.3496066331863403,
      "rewards/rejected": -3.248971939086914,
      "step": 14296
    },
    {
      "epoch": 1.8708595730759794,
      "grad_norm": 7.2369208335876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5912725329399109,
      "logits/rejected": -0.6990456581115723,
      "logps/chosen": -387.48223876953125,
      "logps/rejected": -426.4693298339844,
      "loss": 0.3555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7550548315048218,
      "rewards/margins": 1.2709146738052368,
      "rewards/rejected": -3.0259695053100586,
      "step": 14297
    },
    {
      "epoch": 1.8709904310133312,
      "grad_norm": 8.503331184387207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21783460676670074,
      "logits/rejected": -0.10184712707996368,
      "logps/chosen": -323.97052001953125,
      "logps/rejected": -421.48175048828125,
      "loss": 0.4714,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3629148602485657,
      "rewards/margins": 1.5730308294296265,
      "rewards/rejected": -1.9359455108642578,
      "step": 14298
    },
    {
      "epoch": 1.871121288950683,
      "grad_norm": 11.871530532836914,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09141388535499573,
      "logits/rejected": -0.09263226389884949,
      "logps/chosen": -487.505126953125,
      "logps/rejected": -439.7529296875,
      "loss": 0.7802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.289921760559082,
      "rewards/margins": 0.609950065612793,
      "rewards/rejected": -1.899871826171875,
      "step": 14299
    },
    {
      "epoch": 1.8712521468880348,
      "grad_norm": 8.129412651062012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5507096648216248,
      "logits/rejected": -0.6314899325370789,
      "logps/chosen": -513.8993530273438,
      "logps/rejected": -472.11224365234375,
      "loss": 0.4572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.738994836807251,
      "rewards/margins": 0.6791779398918152,
      "rewards/rejected": -2.418172836303711,
      "step": 14300
    },
    {
      "epoch": 1.8713830048253866,
      "grad_norm": 9.672442436218262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43334677815437317,
      "logits/rejected": -0.2197866290807724,
      "logps/chosen": -465.19061279296875,
      "logps/rejected": -405.5904235839844,
      "loss": 0.4921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2938976287841797,
      "rewards/margins": 1.7181410789489746,
      "rewards/rejected": -3.0120387077331543,
      "step": 14301
    },
    {
      "epoch": 1.8715138627627383,
      "grad_norm": 7.2520880699157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0342741385102272,
      "logits/rejected": -0.2549440860748291,
      "logps/chosen": -522.1949462890625,
      "logps/rejected": -389.46331787109375,
      "loss": 0.2245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2576315402984619,
      "rewards/margins": 2.125978708267212,
      "rewards/rejected": -2.383610248565674,
      "step": 14302
    },
    {
      "epoch": 1.87164472070009,
      "grad_norm": 8.0104341506958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2545495927333832,
      "logits/rejected": -0.14771030843257904,
      "logps/chosen": -418.3595275878906,
      "logps/rejected": -484.1164855957031,
      "loss": 0.3024,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1091160774230957,
      "rewards/margins": 1.7534174919128418,
      "rewards/rejected": -2.8625338077545166,
      "step": 14303
    },
    {
      "epoch": 1.8717755786374417,
      "grad_norm": 5.8110761642456055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21248577535152435,
      "logits/rejected": -0.28617915511131287,
      "logps/chosen": -432.345947265625,
      "logps/rejected": -587.26513671875,
      "loss": 0.2235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.152608871459961,
      "rewards/margins": 1.4722933769226074,
      "rewards/rejected": -2.6249022483825684,
      "step": 14304
    },
    {
      "epoch": 1.8719064365747935,
      "grad_norm": 7.664027690887451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09113036841154099,
      "logits/rejected": -0.08327244967222214,
      "logps/chosen": -544.8198852539062,
      "logps/rejected": -452.80865478515625,
      "loss": 0.4239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7981035709381104,
      "rewards/margins": 1.564591407775879,
      "rewards/rejected": -2.3626949787139893,
      "step": 14305
    },
    {
      "epoch": 1.8720372945121453,
      "grad_norm": 12.415858268737793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01749487966299057,
      "logits/rejected": 0.017569057643413544,
      "logps/chosen": -329.81005859375,
      "logps/rejected": -410.5225830078125,
      "loss": 0.6242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1860531568527222,
      "rewards/margins": 1.5417654514312744,
      "rewards/rejected": -2.727818489074707,
      "step": 14306
    },
    {
      "epoch": 1.8721681524494969,
      "grad_norm": 3.6294496059417725,
      "learning_rate": 5e-06,
      "logits/chosen": 0.053592510521411896,
      "logits/rejected": -0.0818973034620285,
      "logps/chosen": -407.3323059082031,
      "logps/rejected": -449.2054138183594,
      "loss": 0.1284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11356811225414276,
      "rewards/margins": 2.7115070819854736,
      "rewards/rejected": -2.8250749111175537,
      "step": 14307
    },
    {
      "epoch": 1.8722990103868487,
      "grad_norm": 10.093308448791504,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4024699330329895,
      "logits/rejected": -0.28488728404045105,
      "logps/chosen": -427.7253112792969,
      "logps/rejected": -459.32421875,
      "loss": 0.5392,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1452155113220215,
      "rewards/margins": 0.9124865531921387,
      "rewards/rejected": -2.05770206451416,
      "step": 14308
    },
    {
      "epoch": 1.8724298683242004,
      "grad_norm": 12.747587203979492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21882948279380798,
      "logits/rejected": -0.1705470085144043,
      "logps/chosen": -541.5393676757812,
      "logps/rejected": -456.60162353515625,
      "loss": 0.6235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.715052843093872,
      "rewards/margins": 1.1395596265792847,
      "rewards/rejected": -2.8546125888824463,
      "step": 14309
    },
    {
      "epoch": 1.8725607262615522,
      "grad_norm": 7.15052604675293,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030314549803733826,
      "logits/rejected": -0.1970071792602539,
      "logps/chosen": -512.7236938476562,
      "logps/rejected": -670.4908447265625,
      "loss": 0.3423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7231457233428955,
      "rewards/margins": 1.6185716390609741,
      "rewards/rejected": -3.341717481613159,
      "step": 14310
    },
    {
      "epoch": 1.872691584198904,
      "grad_norm": 9.920995712280273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2742149233818054,
      "logits/rejected": -0.3488227128982544,
      "logps/chosen": -438.369873046875,
      "logps/rejected": -405.0995788574219,
      "loss": 0.5985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.105372190475464,
      "rewards/margins": 0.566405713558197,
      "rewards/rejected": -2.6717777252197266,
      "step": 14311
    },
    {
      "epoch": 1.8728224421362558,
      "grad_norm": 8.09037971496582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03591422736644745,
      "logits/rejected": -0.4350389242172241,
      "logps/chosen": -505.18072509765625,
      "logps/rejected": -308.28900146484375,
      "loss": 0.2545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.042706847190857,
      "rewards/margins": 2.2029244899749756,
      "rewards/rejected": -3.245631217956543,
      "step": 14312
    },
    {
      "epoch": 1.8729533000736076,
      "grad_norm": 14.291626930236816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.351723313331604,
      "logits/rejected": -0.08480852097272873,
      "logps/chosen": -289.9786682128906,
      "logps/rejected": -467.6307373046875,
      "loss": 0.952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2417171001434326,
      "rewards/margins": 0.5323959589004517,
      "rewards/rejected": -1.7741129398345947,
      "step": 14313
    },
    {
      "epoch": 1.8730841580109594,
      "grad_norm": 7.72575569152832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08244484663009644,
      "logits/rejected": 0.23173928260803223,
      "logps/chosen": -647.6170654296875,
      "logps/rejected": -601.0906982421875,
      "loss": 0.3589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9013057351112366,
      "rewards/margins": 1.2729438543319702,
      "rewards/rejected": -2.1742496490478516,
      "step": 14314
    },
    {
      "epoch": 1.8732150159483112,
      "grad_norm": 13.46472454071045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23281805217266083,
      "logits/rejected": -0.25369390845298767,
      "logps/chosen": -525.324462890625,
      "logps/rejected": -452.142333984375,
      "loss": 0.8438,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9212733507156372,
      "rewards/margins": 0.683779239654541,
      "rewards/rejected": -2.6050527095794678,
      "step": 14315
    },
    {
      "epoch": 1.873345873885663,
      "grad_norm": 8.66622257232666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10658672451972961,
      "logits/rejected": -0.20902247726917267,
      "logps/chosen": -404.96490478515625,
      "logps/rejected": -490.71875,
      "loss": 0.3671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9259254932403564,
      "rewards/margins": 1.718201756477356,
      "rewards/rejected": -2.644127368927002,
      "step": 14316
    },
    {
      "epoch": 1.8734767318230148,
      "grad_norm": 8.152877807617188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17356541752815247,
      "logits/rejected": -0.3421168029308319,
      "logps/chosen": -406.57855224609375,
      "logps/rejected": -426.7199401855469,
      "loss": 0.4036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9874426126480103,
      "rewards/margins": 1.1445657014846802,
      "rewards/rejected": -3.1320083141326904,
      "step": 14317
    },
    {
      "epoch": 1.8736075897603666,
      "grad_norm": 12.827865600585938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2634556293487549,
      "logits/rejected": -0.2730735242366791,
      "logps/chosen": -449.51373291015625,
      "logps/rejected": -395.3409423828125,
      "loss": 0.7502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1466089487075806,
      "rewards/margins": 1.2671865224838257,
      "rewards/rejected": -2.4137954711914062,
      "step": 14318
    },
    {
      "epoch": 1.8737384476977181,
      "grad_norm": 11.407510757446289,
      "learning_rate": 5e-06,
      "logits/chosen": -0.010253839194774628,
      "logits/rejected": -0.11463756114244461,
      "logps/chosen": -491.523681640625,
      "logps/rejected": -444.5132141113281,
      "loss": 0.569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5979852676391602,
      "rewards/margins": 0.8550500273704529,
      "rewards/rejected": -1.4530354738235474,
      "step": 14319
    },
    {
      "epoch": 1.87386930563507,
      "grad_norm": 5.060575008392334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27138665318489075,
      "logits/rejected": -0.5468136072158813,
      "logps/chosen": -346.967041015625,
      "logps/rejected": -314.5620422363281,
      "loss": 0.2094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8872572183609009,
      "rewards/margins": 2.183088779449463,
      "rewards/rejected": -3.0703461170196533,
      "step": 14320
    },
    {
      "epoch": 1.8740001635724217,
      "grad_norm": 12.616751670837402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.331587016582489,
      "logits/rejected": -0.3778396248817444,
      "logps/chosen": -628.6629028320312,
      "logps/rejected": -433.9133605957031,
      "loss": 0.5918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.520195722579956,
      "rewards/margins": 1.3534080982208252,
      "rewards/rejected": -2.8736038208007812,
      "step": 14321
    },
    {
      "epoch": 1.8741310215097733,
      "grad_norm": 6.396094799041748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5349580645561218,
      "logits/rejected": -0.21565169095993042,
      "logps/chosen": -357.6165771484375,
      "logps/rejected": -368.5615539550781,
      "loss": 0.3279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9987967014312744,
      "rewards/margins": 1.3200267553329468,
      "rewards/rejected": -2.3188233375549316,
      "step": 14322
    },
    {
      "epoch": 1.874261879447125,
      "grad_norm": 13.99363899230957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3590308427810669,
      "logits/rejected": -0.32421940565109253,
      "logps/chosen": -309.1013488769531,
      "logps/rejected": -311.10205078125,
      "loss": 0.8056,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2404447793960571,
      "rewards/margins": 0.09371928870677948,
      "rewards/rejected": -1.3341641426086426,
      "step": 14323
    },
    {
      "epoch": 1.8743927373844769,
      "grad_norm": 3.5303876399993896,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03654634207487106,
      "logits/rejected": -0.06438742578029633,
      "logps/chosen": -439.702392578125,
      "logps/rejected": -548.7326049804688,
      "loss": 0.1104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.699633002281189,
      "rewards/margins": 3.1175801753997803,
      "rewards/rejected": -3.8172130584716797,
      "step": 14324
    },
    {
      "epoch": 1.8745235953218287,
      "grad_norm": 8.962692260742188,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09886432439088821,
      "logits/rejected": -0.15283727645874023,
      "logps/chosen": -563.0089721679688,
      "logps/rejected": -464.7354736328125,
      "loss": 0.476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4066176414489746,
      "rewards/margins": 1.0097296237945557,
      "rewards/rejected": -2.4163472652435303,
      "step": 14325
    },
    {
      "epoch": 1.8746544532591805,
      "grad_norm": 9.126643180847168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10482243448495865,
      "logits/rejected": -0.19420725107192993,
      "logps/chosen": -460.3128967285156,
      "logps/rejected": -468.5159606933594,
      "loss": 0.4596,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8687641024589539,
      "rewards/margins": 0.6324789524078369,
      "rewards/rejected": -1.501242995262146,
      "step": 14326
    },
    {
      "epoch": 1.8747853111965322,
      "grad_norm": 10.457206726074219,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07196919620037079,
      "logits/rejected": -0.0024933181703090668,
      "logps/chosen": -618.5288696289062,
      "logps/rejected": -609.95263671875,
      "loss": 0.4891,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6938192844390869,
      "rewards/margins": 1.156926155090332,
      "rewards/rejected": -1.850745439529419,
      "step": 14327
    },
    {
      "epoch": 1.874916169133884,
      "grad_norm": 7.330129623413086,
      "learning_rate": 5e-06,
      "logits/chosen": 0.045702964067459106,
      "logits/rejected": 0.009870562702417374,
      "logps/chosen": -425.6890869140625,
      "logps/rejected": -488.19207763671875,
      "loss": 0.3063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.209817886352539,
      "rewards/margins": 1.2967792749404907,
      "rewards/rejected": -3.5065972805023193,
      "step": 14328
    },
    {
      "epoch": 1.8750470270712358,
      "grad_norm": 8.948907852172852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05455448850989342,
      "logits/rejected": -0.18069705367088318,
      "logps/chosen": -523.2708129882812,
      "logps/rejected": -506.4269104003906,
      "loss": 0.36,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.854841947555542,
      "rewards/margins": 1.8847239017486572,
      "rewards/rejected": -2.739565849304199,
      "step": 14329
    },
    {
      "epoch": 1.8751778850085876,
      "grad_norm": 5.383046627044678,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15001191198825836,
      "logits/rejected": -0.6717333793640137,
      "logps/chosen": -389.81756591796875,
      "logps/rejected": -399.67572021484375,
      "loss": 0.2685,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8332678079605103,
      "rewards/margins": 2.1213340759277344,
      "rewards/rejected": -2.954602003097534,
      "step": 14330
    },
    {
      "epoch": 1.8753087429459394,
      "grad_norm": 14.92184066772461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13322296738624573,
      "logits/rejected": -0.2856336832046509,
      "logps/chosen": -702.2933959960938,
      "logps/rejected": -477.452392578125,
      "loss": 0.4886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.206066608428955,
      "rewards/margins": 1.4206643104553223,
      "rewards/rejected": -3.6267309188842773,
      "step": 14331
    },
    {
      "epoch": 1.8754396008832912,
      "grad_norm": 10.506754875183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5457698106765747,
      "logits/rejected": -0.4944440722465515,
      "logps/chosen": -324.20123291015625,
      "logps/rejected": -442.5607604980469,
      "loss": 0.4562,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1552635431289673,
      "rewards/margins": 2.1929190158843994,
      "rewards/rejected": -3.3481829166412354,
      "step": 14332
    },
    {
      "epoch": 1.875570458820643,
      "grad_norm": 12.860750198364258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3569239377975464,
      "logits/rejected": -0.23697403073310852,
      "logps/chosen": -434.7141418457031,
      "logps/rejected": -668.124755859375,
      "loss": 0.6296,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1859811544418335,
      "rewards/margins": 2.0056753158569336,
      "rewards/rejected": -3.1916565895080566,
      "step": 14333
    },
    {
      "epoch": 1.8757013167579946,
      "grad_norm": 10.535622596740723,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28461700677871704,
      "logits/rejected": -0.33128708600997925,
      "logps/chosen": -615.3591918945312,
      "logps/rejected": -575.6407470703125,
      "loss": 0.5208,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0231878757476807,
      "rewards/margins": 1.1231682300567627,
      "rewards/rejected": -2.1463558673858643,
      "step": 14334
    },
    {
      "epoch": 1.8758321746953464,
      "grad_norm": 9.59652328491211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30433952808380127,
      "logits/rejected": -0.3986950218677521,
      "logps/chosen": -407.1643981933594,
      "logps/rejected": -450.1820373535156,
      "loss": 0.6281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2294663190841675,
      "rewards/margins": 1.337243676185608,
      "rewards/rejected": -2.5667099952697754,
      "step": 14335
    },
    {
      "epoch": 1.8759630326326981,
      "grad_norm": 9.238443374633789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.121798574924469,
      "logits/rejected": -0.13429315388202667,
      "logps/chosen": -354.59283447265625,
      "logps/rejected": -482.08111572265625,
      "loss": 0.3966,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6900936365127563,
      "rewards/margins": 1.7699836492538452,
      "rewards/rejected": -3.4600775241851807,
      "step": 14336
    },
    {
      "epoch": 1.87609389057005,
      "grad_norm": 5.85640811920166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34611231088638306,
      "logits/rejected": -0.5823107957839966,
      "logps/chosen": -395.8224182128906,
      "logps/rejected": -426.0888671875,
      "loss": 0.2493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.616353988647461,
      "rewards/margins": 1.897491216659546,
      "rewards/rejected": -3.513845443725586,
      "step": 14337
    },
    {
      "epoch": 1.8762247485074015,
      "grad_norm": 6.341434955596924,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48691526055336,
      "logits/rejected": -0.46285295486450195,
      "logps/chosen": -460.23699951171875,
      "logps/rejected": -776.18603515625,
      "loss": 0.3483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2900497913360596,
      "rewards/margins": 2.4300925731658936,
      "rewards/rejected": -3.7201426029205322,
      "step": 14338
    },
    {
      "epoch": 1.8763556064447533,
      "grad_norm": 5.846762657165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3893740475177765,
      "logits/rejected": -0.43505752086639404,
      "logps/chosen": -312.7398376464844,
      "logps/rejected": -392.8948669433594,
      "loss": 0.3143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.892616868019104,
      "rewards/margins": 1.324134349822998,
      "rewards/rejected": -2.2167510986328125,
      "step": 14339
    },
    {
      "epoch": 1.876486464382105,
      "grad_norm": 5.947883605957031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2933087646961212,
      "logits/rejected": -0.32980877161026,
      "logps/chosen": -513.6568603515625,
      "logps/rejected": -530.572998046875,
      "loss": 0.244,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7536430954933167,
      "rewards/margins": 2.3483002185821533,
      "rewards/rejected": -3.101943254470825,
      "step": 14340
    },
    {
      "epoch": 1.8766173223194569,
      "grad_norm": 6.565690517425537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3561599552631378,
      "logits/rejected": -0.47712385654449463,
      "logps/chosen": -403.64617919921875,
      "logps/rejected": -346.8013610839844,
      "loss": 0.3826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8174035549163818,
      "rewards/margins": 1.8647305965423584,
      "rewards/rejected": -3.6821343898773193,
      "step": 14341
    },
    {
      "epoch": 1.8767481802568087,
      "grad_norm": 13.041616439819336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13337068259716034,
      "logits/rejected": -0.20763768255710602,
      "logps/chosen": -581.9140014648438,
      "logps/rejected": -636.3966064453125,
      "loss": 0.6514,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.537743866443634,
      "rewards/margins": 2.1792774200439453,
      "rewards/rejected": -2.7170212268829346,
      "step": 14342
    },
    {
      "epoch": 1.8768790381941605,
      "grad_norm": 8.364558219909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04559333622455597,
      "logits/rejected": -0.16581594944000244,
      "logps/chosen": -418.4173278808594,
      "logps/rejected": -317.9892578125,
      "loss": 0.3803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.060593843460083,
      "rewards/margins": 1.6573153734207153,
      "rewards/rejected": -2.7179088592529297,
      "step": 14343
    },
    {
      "epoch": 1.8770098961315123,
      "grad_norm": 7.982973575592041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16766352951526642,
      "logits/rejected": -0.2869569957256317,
      "logps/chosen": -417.62078857421875,
      "logps/rejected": -440.309814453125,
      "loss": 0.3183,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.067743182182312,
      "rewards/margins": 2.429532289505005,
      "rewards/rejected": -3.4972755908966064,
      "step": 14344
    },
    {
      "epoch": 1.877140754068864,
      "grad_norm": 5.689337253570557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18218810856342316,
      "logits/rejected": -0.03335566446185112,
      "logps/chosen": -701.5466918945312,
      "logps/rejected": -514.95458984375,
      "loss": 0.1651,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.820503294467926,
      "rewards/margins": 2.56197190284729,
      "rewards/rejected": -3.382474899291992,
      "step": 14345
    },
    {
      "epoch": 1.8772716120062158,
      "grad_norm": 6.895954608917236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48342472314834595,
      "logits/rejected": -0.345057874917984,
      "logps/chosen": -414.9669494628906,
      "logps/rejected": -385.5376281738281,
      "loss": 0.4141,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.065014123916626,
      "rewards/margins": 1.3580443859100342,
      "rewards/rejected": -2.42305850982666,
      "step": 14346
    },
    {
      "epoch": 1.8774024699435676,
      "grad_norm": 9.697564125061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32442015409469604,
      "logits/rejected": -0.34491899609565735,
      "logps/chosen": -580.6420288085938,
      "logps/rejected": -570.8919067382812,
      "loss": 0.3345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7436174750328064,
      "rewards/margins": 2.139833927154541,
      "rewards/rejected": -2.883451223373413,
      "step": 14347
    },
    {
      "epoch": 1.8775333278809194,
      "grad_norm": 9.741817474365234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23806777596473694,
      "logits/rejected": -0.09181161224842072,
      "logps/chosen": -437.95062255859375,
      "logps/rejected": -420.4801025390625,
      "loss": 0.5355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8372533917427063,
      "rewards/margins": 1.3669557571411133,
      "rewards/rejected": -2.204209089279175,
      "step": 14348
    },
    {
      "epoch": 1.877664185818271,
      "grad_norm": 9.960545539855957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17940697073936462,
      "logits/rejected": -0.07560145854949951,
      "logps/chosen": -394.28143310546875,
      "logps/rejected": -427.53656005859375,
      "loss": 0.513,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2038425207138062,
      "rewards/margins": 1.2157509326934814,
      "rewards/rejected": -2.419593334197998,
      "step": 14349
    },
    {
      "epoch": 1.8777950437556228,
      "grad_norm": 8.166462898254395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09283143281936646,
      "logits/rejected": -0.32484591007232666,
      "logps/chosen": -604.995849609375,
      "logps/rejected": -416.81683349609375,
      "loss": 0.4598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8690209984779358,
      "rewards/margins": 1.0831258296966553,
      "rewards/rejected": -1.9521470069885254,
      "step": 14350
    },
    {
      "epoch": 1.8779259016929746,
      "grad_norm": 3.507927656173706,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15414439141750336,
      "logits/rejected": -0.07810509204864502,
      "logps/chosen": -470.5553894042969,
      "logps/rejected": -475.04229736328125,
      "loss": 0.1481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2090106010437012,
      "rewards/margins": 2.2244372367858887,
      "rewards/rejected": -3.4334475994110107,
      "step": 14351
    },
    {
      "epoch": 1.8780567596303264,
      "grad_norm": 11.76955795288086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30457302927970886,
      "logits/rejected": -0.4064878225326538,
      "logps/chosen": -422.86761474609375,
      "logps/rejected": -436.0648193359375,
      "loss": 0.4418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7176789045333862,
      "rewards/margins": 1.3010711669921875,
      "rewards/rejected": -3.0187501907348633,
      "step": 14352
    },
    {
      "epoch": 1.878187617567678,
      "grad_norm": 12.255203247070312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2218070924282074,
      "logits/rejected": -0.18209496140480042,
      "logps/chosen": -664.8820190429688,
      "logps/rejected": -600.7128295898438,
      "loss": 0.589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2243423461914062,
      "rewards/margins": 0.9122574329376221,
      "rewards/rejected": -2.1366000175476074,
      "step": 14353
    },
    {
      "epoch": 1.8783184755050297,
      "grad_norm": 8.021682739257812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3079410195350647,
      "logits/rejected": -0.3674300014972687,
      "logps/chosen": -389.32861328125,
      "logps/rejected": -496.36480712890625,
      "loss": 0.3106,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4259291887283325,
      "rewards/margins": 2.5922420024871826,
      "rewards/rejected": -4.018171310424805,
      "step": 14354
    },
    {
      "epoch": 1.8784493334423815,
      "grad_norm": 9.150439262390137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38191527128219604,
      "logits/rejected": -0.46868979930877686,
      "logps/chosen": -384.4811706542969,
      "logps/rejected": -381.9008483886719,
      "loss": 0.4126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.191779375076294,
      "rewards/margins": 1.0934157371520996,
      "rewards/rejected": -2.2851948738098145,
      "step": 14355
    },
    {
      "epoch": 1.8785801913797333,
      "grad_norm": 8.325366020202637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6120893359184265,
      "logits/rejected": -0.18163761496543884,
      "logps/chosen": -270.3717346191406,
      "logps/rejected": -342.6762390136719,
      "loss": 0.3911,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8968244194984436,
      "rewards/margins": 1.3566354513168335,
      "rewards/rejected": -2.253459930419922,
      "step": 14356
    },
    {
      "epoch": 1.878711049317085,
      "grad_norm": 8.803335189819336,
      "learning_rate": 5e-06,
      "logits/chosen": 0.045705199241638184,
      "logits/rejected": -0.14716021716594696,
      "logps/chosen": -599.3045654296875,
      "logps/rejected": -529.9337768554688,
      "loss": 0.4431,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7788101434707642,
      "rewards/margins": 1.5739105939865112,
      "rewards/rejected": -2.3527209758758545,
      "step": 14357
    },
    {
      "epoch": 1.878841907254437,
      "grad_norm": 5.383331775665283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.057231105864048004,
      "logits/rejected": -0.18202711641788483,
      "logps/chosen": -493.9908142089844,
      "logps/rejected": -486.4629821777344,
      "loss": 0.1858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6546962857246399,
      "rewards/margins": 2.0273234844207764,
      "rewards/rejected": -2.6820194721221924,
      "step": 14358
    },
    {
      "epoch": 1.8789727651917887,
      "grad_norm": 8.476258277893066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3784419298171997,
      "logits/rejected": -0.3123549520969391,
      "logps/chosen": -370.1200256347656,
      "logps/rejected": -300.0358581542969,
      "loss": 0.5043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7605577707290649,
      "rewards/margins": 0.8431362509727478,
      "rewards/rejected": -1.603693962097168,
      "step": 14359
    },
    {
      "epoch": 1.8791036231291405,
      "grad_norm": 5.2392258644104,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20148777961730957,
      "logits/rejected": -0.35532909631729126,
      "logps/chosen": -641.291259765625,
      "logps/rejected": -442.1092224121094,
      "loss": 0.1718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8880471587181091,
      "rewards/margins": 2.242452383041382,
      "rewards/rejected": -3.1304993629455566,
      "step": 14360
    },
    {
      "epoch": 1.8792344810664923,
      "grad_norm": 5.529238224029541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01505633257329464,
      "logits/rejected": -0.3516941964626312,
      "logps/chosen": -510.245361328125,
      "logps/rejected": -399.8135986328125,
      "loss": 0.2307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4544179439544678,
      "rewards/margins": 1.7220968008041382,
      "rewards/rejected": -3.1765148639678955,
      "step": 14361
    },
    {
      "epoch": 1.879365339003844,
      "grad_norm": 8.04856014251709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2636703550815582,
      "logits/rejected": -0.23887574672698975,
      "logps/chosen": -368.4045104980469,
      "logps/rejected": -526.87548828125,
      "loss": 0.345,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4989567995071411,
      "rewards/margins": 2.3467414379119873,
      "rewards/rejected": -3.845698356628418,
      "step": 14362
    },
    {
      "epoch": 1.8794961969411959,
      "grad_norm": 9.721802711486816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16516906023025513,
      "logits/rejected": -0.4940221607685089,
      "logps/chosen": -587.091796875,
      "logps/rejected": -567.7503051757812,
      "loss": 0.4668,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3478484153747559,
      "rewards/margins": 1.3690025806427002,
      "rewards/rejected": -2.716850996017456,
      "step": 14363
    },
    {
      "epoch": 1.8796270548785476,
      "grad_norm": 8.383743286132812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45331937074661255,
      "logits/rejected": -0.3036576211452484,
      "logps/chosen": -336.554443359375,
      "logps/rejected": -383.7303161621094,
      "loss": 0.4904,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3972575664520264,
      "rewards/margins": 1.0732896327972412,
      "rewards/rejected": -2.4705474376678467,
      "step": 14364
    },
    {
      "epoch": 1.8797579128158992,
      "grad_norm": 10.746651649475098,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1004212349653244,
      "logits/rejected": 0.004261739552021027,
      "logps/chosen": -350.5400695800781,
      "logps/rejected": -372.1399841308594,
      "loss": 0.4187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.069342017173767,
      "rewards/margins": 1.5636602640151978,
      "rewards/rejected": -2.633002281188965,
      "step": 14365
    },
    {
      "epoch": 1.879888770753251,
      "grad_norm": 11.175326347351074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20736485719680786,
      "logits/rejected": -0.26045939326286316,
      "logps/chosen": -381.06011962890625,
      "logps/rejected": -474.22869873046875,
      "loss": 0.6311,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.361533284187317,
      "rewards/margins": 1.746989369392395,
      "rewards/rejected": -3.108522653579712,
      "step": 14366
    },
    {
      "epoch": 1.8800196286906028,
      "grad_norm": 12.855775833129883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1831532120704651,
      "logits/rejected": -0.09612931311130524,
      "logps/chosen": -440.4417724609375,
      "logps/rejected": -328.7693176269531,
      "loss": 0.6326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.279307246208191,
      "rewards/margins": 1.0166990756988525,
      "rewards/rejected": -2.296006202697754,
      "step": 14367
    },
    {
      "epoch": 1.8801504866279544,
      "grad_norm": 10.995251655578613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17387081682682037,
      "logits/rejected": 0.022806206718087196,
      "logps/chosen": -399.8298645019531,
      "logps/rejected": -474.3218078613281,
      "loss": 0.527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8697401285171509,
      "rewards/margins": 0.8303704857826233,
      "rewards/rejected": -2.700110673904419,
      "step": 14368
    },
    {
      "epoch": 1.8802813445653062,
      "grad_norm": 8.897774696350098,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2687123119831085,
      "logits/rejected": -0.5224127769470215,
      "logps/chosen": -303.7215881347656,
      "logps/rejected": -500.0452880859375,
      "loss": 0.345,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.471274971961975,
      "rewards/margins": 2.662775993347168,
      "rewards/rejected": -4.134050369262695,
      "step": 14369
    },
    {
      "epoch": 1.880412202502658,
      "grad_norm": 13.048290252685547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1773163229227066,
      "logits/rejected": -0.11219925433397293,
      "logps/chosen": -520.3150634765625,
      "logps/rejected": -602.3594970703125,
      "loss": 0.6279,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1816695928573608,
      "rewards/margins": 1.4193168878555298,
      "rewards/rejected": -2.6009864807128906,
      "step": 14370
    },
    {
      "epoch": 1.8805430604400097,
      "grad_norm": 11.36382007598877,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2189716100692749,
      "logits/rejected": -0.2138834297657013,
      "logps/chosen": -491.0928955078125,
      "logps/rejected": -500.4027099609375,
      "loss": 0.4122,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2496354579925537,
      "rewards/margins": 1.5633326768875122,
      "rewards/rejected": -2.8129680156707764,
      "step": 14371
    },
    {
      "epoch": 1.8806739183773615,
      "grad_norm": 8.053237915039062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15573915839195251,
      "logits/rejected": -0.3234313726425171,
      "logps/chosen": -590.1618041992188,
      "logps/rejected": -516.18310546875,
      "loss": 0.3164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9348232746124268,
      "rewards/margins": 1.9144740104675293,
      "rewards/rejected": -2.849297285079956,
      "step": 14372
    },
    {
      "epoch": 1.8808047763147133,
      "grad_norm": 8.079996109008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08305369317531586,
      "logits/rejected": -0.2521708011627197,
      "logps/chosen": -221.76914978027344,
      "logps/rejected": -387.659912109375,
      "loss": 0.3608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4507191181182861,
      "rewards/margins": 1.8691374063491821,
      "rewards/rejected": -3.319856643676758,
      "step": 14373
    },
    {
      "epoch": 1.8809356342520651,
      "grad_norm": 7.2517619132995605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09609324485063553,
      "logits/rejected": 0.016636710613965988,
      "logps/chosen": -559.8575439453125,
      "logps/rejected": -633.720458984375,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.751264214515686,
      "rewards/margins": 2.007998466491699,
      "rewards/rejected": -2.759262800216675,
      "step": 14374
    },
    {
      "epoch": 1.881066492189417,
      "grad_norm": 10.39414119720459,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17898641526699066,
      "logits/rejected": -0.2961418032646179,
      "logps/chosen": -322.1284484863281,
      "logps/rejected": -336.3755798339844,
      "loss": 0.4057,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2843736410140991,
      "rewards/margins": 2.432889461517334,
      "rewards/rejected": -3.7172634601593018,
      "step": 14375
    },
    {
      "epoch": 1.8811973501267687,
      "grad_norm": 5.520267009735107,
      "learning_rate": 5e-06,
      "logits/chosen": 0.046089429408311844,
      "logits/rejected": -0.1548973023891449,
      "logps/chosen": -512.045654296875,
      "logps/rejected": -621.5132446289062,
      "loss": 0.2678,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.71199631690979,
      "rewards/margins": 2.195483446121216,
      "rewards/rejected": -2.9074795246124268,
      "step": 14376
    },
    {
      "epoch": 1.8813282080641205,
      "grad_norm": 5.360243797302246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2924968898296356,
      "logits/rejected": -0.5978181958198547,
      "logps/chosen": -405.1085205078125,
      "logps/rejected": -834.2493286132812,
      "loss": 0.2831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.567563772201538,
      "rewards/margins": 2.1472363471984863,
      "rewards/rejected": -3.7148003578186035,
      "step": 14377
    },
    {
      "epoch": 1.8814590660014723,
      "grad_norm": 13.07548713684082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3574005365371704,
      "logits/rejected": -0.36462104320526123,
      "logps/chosen": -440.5977783203125,
      "logps/rejected": -303.8619079589844,
      "loss": 0.6859,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.5352983474731445,
      "rewards/margins": 0.9322454929351807,
      "rewards/rejected": -3.467543840408325,
      "step": 14378
    },
    {
      "epoch": 1.881589923938824,
      "grad_norm": 5.529694080352783,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1936282366514206,
      "logits/rejected": 0.2728293538093567,
      "logps/chosen": -396.8207092285156,
      "logps/rejected": -476.6944580078125,
      "loss": 0.1864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38530826568603516,
      "rewards/margins": 2.667665958404541,
      "rewards/rejected": -3.0529744625091553,
      "step": 14379
    },
    {
      "epoch": 1.8817207818761756,
      "grad_norm": 13.636592864990234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07660207897424698,
      "logits/rejected": -0.16539934277534485,
      "logps/chosen": -505.0606994628906,
      "logps/rejected": -483.2351379394531,
      "loss": 0.7239,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.9666259288787842,
      "rewards/margins": 0.15640486776828766,
      "rewards/rejected": -2.1230309009552,
      "step": 14380
    },
    {
      "epoch": 1.8818516398135274,
      "grad_norm": 12.420980453491211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0028092674911022186,
      "logits/rejected": 0.11606067419052124,
      "logps/chosen": -439.26220703125,
      "logps/rejected": -468.76898193359375,
      "loss": 0.6284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6359642148017883,
      "rewards/margins": 1.429956078529358,
      "rewards/rejected": -2.065920352935791,
      "step": 14381
    },
    {
      "epoch": 1.8819824977508792,
      "grad_norm": 12.577402114868164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38841724395751953,
      "logits/rejected": -0.3641566038131714,
      "logps/chosen": -370.54931640625,
      "logps/rejected": -307.52001953125,
      "loss": 0.5901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.068302869796753,
      "rewards/margins": 1.0421104431152344,
      "rewards/rejected": -2.1104135513305664,
      "step": 14382
    },
    {
      "epoch": 1.882113355688231,
      "grad_norm": 18.810800552368164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.349587082862854,
      "logits/rejected": -0.1710803508758545,
      "logps/chosen": -578.40185546875,
      "logps/rejected": -422.9173583984375,
      "loss": 0.9787,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5660699605941772,
      "rewards/margins": 0.2325497716665268,
      "rewards/rejected": -1.7986196279525757,
      "step": 14383
    },
    {
      "epoch": 1.8822442136255826,
      "grad_norm": 7.913538932800293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33769023418426514,
      "logits/rejected": -0.0534566268324852,
      "logps/chosen": -447.2738952636719,
      "logps/rejected": -401.88232421875,
      "loss": 0.4025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5398876667022705,
      "rewards/margins": 1.6102936267852783,
      "rewards/rejected": -2.150181293487549,
      "step": 14384
    },
    {
      "epoch": 1.8823750715629344,
      "grad_norm": 7.8791656494140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1467566192150116,
      "logits/rejected": -0.34561553597450256,
      "logps/chosen": -402.52899169921875,
      "logps/rejected": -505.13262939453125,
      "loss": 0.3434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6599408984184265,
      "rewards/margins": 2.5130457878112793,
      "rewards/rejected": -3.1729865074157715,
      "step": 14385
    },
    {
      "epoch": 1.8825059295002862,
      "grad_norm": 8.060165405273438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6435004472732544,
      "logits/rejected": -0.3916858434677124,
      "logps/chosen": -377.9896545410156,
      "logps/rejected": -493.58843994140625,
      "loss": 0.2913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7546093463897705,
      "rewards/margins": 1.6987621784210205,
      "rewards/rejected": -2.453371524810791,
      "step": 14386
    },
    {
      "epoch": 1.882636787437638,
      "grad_norm": 7.472398281097412,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34877991676330566,
      "logits/rejected": -0.15560637414455414,
      "logps/chosen": -535.416748046875,
      "logps/rejected": -315.1915283203125,
      "loss": 0.3295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9536111950874329,
      "rewards/margins": 2.128577709197998,
      "rewards/rejected": -3.082188844680786,
      "step": 14387
    },
    {
      "epoch": 1.8827676453749898,
      "grad_norm": 13.051136016845703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29095739126205444,
      "logits/rejected": 0.052244722843170166,
      "logps/chosen": -405.72735595703125,
      "logps/rejected": -496.7264099121094,
      "loss": 0.5243,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2418241500854492,
      "rewards/margins": 0.9148256778717041,
      "rewards/rejected": -2.156649589538574,
      "step": 14388
    },
    {
      "epoch": 1.8828985033123415,
      "grad_norm": 6.5174241065979,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05196421593427658,
      "logits/rejected": -0.3057364821434021,
      "logps/chosen": -596.11962890625,
      "logps/rejected": -398.3625793457031,
      "loss": 0.2651,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.020431637763977,
      "rewards/margins": 2.463810443878174,
      "rewards/rejected": -3.4842419624328613,
      "step": 14389
    },
    {
      "epoch": 1.8830293612496933,
      "grad_norm": 6.167669296264648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20073363184928894,
      "logits/rejected": -0.07241424918174744,
      "logps/chosen": -339.70599365234375,
      "logps/rejected": -610.3157348632812,
      "loss": 0.1839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9276497960090637,
      "rewards/margins": 2.6655001640319824,
      "rewards/rejected": -3.5931503772735596,
      "step": 14390
    },
    {
      "epoch": 1.8831602191870451,
      "grad_norm": 11.396858215332031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0544591061770916,
      "logits/rejected": -0.0019526854157447815,
      "logps/chosen": -358.68157958984375,
      "logps/rejected": -488.8589172363281,
      "loss": 0.4664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9641743898391724,
      "rewards/margins": 1.2108203172683716,
      "rewards/rejected": -2.174994707107544,
      "step": 14391
    },
    {
      "epoch": 1.883291077124397,
      "grad_norm": 7.1378936767578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2034628540277481,
      "logits/rejected": -0.42633694410324097,
      "logps/chosen": -571.8356323242188,
      "logps/rejected": -403.9932556152344,
      "loss": 0.1878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48582571744918823,
      "rewards/margins": 2.6324057579040527,
      "rewards/rejected": -3.118231773376465,
      "step": 14392
    },
    {
      "epoch": 1.8834219350617487,
      "grad_norm": 8.768528938293457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40722447633743286,
      "logits/rejected": -0.6791538000106812,
      "logps/chosen": -371.34930419921875,
      "logps/rejected": -404.91986083984375,
      "loss": 0.4017,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7528202533721924,
      "rewards/margins": 1.5586140155792236,
      "rewards/rejected": -2.311434268951416,
      "step": 14393
    },
    {
      "epoch": 1.8835527929991005,
      "grad_norm": 5.097569942474365,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08633295446634293,
      "logits/rejected": 0.09087269008159637,
      "logps/chosen": -522.8749389648438,
      "logps/rejected": -451.36883544921875,
      "loss": 0.1622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5999571084976196,
      "rewards/margins": 2.4279613494873047,
      "rewards/rejected": -3.0279183387756348,
      "step": 14394
    },
    {
      "epoch": 1.883683650936452,
      "grad_norm": 9.11927318572998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3805650770664215,
      "logits/rejected": -0.5411757826805115,
      "logps/chosen": -499.9662170410156,
      "logps/rejected": -582.0948486328125,
      "loss": 0.513,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.057807207107544,
      "rewards/margins": 1.5695804357528687,
      "rewards/rejected": -2.627387762069702,
      "step": 14395
    },
    {
      "epoch": 1.8838145088738039,
      "grad_norm": 15.003437995910645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3759962022304535,
      "logits/rejected": -0.2586935758590698,
      "logps/chosen": -500.3784484863281,
      "logps/rejected": -477.108154296875,
      "loss": 0.949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6346709728240967,
      "rewards/margins": -0.257811963558197,
      "rewards/rejected": -1.3768589496612549,
      "step": 14396
    },
    {
      "epoch": 1.8839453668111557,
      "grad_norm": 6.4487080574035645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5258141756057739,
      "logits/rejected": -0.5107753276824951,
      "logps/chosen": -504.8357238769531,
      "logps/rejected": -445.67822265625,
      "loss": 0.2313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5789221525192261,
      "rewards/margins": 2.053292751312256,
      "rewards/rejected": -2.6322150230407715,
      "step": 14397
    },
    {
      "epoch": 1.8840762247485074,
      "grad_norm": 8.177131652832031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36373087763786316,
      "logits/rejected": -0.2876077890396118,
      "logps/chosen": -684.540771484375,
      "logps/rejected": -492.3941955566406,
      "loss": 0.2883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39467355608940125,
      "rewards/margins": 1.9010624885559082,
      "rewards/rejected": -2.2957358360290527,
      "step": 14398
    },
    {
      "epoch": 1.884207082685859,
      "grad_norm": 11.324761390686035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3267155587673187,
      "logits/rejected": -0.1564437299966812,
      "logps/chosen": -443.330810546875,
      "logps/rejected": -343.8765869140625,
      "loss": 0.4866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3514259457588196,
      "rewards/margins": 1.3693052530288696,
      "rewards/rejected": -1.7207311391830444,
      "step": 14399
    },
    {
      "epoch": 1.8843379406232108,
      "grad_norm": 5.323622226715088,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2188381552696228,
      "logits/rejected": -0.298647940158844,
      "logps/chosen": -622.6017456054688,
      "logps/rejected": -464.9340515136719,
      "loss": 0.3375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5740873217582703,
      "rewards/margins": 1.5710161924362183,
      "rewards/rejected": -2.1451034545898438,
      "step": 14400
    },
    {
      "epoch": 1.8844687985605626,
      "grad_norm": 7.691006183624268,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18024484813213348,
      "logits/rejected": -0.21230599284172058,
      "logps/chosen": -606.6510009765625,
      "logps/rejected": -477.3645324707031,
      "loss": 0.3928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5860170125961304,
      "rewards/margins": 1.4213236570358276,
      "rewards/rejected": -2.007340669631958,
      "step": 14401
    },
    {
      "epoch": 1.8845996564979144,
      "grad_norm": 8.959442138671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18106964230537415,
      "logits/rejected": -0.14124134182929993,
      "logps/chosen": -433.5922546386719,
      "logps/rejected": -564.4020385742188,
      "loss": 0.4751,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2857721447944641,
      "rewards/margins": 0.7489474415779114,
      "rewards/rejected": -1.0347195863723755,
      "step": 14402
    },
    {
      "epoch": 1.8847305144352662,
      "grad_norm": 8.949511528015137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3221786916255951,
      "logits/rejected": -0.4519018828868866,
      "logps/chosen": -400.4267272949219,
      "logps/rejected": -470.8142395019531,
      "loss": 0.4897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8740379214286804,
      "rewards/margins": 1.5801231861114502,
      "rewards/rejected": -2.4541609287261963,
      "step": 14403
    },
    {
      "epoch": 1.884861372372618,
      "grad_norm": 8.2672758102417,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32299691438674927,
      "logits/rejected": -0.49504172801971436,
      "logps/chosen": -497.5287780761719,
      "logps/rejected": -439.109375,
      "loss": 0.3716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6073662042617798,
      "rewards/margins": 1.5214569568634033,
      "rewards/rejected": -2.1288232803344727,
      "step": 14404
    },
    {
      "epoch": 1.8849922303099698,
      "grad_norm": 7.315439224243164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2874211370944977,
      "logits/rejected": -0.10433337837457657,
      "logps/chosen": -546.823974609375,
      "logps/rejected": -567.3895263671875,
      "loss": 0.2223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36682718992233276,
      "rewards/margins": 2.1972384452819824,
      "rewards/rejected": -2.56406569480896,
      "step": 14405
    },
    {
      "epoch": 1.8851230882473216,
      "grad_norm": 9.253236770629883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019036628305912018,
      "logits/rejected": -0.12516739964485168,
      "logps/chosen": -489.3178405761719,
      "logps/rejected": -521.7863159179688,
      "loss": 0.3829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2112106084823608,
      "rewards/margins": 2.037553548812866,
      "rewards/rejected": -3.2487640380859375,
      "step": 14406
    },
    {
      "epoch": 1.8852539461846733,
      "grad_norm": 7.820230484008789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26970183849334717,
      "logits/rejected": -0.30297717452049255,
      "logps/chosen": -287.3482666015625,
      "logps/rejected": -350.1120910644531,
      "loss": 0.4603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0377575159072876,
      "rewards/margins": 2.7368736267089844,
      "rewards/rejected": -3.7746310234069824,
      "step": 14407
    },
    {
      "epoch": 1.8853848041220251,
      "grad_norm": 8.231278419494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15521100163459778,
      "logits/rejected": -0.3355698585510254,
      "logps/chosen": -311.32421875,
      "logps/rejected": -361.88861083984375,
      "loss": 0.3625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7322315573692322,
      "rewards/margins": 1.6368653774261475,
      "rewards/rejected": -2.3690967559814453,
      "step": 14408
    },
    {
      "epoch": 1.885515662059377,
      "grad_norm": 7.856188774108887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24891157448291779,
      "logits/rejected": -0.5901057124137878,
      "logps/chosen": -522.3765258789062,
      "logps/rejected": -368.7228088378906,
      "loss": 0.2157,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7600030899047852,
      "rewards/margins": 2.63612699508667,
      "rewards/rejected": -3.396130084991455,
      "step": 14409
    },
    {
      "epoch": 1.8856465199967285,
      "grad_norm": 5.620047569274902,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47711652517318726,
      "logits/rejected": -0.37556028366088867,
      "logps/chosen": -378.4385986328125,
      "logps/rejected": -507.6307373046875,
      "loss": 0.2329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.856497585773468,
      "rewards/margins": 1.9530656337738037,
      "rewards/rejected": -2.809562921524048,
      "step": 14410
    },
    {
      "epoch": 1.8857773779340803,
      "grad_norm": 14.61277961730957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.317905068397522,
      "logits/rejected": -0.5219878554344177,
      "logps/chosen": -366.6322937011719,
      "logps/rejected": -506.98468017578125,
      "loss": 0.8431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5110437870025635,
      "rewards/margins": 0.5195939540863037,
      "rewards/rejected": -3.0306379795074463,
      "step": 14411
    },
    {
      "epoch": 1.885908235871432,
      "grad_norm": 5.690517425537109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16836202144622803,
      "logits/rejected": -0.5150619149208069,
      "logps/chosen": -376.570556640625,
      "logps/rejected": -372.95196533203125,
      "loss": 0.2077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9938808679580688,
      "rewards/margins": 2.5780794620513916,
      "rewards/rejected": -3.57196044921875,
      "step": 14412
    },
    {
      "epoch": 1.8860390938087839,
      "grad_norm": 12.570402145385742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3313470184803009,
      "logits/rejected": -0.00010789930820465088,
      "logps/chosen": -380.47064208984375,
      "logps/rejected": -403.201171875,
      "loss": 0.6189,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6994572281837463,
      "rewards/margins": 0.7401098012924194,
      "rewards/rejected": -1.4395670890808105,
      "step": 14413
    },
    {
      "epoch": 1.8861699517461354,
      "grad_norm": 8.27002239227295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5066595673561096,
      "logits/rejected": -0.5134456157684326,
      "logps/chosen": -441.7318115234375,
      "logps/rejected": -474.9389343261719,
      "loss": 0.3793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.945855438709259,
      "rewards/margins": 2.2817020416259766,
      "rewards/rejected": -3.2275571823120117,
      "step": 14414
    },
    {
      "epoch": 1.8863008096834872,
      "grad_norm": 9.793867111206055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3645571172237396,
      "logits/rejected": -0.5202459096908569,
      "logps/chosen": -495.42205810546875,
      "logps/rejected": -422.7960205078125,
      "loss": 0.2465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6995616555213928,
      "rewards/margins": 2.6073038578033447,
      "rewards/rejected": -3.3068652153015137,
      "step": 14415
    },
    {
      "epoch": 1.886431667620839,
      "grad_norm": 5.1501688957214355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02771236002445221,
      "logits/rejected": -0.19445094466209412,
      "logps/chosen": -358.2127685546875,
      "logps/rejected": -457.5460205078125,
      "loss": 0.196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5813353061676025,
      "rewards/margins": 1.8999379873275757,
      "rewards/rejected": -3.4812731742858887,
      "step": 14416
    },
    {
      "epoch": 1.8865625255581908,
      "grad_norm": 8.953707695007324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0647495836019516,
      "logits/rejected": -0.06470983475446701,
      "logps/chosen": -384.7364807128906,
      "logps/rejected": -391.4254455566406,
      "loss": 0.5335,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3247833549976349,
      "rewards/margins": 1.1140307188034058,
      "rewards/rejected": -1.4388139247894287,
      "step": 14417
    },
    {
      "epoch": 1.8866933834955426,
      "grad_norm": 16.573673248291016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2614429295063019,
      "logits/rejected": -0.18755069375038147,
      "logps/chosen": -373.83013916015625,
      "logps/rejected": -483.866943359375,
      "loss": 1.012,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.7734606266021729,
      "rewards/margins": 0.134107768535614,
      "rewards/rejected": -1.907568335533142,
      "step": 14418
    },
    {
      "epoch": 1.8868242414328944,
      "grad_norm": 15.731119155883789,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19876909255981445,
      "logits/rejected": -0.3841516077518463,
      "logps/chosen": -520.0426025390625,
      "logps/rejected": -830.2572631835938,
      "loss": 0.5828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.557328462600708,
      "rewards/margins": 1.4774311780929565,
      "rewards/rejected": -2.034759521484375,
      "step": 14419
    },
    {
      "epoch": 1.8869550993702462,
      "grad_norm": 12.81689167022705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.014926042407751083,
      "logits/rejected": -0.008134875446557999,
      "logps/chosen": -513.658447265625,
      "logps/rejected": -502.419189453125,
      "loss": 0.5532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9411176443099976,
      "rewards/margins": 0.7570098042488098,
      "rewards/rejected": -1.6981275081634521,
      "step": 14420
    },
    {
      "epoch": 1.887085957307598,
      "grad_norm": 9.15302848815918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15249814093112946,
      "logits/rejected": -0.09785232692956924,
      "logps/chosen": -546.0911254882812,
      "logps/rejected": -525.4652099609375,
      "loss": 0.3566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7884224653244019,
      "rewards/margins": 1.8407009840011597,
      "rewards/rejected": -2.6291234493255615,
      "step": 14421
    },
    {
      "epoch": 1.8872168152449498,
      "grad_norm": 8.947826385498047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6032826900482178,
      "logits/rejected": -0.6329900026321411,
      "logps/chosen": -283.1553955078125,
      "logps/rejected": -313.02044677734375,
      "loss": 0.3254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8220788836479187,
      "rewards/margins": 1.8405253887176514,
      "rewards/rejected": -2.662604331970215,
      "step": 14422
    },
    {
      "epoch": 1.8873476731823016,
      "grad_norm": 7.75479793548584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21471935510635376,
      "logits/rejected": -0.4935731589794159,
      "logps/chosen": -517.9293212890625,
      "logps/rejected": -410.92132568359375,
      "loss": 0.2925,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8592500686645508,
      "rewards/margins": 2.0491061210632324,
      "rewards/rejected": -2.908356189727783,
      "step": 14423
    },
    {
      "epoch": 1.8874785311196534,
      "grad_norm": 9.208745956420898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6856772899627686,
      "logits/rejected": -0.5550171732902527,
      "logps/chosen": -463.8809509277344,
      "logps/rejected": -431.78912353515625,
      "loss": 0.5248,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7616302967071533,
      "rewards/margins": 0.871665358543396,
      "rewards/rejected": -1.6332955360412598,
      "step": 14424
    },
    {
      "epoch": 1.8876093890570051,
      "grad_norm": 8.031351089477539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2582128345966339,
      "logits/rejected": -0.2913725674152374,
      "logps/chosen": -423.58270263671875,
      "logps/rejected": -464.4706115722656,
      "loss": 0.4057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8356006741523743,
      "rewards/margins": 1.2057487964630127,
      "rewards/rejected": -2.041349411010742,
      "step": 14425
    },
    {
      "epoch": 1.8877402469943567,
      "grad_norm": 8.826205253601074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3216046392917633,
      "logits/rejected": -0.38560593128204346,
      "logps/chosen": -405.87518310546875,
      "logps/rejected": -395.2328796386719,
      "loss": 0.5117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8702044486999512,
      "rewards/margins": 1.197511076927185,
      "rewards/rejected": -3.067715644836426,
      "step": 14426
    },
    {
      "epoch": 1.8878711049317085,
      "grad_norm": 7.017767429351807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31745076179504395,
      "logits/rejected": -0.22896862030029297,
      "logps/chosen": -387.83990478515625,
      "logps/rejected": -475.95428466796875,
      "loss": 0.2878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.179947018623352,
      "rewards/margins": 1.2740503549575806,
      "rewards/rejected": -2.4539973735809326,
      "step": 14427
    },
    {
      "epoch": 1.8880019628690603,
      "grad_norm": 10.025519371032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4582534432411194,
      "logits/rejected": -0.48197484016418457,
      "logps/chosen": -442.20123291015625,
      "logps/rejected": -475.3471374511719,
      "loss": 0.5862,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2429049015045166,
      "rewards/margins": 1.0091791152954102,
      "rewards/rejected": -2.2520837783813477,
      "step": 14428
    },
    {
      "epoch": 1.888132820806412,
      "grad_norm": 10.17505931854248,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07005834579467773,
      "logits/rejected": -0.22691616415977478,
      "logps/chosen": -606.9093017578125,
      "logps/rejected": -524.66064453125,
      "loss": 0.5275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9379576444625854,
      "rewards/margins": 1.5969126224517822,
      "rewards/rejected": -2.5348703861236572,
      "step": 14429
    },
    {
      "epoch": 1.8882636787437637,
      "grad_norm": 27.421770095825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27716130018234253,
      "logits/rejected": -0.5021044015884399,
      "logps/chosen": -658.3902587890625,
      "logps/rejected": -436.58538818359375,
      "loss": 1.1533,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.0750155448913574,
      "rewards/margins": -0.11743733286857605,
      "rewards/rejected": -1.957578182220459,
      "step": 14430
    },
    {
      "epoch": 1.8883945366811155,
      "grad_norm": 8.007355690002441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1663692593574524,
      "logits/rejected": -0.15294358134269714,
      "logps/chosen": -427.77349853515625,
      "logps/rejected": -473.0173034667969,
      "loss": 0.4413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7708925604820251,
      "rewards/margins": 1.4204871654510498,
      "rewards/rejected": -2.1913797855377197,
      "step": 14431
    },
    {
      "epoch": 1.8885253946184672,
      "grad_norm": 9.38311767578125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3278302550315857,
      "logits/rejected": -0.2067449986934662,
      "logps/chosen": -466.8138427734375,
      "logps/rejected": -462.73187255859375,
      "loss": 0.4967,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12348155677318573,
      "rewards/margins": 1.216912865638733,
      "rewards/rejected": -1.340394377708435,
      "step": 14432
    },
    {
      "epoch": 1.888656252555819,
      "grad_norm": 7.006447792053223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07314100116491318,
      "logits/rejected": -0.3993890881538391,
      "logps/chosen": -781.8947143554688,
      "logps/rejected": -485.59259033203125,
      "loss": 0.2688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2891424894332886,
      "rewards/margins": 2.506580352783203,
      "rewards/rejected": -2.7957229614257812,
      "step": 14433
    },
    {
      "epoch": 1.8887871104931708,
      "grad_norm": 10.064247131347656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10786481201648712,
      "logits/rejected": -0.12249775975942612,
      "logps/chosen": -611.027099609375,
      "logps/rejected": -539.3319091796875,
      "loss": 0.41,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1501911878585815,
      "rewards/margins": 1.2501953840255737,
      "rewards/rejected": -2.4003865718841553,
      "step": 14434
    },
    {
      "epoch": 1.8889179684305226,
      "grad_norm": 9.610045433044434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06106063351035118,
      "logits/rejected": -0.05313071981072426,
      "logps/chosen": -335.7998962402344,
      "logps/rejected": -395.9704895019531,
      "loss": 0.5242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29608649015426636,
      "rewards/margins": 1.3066487312316895,
      "rewards/rejected": -1.6027350425720215,
      "step": 14435
    },
    {
      "epoch": 1.8890488263678744,
      "grad_norm": 12.020720481872559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22295793890953064,
      "logits/rejected": -0.11112120747566223,
      "logps/chosen": -638.0234375,
      "logps/rejected": -770.328369140625,
      "loss": 0.5722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5003959536552429,
      "rewards/margins": 0.5983555316925049,
      "rewards/rejected": -1.0987515449523926,
      "step": 14436
    },
    {
      "epoch": 1.8891796843052262,
      "grad_norm": 8.84338092803955,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1609138399362564,
      "logits/rejected": -0.2876359820365906,
      "logps/chosen": -334.6312561035156,
      "logps/rejected": -335.9765319824219,
      "loss": 0.4085,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8636614084243774,
      "rewards/margins": 1.5357141494750977,
      "rewards/rejected": -2.3993756771087646,
      "step": 14437
    },
    {
      "epoch": 1.889310542242578,
      "grad_norm": 5.2352752685546875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08996203541755676,
      "logits/rejected": -0.29944124817848206,
      "logps/chosen": -414.8882751464844,
      "logps/rejected": -364.95831298828125,
      "loss": 0.295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7536436319351196,
      "rewards/margins": 1.358018398284912,
      "rewards/rejected": -2.111661911010742,
      "step": 14438
    },
    {
      "epoch": 1.8894414001799298,
      "grad_norm": 10.28589153289795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11173656582832336,
      "logits/rejected": -0.33025360107421875,
      "logps/chosen": -525.4253540039062,
      "logps/rejected": -448.547607421875,
      "loss": 0.4415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.333315372467041,
      "rewards/margins": 1.2882928848266602,
      "rewards/rejected": -2.621608018875122,
      "step": 14439
    },
    {
      "epoch": 1.8895722581172816,
      "grad_norm": 7.877607822418213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.560455322265625,
      "logits/rejected": -0.30615776777267456,
      "logps/chosen": -387.04595947265625,
      "logps/rejected": -674.5885620117188,
      "loss": 0.3366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42155832052230835,
      "rewards/margins": 2.1444482803344727,
      "rewards/rejected": -2.5660064220428467,
      "step": 14440
    },
    {
      "epoch": 1.8897031160546331,
      "grad_norm": 15.879667282104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24415834248065948,
      "logits/rejected": -0.04579947888851166,
      "logps/chosen": -490.4992980957031,
      "logps/rejected": -417.4544372558594,
      "loss": 0.9758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0495268106460571,
      "rewards/margins": -0.08740594983100891,
      "rewards/rejected": -0.9621208310127258,
      "step": 14441
    },
    {
      "epoch": 1.889833973991985,
      "grad_norm": 8.563166618347168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14097994565963745,
      "logits/rejected": -0.1857549250125885,
      "logps/chosen": -557.8028564453125,
      "logps/rejected": -460.3230895996094,
      "loss": 0.4937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.023567184805870056,
      "rewards/margins": 1.1975067853927612,
      "rewards/rejected": -1.221073865890503,
      "step": 14442
    },
    {
      "epoch": 1.8899648319293367,
      "grad_norm": 11.532319068908691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40698155760765076,
      "logits/rejected": -0.2585897147655487,
      "logps/chosen": -439.387451171875,
      "logps/rejected": -579.9305419921875,
      "loss": 0.4336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6563867926597595,
      "rewards/margins": 1.1411867141723633,
      "rewards/rejected": -1.797573447227478,
      "step": 14443
    },
    {
      "epoch": 1.8900956898666885,
      "grad_norm": 16.877593994140625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4775657057762146,
      "logits/rejected": -0.6616086959838867,
      "logps/chosen": -427.90240478515625,
      "logps/rejected": -386.6524353027344,
      "loss": 0.6668,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5344799757003784,
      "rewards/margins": 0.6678180694580078,
      "rewards/rejected": -2.202298164367676,
      "step": 14444
    },
    {
      "epoch": 1.89022654780404,
      "grad_norm": 8.345630645751953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13206817209720612,
      "logits/rejected": -0.25889554619789124,
      "logps/chosen": -475.4732971191406,
      "logps/rejected": -757.2236328125,
      "loss": 0.25,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6105741858482361,
      "rewards/margins": 2.100093364715576,
      "rewards/rejected": -2.710667610168457,
      "step": 14445
    },
    {
      "epoch": 1.8903574057413919,
      "grad_norm": 3.9415225982666016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2651299834251404,
      "logits/rejected": -0.4331890046596527,
      "logps/chosen": -613.4845581054688,
      "logps/rejected": -517.0479736328125,
      "loss": 0.1303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43611353635787964,
      "rewards/margins": 2.815993309020996,
      "rewards/rejected": -2.3798797130584717,
      "step": 14446
    },
    {
      "epoch": 1.8904882636787437,
      "grad_norm": 9.389559745788574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6937968730926514,
      "logits/rejected": -0.8055976629257202,
      "logps/chosen": -665.0682983398438,
      "logps/rejected": -912.3930053710938,
      "loss": 0.2929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7501994967460632,
      "rewards/margins": 2.257977247238159,
      "rewards/rejected": -3.0081770420074463,
      "step": 14447
    },
    {
      "epoch": 1.8906191216160955,
      "grad_norm": 3.756063461303711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22309213876724243,
      "logits/rejected": -0.2860092520713806,
      "logps/chosen": -363.4598083496094,
      "logps/rejected": -447.93841552734375,
      "loss": 0.1521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5852553248405457,
      "rewards/margins": 3.4492578506469727,
      "rewards/rejected": -4.034512519836426,
      "step": 14448
    },
    {
      "epoch": 1.8907499795534473,
      "grad_norm": 6.062945365905762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3348485231399536,
      "logits/rejected": -0.5275304317474365,
      "logps/chosen": -648.0357666015625,
      "logps/rejected": -370.2109069824219,
      "loss": 0.1958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40155747532844543,
      "rewards/margins": 2.5236728191375732,
      "rewards/rejected": -2.9252305030822754,
      "step": 14449
    },
    {
      "epoch": 1.890880837490799,
      "grad_norm": 6.110372066497803,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10807344317436218,
      "logits/rejected": -0.15252971649169922,
      "logps/chosen": -535.1768798828125,
      "logps/rejected": -493.0046081542969,
      "loss": 0.2279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21079331636428833,
      "rewards/margins": 1.9260233640670776,
      "rewards/rejected": -1.715230107307434,
      "step": 14450
    },
    {
      "epoch": 1.8910116954281508,
      "grad_norm": 5.791382312774658,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14409692585468292,
      "logits/rejected": -0.3436203896999359,
      "logps/chosen": -469.910400390625,
      "logps/rejected": -529.7088623046875,
      "loss": 0.1712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.686981201171875,
      "rewards/margins": 2.4794349670410156,
      "rewards/rejected": -3.1664156913757324,
      "step": 14451
    },
    {
      "epoch": 1.8911425533655026,
      "grad_norm": 12.470440864562988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15776222944259644,
      "logits/rejected": -0.19000683724880219,
      "logps/chosen": -445.5380859375,
      "logps/rejected": -538.3545532226562,
      "loss": 0.7028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4682241678237915,
      "rewards/margins": 1.1191133260726929,
      "rewards/rejected": -2.5873374938964844,
      "step": 14452
    },
    {
      "epoch": 1.8912734113028544,
      "grad_norm": 5.413515567779541,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01278657466173172,
      "logits/rejected": -0.1392713040113449,
      "logps/chosen": -505.3414306640625,
      "logps/rejected": -439.3513488769531,
      "loss": 0.2249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2736268043518066,
      "rewards/margins": 2.0001842975616455,
      "rewards/rejected": -3.273811101913452,
      "step": 14453
    },
    {
      "epoch": 1.8914042692402062,
      "grad_norm": 6.70145320892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23280149698257446,
      "logits/rejected": -0.11470961570739746,
      "logps/chosen": -476.79290771484375,
      "logps/rejected": -368.99468994140625,
      "loss": 0.248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0135916471481323,
      "rewards/margins": 1.593911051750183,
      "rewards/rejected": -2.6075024604797363,
      "step": 14454
    },
    {
      "epoch": 1.891535127177558,
      "grad_norm": 7.037484645843506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09733127057552338,
      "logits/rejected": -0.4879886209964752,
      "logps/chosen": -575.1494140625,
      "logps/rejected": -420.60247802734375,
      "loss": 0.3006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0629533976316452,
      "rewards/margins": 1.565352439880371,
      "rewards/rejected": -1.50239896774292,
      "step": 14455
    },
    {
      "epoch": 1.8916659851149096,
      "grad_norm": 5.95327615737915,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25539630651474,
      "logits/rejected": -0.28575390577316284,
      "logps/chosen": -454.8277587890625,
      "logps/rejected": -446.6305847167969,
      "loss": 0.2106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18398436903953552,
      "rewards/margins": 2.088131904602051,
      "rewards/rejected": -1.904147744178772,
      "step": 14456
    },
    {
      "epoch": 1.8917968430522614,
      "grad_norm": 4.909421443939209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09875009953975677,
      "logits/rejected": -0.1612851768732071,
      "logps/chosen": -393.4384460449219,
      "logps/rejected": -365.8803405761719,
      "loss": 0.2496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0961851179599762,
      "rewards/margins": 1.8934221267700195,
      "rewards/rejected": -1.7972370386123657,
      "step": 14457
    },
    {
      "epoch": 1.8919277009896132,
      "grad_norm": 10.170126914978027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5152122378349304,
      "logits/rejected": -0.5306940078735352,
      "logps/chosen": -364.8667297363281,
      "logps/rejected": -367.89068603515625,
      "loss": 0.4152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5718379616737366,
      "rewards/margins": 1.4324204921722412,
      "rewards/rejected": -2.004258394241333,
      "step": 14458
    },
    {
      "epoch": 1.892058558926965,
      "grad_norm": 9.087482452392578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26444798707962036,
      "logits/rejected": -0.4282374083995819,
      "logps/chosen": -568.4269409179688,
      "logps/rejected": -577.5325317382812,
      "loss": 0.47,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.272573471069336,
      "rewards/margins": 0.8073713183403015,
      "rewards/rejected": -2.0799448490142822,
      "step": 14459
    },
    {
      "epoch": 1.8921894168643165,
      "grad_norm": 14.405170440673828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14761576056480408,
      "logits/rejected": -0.08083730936050415,
      "logps/chosen": -533.1640625,
      "logps/rejected": -548.8258666992188,
      "loss": 0.7106,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.599880039691925,
      "rewards/margins": 0.5124616026878357,
      "rewards/rejected": -1.1123416423797607,
      "step": 14460
    },
    {
      "epoch": 1.8923202748016683,
      "grad_norm": 8.262238502502441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4186252951622009,
      "logits/rejected": -0.2521700859069824,
      "logps/chosen": -344.6893615722656,
      "logps/rejected": -374.9334716796875,
      "loss": 0.2525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6765899658203125,
      "rewards/margins": 2.1223626136779785,
      "rewards/rejected": -2.798952341079712,
      "step": 14461
    },
    {
      "epoch": 1.89245113273902,
      "grad_norm": 9.877327919006348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4960014522075653,
      "logits/rejected": -0.5582612752914429,
      "logps/chosen": -556.0762939453125,
      "logps/rejected": -483.1485595703125,
      "loss": 0.5844,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9057992696762085,
      "rewards/margins": 0.48035961389541626,
      "rewards/rejected": -1.386159062385559,
      "step": 14462
    },
    {
      "epoch": 1.892581990676372,
      "grad_norm": 8.352461814880371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22013399004936218,
      "logits/rejected": -0.0700063705444336,
      "logps/chosen": -365.6690979003906,
      "logps/rejected": -598.6353149414062,
      "loss": 0.3278,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2098981142044067,
      "rewards/margins": 2.1912505626678467,
      "rewards/rejected": -3.401148557662964,
      "step": 14463
    },
    {
      "epoch": 1.8927128486137237,
      "grad_norm": 9.78880500793457,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13261371850967407,
      "logits/rejected": -0.15326479077339172,
      "logps/chosen": -538.4342651367188,
      "logps/rejected": -472.0796203613281,
      "loss": 0.4209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09297750890254974,
      "rewards/margins": 1.4327880144119263,
      "rewards/rejected": -1.3398106098175049,
      "step": 14464
    },
    {
      "epoch": 1.8928437065510755,
      "grad_norm": 2.126986265182495,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21371355652809143,
      "logits/rejected": -0.331753134727478,
      "logps/chosen": -383.9543762207031,
      "logps/rejected": -361.46746826171875,
      "loss": 0.195,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44311976432800293,
      "rewards/margins": 3.343686580657959,
      "rewards/rejected": -3.786806583404541,
      "step": 14465
    },
    {
      "epoch": 1.8929745644884273,
      "grad_norm": 8.697186470031738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3767278492450714,
      "logits/rejected": -0.11367122828960419,
      "logps/chosen": -392.43621826171875,
      "logps/rejected": -460.03118896484375,
      "loss": 0.2868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8133498430252075,
      "rewards/margins": 1.9056845903396606,
      "rewards/rejected": -2.7190346717834473,
      "step": 14466
    },
    {
      "epoch": 1.893105422425779,
      "grad_norm": 8.683968544006348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3327915668487549,
      "logits/rejected": -0.3484955430030823,
      "logps/chosen": -368.5254821777344,
      "logps/rejected": -438.8246765136719,
      "loss": 0.4456,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.007598265074193478,
      "rewards/margins": 0.7925845384597778,
      "rewards/rejected": -0.8001827001571655,
      "step": 14467
    },
    {
      "epoch": 1.8932362803631309,
      "grad_norm": 14.138264656066895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3932259678840637,
      "logits/rejected": -0.5558730959892273,
      "logps/chosen": -510.43157958984375,
      "logps/rejected": -766.9182739257812,
      "loss": 0.4948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.775466799736023,
      "rewards/margins": 1.7858312129974365,
      "rewards/rejected": -2.561298131942749,
      "step": 14468
    },
    {
      "epoch": 1.8933671383004826,
      "grad_norm": 8.737709999084473,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34870609641075134,
      "logits/rejected": -0.5921466946601868,
      "logps/chosen": -445.1634521484375,
      "logps/rejected": -464.51727294921875,
      "loss": 0.3618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8642266392707825,
      "rewards/margins": 1.6858384609222412,
      "rewards/rejected": -2.550065040588379,
      "step": 14469
    },
    {
      "epoch": 1.8934979962378344,
      "grad_norm": 10.2903413772583,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27139168977737427,
      "logits/rejected": -0.256603479385376,
      "logps/chosen": -464.89129638671875,
      "logps/rejected": -352.50732421875,
      "loss": 0.5429,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14103031158447266,
      "rewards/margins": 1.7521198987960815,
      "rewards/rejected": -1.611089825630188,
      "step": 14470
    },
    {
      "epoch": 1.8936288541751862,
      "grad_norm": 4.4900336265563965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17814838886260986,
      "logits/rejected": -0.1663101762533188,
      "logps/chosen": -510.6840515136719,
      "logps/rejected": -436.4647216796875,
      "loss": 0.2831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6525202393531799,
      "rewards/margins": 1.6356505155563354,
      "rewards/rejected": -2.28817081451416,
      "step": 14471
    },
    {
      "epoch": 1.8937597121125378,
      "grad_norm": 10.018651008605957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08902189880609512,
      "logits/rejected": -0.10086985677480698,
      "logps/chosen": -574.9542236328125,
      "logps/rejected": -459.7760314941406,
      "loss": 0.4025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.768410861492157,
      "rewards/margins": 1.849333643913269,
      "rewards/rejected": -2.6177444458007812,
      "step": 14472
    },
    {
      "epoch": 1.8938905700498896,
      "grad_norm": 10.987889289855957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06354717910289764,
      "logits/rejected": 0.07072373479604721,
      "logps/chosen": -504.23992919921875,
      "logps/rejected": -531.48486328125,
      "loss": 0.5614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.525430679321289,
      "rewards/margins": 0.5137162804603577,
      "rewards/rejected": -2.039146900177002,
      "step": 14473
    },
    {
      "epoch": 1.8940214279872414,
      "grad_norm": 5.680482864379883,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00822487473487854,
      "logits/rejected": -0.07244260609149933,
      "logps/chosen": -647.9137573242188,
      "logps/rejected": -516.9951171875,
      "loss": 0.2689,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08348885178565979,
      "rewards/margins": 1.5766273736953735,
      "rewards/rejected": -1.4931384325027466,
      "step": 14474
    },
    {
      "epoch": 1.894152285924593,
      "grad_norm": 11.063518524169922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3199138343334198,
      "logits/rejected": -0.29494601488113403,
      "logps/chosen": -391.773193359375,
      "logps/rejected": -505.6462707519531,
      "loss": 0.5368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7473345398902893,
      "rewards/margins": 0.9764392971992493,
      "rewards/rejected": -1.723773717880249,
      "step": 14475
    },
    {
      "epoch": 1.8942831438619447,
      "grad_norm": 7.836373329162598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2282305508852005,
      "logits/rejected": -0.45476895570755005,
      "logps/chosen": -704.55224609375,
      "logps/rejected": -933.1322021484375,
      "loss": 0.4758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5685482025146484,
      "rewards/margins": 1.700892686843872,
      "rewards/rejected": -3.2694408893585205,
      "step": 14476
    },
    {
      "epoch": 1.8944140017992965,
      "grad_norm": 9.343185424804688,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07012616097927094,
      "logits/rejected": -0.3841054141521454,
      "logps/chosen": -334.7546691894531,
      "logps/rejected": -321.0650939941406,
      "loss": 0.4094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9456996321678162,
      "rewards/margins": 1.3070582151412964,
      "rewards/rejected": -2.2527577877044678,
      "step": 14477
    },
    {
      "epoch": 1.8945448597366483,
      "grad_norm": 10.021629333496094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2663005590438843,
      "logits/rejected": -0.28968843817710876,
      "logps/chosen": -387.89715576171875,
      "logps/rejected": -487.8399658203125,
      "loss": 0.3237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9022177457809448,
      "rewards/margins": 1.4348183870315552,
      "rewards/rejected": -2.337035894393921,
      "step": 14478
    },
    {
      "epoch": 1.8946757176740001,
      "grad_norm": 4.198040008544922,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03531237691640854,
      "logits/rejected": -0.06404902040958405,
      "logps/chosen": -718.261962890625,
      "logps/rejected": -475.4700927734375,
      "loss": 0.1361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38889312744140625,
      "rewards/margins": 2.577939033508301,
      "rewards/rejected": -2.1890459060668945,
      "step": 14479
    },
    {
      "epoch": 1.894806575611352,
      "grad_norm": 8.098011016845703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31735092401504517,
      "logits/rejected": -0.32401949167251587,
      "logps/chosen": -460.7518615722656,
      "logps/rejected": -335.87786865234375,
      "loss": 0.4989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49521583318710327,
      "rewards/margins": 1.4646680355072021,
      "rewards/rejected": -1.9598839282989502,
      "step": 14480
    },
    {
      "epoch": 1.8949374335487037,
      "grad_norm": 8.465254783630371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26290827989578247,
      "logits/rejected": -0.22546055912971497,
      "logps/chosen": -389.6093444824219,
      "logps/rejected": -460.71185302734375,
      "loss": 0.3117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8309555053710938,
      "rewards/margins": 1.6744632720947266,
      "rewards/rejected": -2.5054187774658203,
      "step": 14481
    },
    {
      "epoch": 1.8950682914860555,
      "grad_norm": 12.854120254516602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17529264092445374,
      "logits/rejected": -0.059973109513521194,
      "logps/chosen": -580.4773559570312,
      "logps/rejected": -521.087890625,
      "loss": 0.582,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6932991147041321,
      "rewards/margins": 0.9391505122184753,
      "rewards/rejected": -1.632449746131897,
      "step": 14482
    },
    {
      "epoch": 1.8951991494234073,
      "grad_norm": 8.479326248168945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.006226571276783943,
      "logits/rejected": -0.004829181358218193,
      "logps/chosen": -382.7024230957031,
      "logps/rejected": -454.509521484375,
      "loss": 0.2606,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09503022581338882,
      "rewards/margins": 2.005340099334717,
      "rewards/rejected": -2.100370407104492,
      "step": 14483
    },
    {
      "epoch": 1.895330007360759,
      "grad_norm": 6.743690490722656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.050051234662532806,
      "logits/rejected": -0.1284816712141037,
      "logps/chosen": -469.19818115234375,
      "logps/rejected": -576.8541870117188,
      "loss": 0.3376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2914142608642578,
      "rewards/margins": 1.6617331504821777,
      "rewards/rejected": -1.953147530555725,
      "step": 14484
    },
    {
      "epoch": 1.8954608652981109,
      "grad_norm": 8.623162269592285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.020788080990314484,
      "logits/rejected": -0.4813941717147827,
      "logps/chosen": -573.7118530273438,
      "logps/rejected": -1287.046142578125,
      "loss": 0.4145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5761793255805969,
      "rewards/margins": 2.473132371902466,
      "rewards/rejected": -3.049311399459839,
      "step": 14485
    },
    {
      "epoch": 1.8955917232354627,
      "grad_norm": 8.934638977050781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25140702724456787,
      "logits/rejected": -0.19587595760822296,
      "logps/chosen": -349.5986022949219,
      "logps/rejected": -513.6969604492188,
      "loss": 0.3079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7466316223144531,
      "rewards/margins": 2.004903793334961,
      "rewards/rejected": -2.751535415649414,
      "step": 14486
    },
    {
      "epoch": 1.8957225811728142,
      "grad_norm": 7.851562023162842,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03606840968132019,
      "logits/rejected": -0.25800246000289917,
      "logps/chosen": -495.16363525390625,
      "logps/rejected": -515.2079467773438,
      "loss": 0.2991,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5695455074310303,
      "rewards/margins": 2.370958089828491,
      "rewards/rejected": -1.801412582397461,
      "step": 14487
    },
    {
      "epoch": 1.895853439110166,
      "grad_norm": 9.458879470825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.037481509149074554,
      "logits/rejected": -0.02807774394750595,
      "logps/chosen": -392.1502685546875,
      "logps/rejected": -387.84954833984375,
      "loss": 0.4097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11048239469528198,
      "rewards/margins": 1.2318007946014404,
      "rewards/rejected": -1.1213184595108032,
      "step": 14488
    },
    {
      "epoch": 1.8959842970475178,
      "grad_norm": 6.883031368255615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19942675530910492,
      "logits/rejected": -0.24751614034175873,
      "logps/chosen": -442.8663635253906,
      "logps/rejected": -431.6171875,
      "loss": 0.4562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09031695872545242,
      "rewards/margins": 1.4420000314712524,
      "rewards/rejected": -1.5323169231414795,
      "step": 14489
    },
    {
      "epoch": 1.8961151549848696,
      "grad_norm": 13.808484077453613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20709624886512756,
      "logits/rejected": -0.2460957020521164,
      "logps/chosen": -350.8515319824219,
      "logps/rejected": -449.4631042480469,
      "loss": 0.7564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4250683784484863,
      "rewards/margins": 0.8725466728210449,
      "rewards/rejected": -2.2976150512695312,
      "step": 14490
    },
    {
      "epoch": 1.8962460129222212,
      "grad_norm": 12.250837326049805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2310837358236313,
      "logits/rejected": -0.3344320058822632,
      "logps/chosen": -479.7170104980469,
      "logps/rejected": -461.7634582519531,
      "loss": 0.3645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8158084750175476,
      "rewards/margins": 1.8314939737319946,
      "rewards/rejected": -2.6473021507263184,
      "step": 14491
    },
    {
      "epoch": 1.896376870859573,
      "grad_norm": 11.467432975769043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4454704821109772,
      "logits/rejected": -0.2774375081062317,
      "logps/chosen": -356.4249267578125,
      "logps/rejected": -455.9816589355469,
      "loss": 0.5136,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3461771309375763,
      "rewards/margins": 0.9214698672294617,
      "rewards/rejected": -1.2676470279693604,
      "step": 14492
    },
    {
      "epoch": 1.8965077287969248,
      "grad_norm": 7.635475158691406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3927623927593231,
      "logits/rejected": -0.3314927816390991,
      "logps/chosen": -585.848876953125,
      "logps/rejected": -402.6125793457031,
      "loss": 0.4394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3068651556968689,
      "rewards/margins": 1.1286314725875854,
      "rewards/rejected": -1.4354965686798096,
      "step": 14493
    },
    {
      "epoch": 1.8966385867342765,
      "grad_norm": 5.178969860076904,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12607869505882263,
      "logits/rejected": -0.06318576633930206,
      "logps/chosen": -484.8634948730469,
      "logps/rejected": -566.2174682617188,
      "loss": 0.1292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07418900728225708,
      "rewards/margins": 3.203805446624756,
      "rewards/rejected": -3.1296162605285645,
      "step": 14494
    },
    {
      "epoch": 1.8967694446716283,
      "grad_norm": 10.170219421386719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09361881017684937,
      "logits/rejected": -0.1748349517583847,
      "logps/chosen": -531.8971557617188,
      "logps/rejected": -451.23822021484375,
      "loss": 0.508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23822328448295593,
      "rewards/margins": 1.2999005317687988,
      "rewards/rejected": -1.5381237268447876,
      "step": 14495
    },
    {
      "epoch": 1.8969003026089801,
      "grad_norm": 8.756010055541992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07932525873184204,
      "logits/rejected": -0.2703220248222351,
      "logps/chosen": -554.5421752929688,
      "logps/rejected": -445.82342529296875,
      "loss": 0.5297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06648120284080505,
      "rewards/margins": 0.9779453873634338,
      "rewards/rejected": -1.0444265604019165,
      "step": 14496
    },
    {
      "epoch": 1.897031160546332,
      "grad_norm": 7.855424880981445,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0005330294370651245,
      "logits/rejected": 0.004884209483861923,
      "logps/chosen": -513.9353637695312,
      "logps/rejected": -412.3763732910156,
      "loss": 0.3158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10791760683059692,
      "rewards/margins": 1.4421355724334717,
      "rewards/rejected": -1.334218144416809,
      "step": 14497
    },
    {
      "epoch": 1.8971620184836837,
      "grad_norm": 12.71645450592041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13978305459022522,
      "logits/rejected": -0.09631842374801636,
      "logps/chosen": -342.64373779296875,
      "logps/rejected": -400.88616943359375,
      "loss": 0.7057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8457422256469727,
      "rewards/margins": 0.9119066596031189,
      "rewards/rejected": -1.7576487064361572,
      "step": 14498
    },
    {
      "epoch": 1.8972928764210355,
      "grad_norm": 7.611123085021973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2377806156873703,
      "logits/rejected": -0.1384655237197876,
      "logps/chosen": -436.7239990234375,
      "logps/rejected": -323.5352783203125,
      "loss": 0.3751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9003565907478333,
      "rewards/margins": 1.3573381900787354,
      "rewards/rejected": -2.257694959640503,
      "step": 14499
    },
    {
      "epoch": 1.8974237343583873,
      "grad_norm": 6.450515270233154,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1261720359325409,
      "logits/rejected": -0.30439314246177673,
      "logps/chosen": -601.42578125,
      "logps/rejected": -490.5374450683594,
      "loss": 0.2824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05768376961350441,
      "rewards/margins": 1.6607024669647217,
      "rewards/rejected": -1.7183862924575806,
      "step": 14500
    },
    {
      "epoch": 1.897554592295739,
      "grad_norm": 13.329412460327148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1474417746067047,
      "logits/rejected": -0.37332263588905334,
      "logps/chosen": -392.4356994628906,
      "logps/rejected": -438.88348388671875,
      "loss": 0.6706,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5753735303878784,
      "rewards/margins": 1.2689712047576904,
      "rewards/rejected": -1.8443448543548584,
      "step": 14501
    },
    {
      "epoch": 1.8976854502330907,
      "grad_norm": 5.651398181915283,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6367030143737793,
      "logits/rejected": -0.43545782566070557,
      "logps/chosen": -227.98391723632812,
      "logps/rejected": -323.4789123535156,
      "loss": 0.33,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5767467617988586,
      "rewards/margins": 1.8729678392410278,
      "rewards/rejected": -2.4497146606445312,
      "step": 14502
    },
    {
      "epoch": 1.8978163081704424,
      "grad_norm": 21.264127731323242,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2696485221385956,
      "logits/rejected": -0.24887852370738983,
      "logps/chosen": -599.8543701171875,
      "logps/rejected": -366.5380859375,
      "loss": 1.3944,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.7556339502334595,
      "rewards/margins": -0.7664496898651123,
      "rewards/rejected": -0.9891843199729919,
      "step": 14503
    },
    {
      "epoch": 1.8979471661077942,
      "grad_norm": 2.8260726928710938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15626481175422668,
      "logits/rejected": -0.3187733292579651,
      "logps/chosen": -428.2108459472656,
      "logps/rejected": -371.98565673828125,
      "loss": 0.1407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3290250599384308,
      "rewards/margins": 2.221738576889038,
      "rewards/rejected": -1.8927135467529297,
      "step": 14504
    },
    {
      "epoch": 1.898078024045146,
      "grad_norm": 12.031925201416016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04321517422795296,
      "logits/rejected": -0.37923112511634827,
      "logps/chosen": -549.4422607421875,
      "logps/rejected": -454.6385498046875,
      "loss": 0.361,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1554864645004272,
      "rewards/margins": 1.7606682777404785,
      "rewards/rejected": -2.9161548614501953,
      "step": 14505
    },
    {
      "epoch": 1.8982088819824976,
      "grad_norm": 8.325124740600586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09654193371534348,
      "logits/rejected": -0.28059762716293335,
      "logps/chosen": -380.81512451171875,
      "logps/rejected": -327.93865966796875,
      "loss": 0.4231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.36997470259666443,
      "rewards/margins": 1.3604727983474731,
      "rewards/rejected": -0.9904981255531311,
      "step": 14506
    },
    {
      "epoch": 1.8983397399198494,
      "grad_norm": 7.474005699157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41335439682006836,
      "logits/rejected": -0.38874173164367676,
      "logps/chosen": -361.765625,
      "logps/rejected": -321.43035888671875,
      "loss": 0.4186,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6923822164535522,
      "rewards/margins": 1.8328748941421509,
      "rewards/rejected": -2.525257110595703,
      "step": 14507
    },
    {
      "epoch": 1.8984705978572012,
      "grad_norm": 6.993773937225342,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07271820306777954,
      "logits/rejected": -0.09015215933322906,
      "logps/chosen": -480.9894714355469,
      "logps/rejected": -517.6673583984375,
      "loss": 0.2837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5688823461532593,
      "rewards/margins": 2.384845495223999,
      "rewards/rejected": -1.815962791442871,
      "step": 14508
    },
    {
      "epoch": 1.898601455794553,
      "grad_norm": 7.682574272155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20468775928020477,
      "logits/rejected": -0.1751624494791031,
      "logps/chosen": -316.6007995605469,
      "logps/rejected": -399.6546936035156,
      "loss": 0.3582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6271770596504211,
      "rewards/margins": 1.788344144821167,
      "rewards/rejected": -2.4155213832855225,
      "step": 14509
    },
    {
      "epoch": 1.8987323137319048,
      "grad_norm": 12.489605903625488,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2818507254123688,
      "logits/rejected": -0.37236902117729187,
      "logps/chosen": -348.8204650878906,
      "logps/rejected": -297.8199768066406,
      "loss": 0.7408,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0003998279571533,
      "rewards/margins": 0.6097797751426697,
      "rewards/rejected": -1.6101795434951782,
      "step": 14510
    },
    {
      "epoch": 1.8988631716692566,
      "grad_norm": 5.171056270599365,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20034290850162506,
      "logits/rejected": -0.36496856808662415,
      "logps/chosen": -428.275390625,
      "logps/rejected": -597.3968505859375,
      "loss": 0.2018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4334825277328491,
      "rewards/margins": 2.4463655948638916,
      "rewards/rejected": -2.879848003387451,
      "step": 14511
    },
    {
      "epoch": 1.8989940296066083,
      "grad_norm": 8.689526557922363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2736435830593109,
      "logits/rejected": -0.37362709641456604,
      "logps/chosen": -355.4703674316406,
      "logps/rejected": -486.579833984375,
      "loss": 0.2971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7039774656295776,
      "rewards/margins": 1.9135735034942627,
      "rewards/rejected": -2.617550849914551,
      "step": 14512
    },
    {
      "epoch": 1.8991248875439601,
      "grad_norm": 8.370845794677734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25969210267066956,
      "logits/rejected": -0.17287780344486237,
      "logps/chosen": -468.1913757324219,
      "logps/rejected": -360.2898864746094,
      "loss": 0.4743,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2379300892353058,
      "rewards/margins": 0.6732959747314453,
      "rewards/rejected": -0.9112260341644287,
      "step": 14513
    },
    {
      "epoch": 1.899255745481312,
      "grad_norm": 7.198531150817871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10864546149969101,
      "logits/rejected": -0.3938364088535309,
      "logps/chosen": -485.2708740234375,
      "logps/rejected": -566.5191040039062,
      "loss": 0.2851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5284626483917236,
      "rewards/margins": 1.2666068077087402,
      "rewards/rejected": -1.7950693368911743,
      "step": 14514
    },
    {
      "epoch": 1.8993866034186637,
      "grad_norm": 2.781348705291748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10444783419370651,
      "logits/rejected": -0.25861769914627075,
      "logps/chosen": -628.518310546875,
      "logps/rejected": -511.30120849609375,
      "loss": 0.0937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6335628032684326,
      "rewards/margins": 2.5570263862609863,
      "rewards/rejected": -1.9234634637832642,
      "step": 14515
    },
    {
      "epoch": 1.8995174613560155,
      "grad_norm": 7.366876125335693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11251168698072433,
      "logits/rejected": -0.020413324236869812,
      "logps/chosen": -479.78887939453125,
      "logps/rejected": -539.1893920898438,
      "loss": 0.3536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10716687142848969,
      "rewards/margins": 1.5949029922485352,
      "rewards/rejected": -1.4877362251281738,
      "step": 14516
    },
    {
      "epoch": 1.8996483192933673,
      "grad_norm": 7.953769207000732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32848137617111206,
      "logits/rejected": -0.2835817337036133,
      "logps/chosen": -360.36083984375,
      "logps/rejected": -381.0075378417969,
      "loss": 0.4404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4259233474731445,
      "rewards/margins": 1.2562282085418701,
      "rewards/rejected": -2.6821517944335938,
      "step": 14517
    },
    {
      "epoch": 1.8997791772307189,
      "grad_norm": 15.695839881896973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5326539278030396,
      "logits/rejected": -0.522239089012146,
      "logps/chosen": -462.99078369140625,
      "logps/rejected": -343.32550048828125,
      "loss": 0.6495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0021250247955322,
      "rewards/margins": 1.7847089767456055,
      "rewards/rejected": -2.786834239959717,
      "step": 14518
    },
    {
      "epoch": 1.8999100351680707,
      "grad_norm": 2.981973171234131,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1907726228237152,
      "logits/rejected": -0.2108808308839798,
      "logps/chosen": -464.5618591308594,
      "logps/rejected": -369.11395263671875,
      "loss": 0.1112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3076317012310028,
      "rewards/margins": 2.6053202152252197,
      "rewards/rejected": -2.2976884841918945,
      "step": 14519
    },
    {
      "epoch": 1.9000408931054225,
      "grad_norm": 9.00537109375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10691922158002853,
      "logits/rejected": -0.17449352145195007,
      "logps/chosen": -469.5445251464844,
      "logps/rejected": -447.59002685546875,
      "loss": 0.2586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5340732336044312,
      "rewards/margins": 1.7534282207489014,
      "rewards/rejected": -2.287501573562622,
      "step": 14520
    },
    {
      "epoch": 1.900171751042774,
      "grad_norm": 8.645689010620117,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08380918204784393,
      "logits/rejected": -0.030059514567255974,
      "logps/chosen": -439.3667297363281,
      "logps/rejected": -401.72369384765625,
      "loss": 0.388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7046136856079102,
      "rewards/margins": 1.0261335372924805,
      "rewards/rejected": -1.7307472229003906,
      "step": 14521
    },
    {
      "epoch": 1.9003026089801258,
      "grad_norm": 6.677073955535889,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1713273674249649,
      "logits/rejected": -0.5020178556442261,
      "logps/chosen": -400.0646667480469,
      "logps/rejected": -303.52557373046875,
      "loss": 0.3018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08774824440479279,
      "rewards/margins": 1.7046059370040894,
      "rewards/rejected": -1.7923541069030762,
      "step": 14522
    },
    {
      "epoch": 1.9004334669174776,
      "grad_norm": 13.305827140808105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2655324339866638,
      "logits/rejected": -0.37850451469421387,
      "logps/chosen": -363.39898681640625,
      "logps/rejected": -543.3798217773438,
      "loss": 0.7214,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4502063989639282,
      "rewards/margins": 1.6526234149932861,
      "rewards/rejected": -2.102829694747925,
      "step": 14523
    },
    {
      "epoch": 1.9005643248548294,
      "grad_norm": 7.180826187133789,
      "learning_rate": 5e-06,
      "logits/chosen": 0.010666996240615845,
      "logits/rejected": -0.06178583204746246,
      "logps/chosen": -498.7673034667969,
      "logps/rejected": -450.4601745605469,
      "loss": 0.3371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06056058406829834,
      "rewards/margins": 1.4759032726287842,
      "rewards/rejected": -1.4153425693511963,
      "step": 14524
    },
    {
      "epoch": 1.9006951827921812,
      "grad_norm": 8.234292030334473,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0373520627617836,
      "logits/rejected": -0.037544019520282745,
      "logps/chosen": -505.41668701171875,
      "logps/rejected": -474.2601318359375,
      "loss": 0.4498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29587554931640625,
      "rewards/margins": 0.9669060111045837,
      "rewards/rejected": -1.2627816200256348,
      "step": 14525
    },
    {
      "epoch": 1.900826040729533,
      "grad_norm": 7.591488361358643,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2853092849254608,
      "logits/rejected": -0.4017299711704254,
      "logps/chosen": -556.629638671875,
      "logps/rejected": -535.5181884765625,
      "loss": 0.3118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7397161722183228,
      "rewards/margins": 1.7162387371063232,
      "rewards/rejected": -2.4559550285339355,
      "step": 14526
    },
    {
      "epoch": 1.9009568986668848,
      "grad_norm": 16.649829864501953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15114223957061768,
      "logits/rejected": 0.1683969497680664,
      "logps/chosen": -396.8921203613281,
      "logps/rejected": -427.89410400390625,
      "loss": 0.9631,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5685333609580994,
      "rewards/margins": 0.14419099688529968,
      "rewards/rejected": -0.7127243280410767,
      "step": 14527
    },
    {
      "epoch": 1.9010877566042366,
      "grad_norm": 7.78070592880249,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3318749666213989,
      "logits/rejected": -0.3861995339393616,
      "logps/chosen": -418.6199951171875,
      "logps/rejected": -451.94952392578125,
      "loss": 0.298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.43380987644195557,
      "rewards/margins": 1.5670112371444702,
      "rewards/rejected": -2.000821113586426,
      "step": 14528
    },
    {
      "epoch": 1.9012186145415884,
      "grad_norm": 10.156587600708008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18607239425182343,
      "logits/rejected": -0.32660016417503357,
      "logps/chosen": -472.8089904785156,
      "logps/rejected": -445.305419921875,
      "loss": 0.6199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011072371155023575,
      "rewards/margins": 1.1668394804000854,
      "rewards/rejected": -1.1779117584228516,
      "step": 14529
    },
    {
      "epoch": 1.9013494724789402,
      "grad_norm": 8.952929496765137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16648180782794952,
      "logits/rejected": -0.11859376728534698,
      "logps/chosen": -476.973876953125,
      "logps/rejected": -502.7219543457031,
      "loss": 0.4952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8251773118972778,
      "rewards/margins": 0.7113860845565796,
      "rewards/rejected": -1.5365633964538574,
      "step": 14530
    },
    {
      "epoch": 1.901480330416292,
      "grad_norm": 12.582189559936523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.312155544757843,
      "logits/rejected": -0.302802711725235,
      "logps/chosen": -326.1500244140625,
      "logps/rejected": -313.1363830566406,
      "loss": 0.5335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6892286539077759,
      "rewards/margins": 1.1328932046890259,
      "rewards/rejected": -1.8221218585968018,
      "step": 14531
    },
    {
      "epoch": 1.9016111883536437,
      "grad_norm": 12.086297035217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047631457448005676,
      "logits/rejected": 0.07716052234172821,
      "logps/chosen": -531.0489501953125,
      "logps/rejected": -520.9739990234375,
      "loss": 0.9265,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8311302661895752,
      "rewards/margins": 0.30663231015205383,
      "rewards/rejected": -1.137762427330017,
      "step": 14532
    },
    {
      "epoch": 1.9017420462909953,
      "grad_norm": 5.966639518737793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3926734924316406,
      "logits/rejected": -0.31597456336021423,
      "logps/chosen": -274.3287048339844,
      "logps/rejected": -364.0295104980469,
      "loss": 0.2561,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.053820811212062836,
      "rewards/margins": 2.2072298526763916,
      "rewards/rejected": -2.153409004211426,
      "step": 14533
    },
    {
      "epoch": 1.901872904228347,
      "grad_norm": 7.01729679107666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17033180594444275,
      "logits/rejected": -0.20301276445388794,
      "logps/chosen": -515.18310546875,
      "logps/rejected": -572.648193359375,
      "loss": 0.3855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6790352463722229,
      "rewards/margins": 1.6247341632843018,
      "rewards/rejected": -2.30376935005188,
      "step": 14534
    },
    {
      "epoch": 1.9020037621656989,
      "grad_norm": 11.979557037353516,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10554506629705429,
      "logits/rejected": -0.1644093245267868,
      "logps/chosen": -415.80999755859375,
      "logps/rejected": -455.81524658203125,
      "loss": 0.8528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.812729001045227,
      "rewards/margins": 0.9211763143539429,
      "rewards/rejected": -2.733905553817749,
      "step": 14535
    },
    {
      "epoch": 1.9021346201030507,
      "grad_norm": 15.171859741210938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1487438827753067,
      "logits/rejected": -0.1353367418050766,
      "logps/chosen": -712.68994140625,
      "logps/rejected": -618.9287719726562,
      "loss": 1.0716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.106919288635254,
      "rewards/margins": 0.24640105664730072,
      "rewards/rejected": -1.3533202409744263,
      "step": 14536
    },
    {
      "epoch": 1.9022654780404022,
      "grad_norm": 8.67054557800293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17516770958900452,
      "logits/rejected": -0.24769437313079834,
      "logps/chosen": -579.3600463867188,
      "logps/rejected": -552.1549072265625,
      "loss": 0.2533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8696193099021912,
      "rewards/margins": 1.8528048992156982,
      "rewards/rejected": -2.722424268722534,
      "step": 14537
    },
    {
      "epoch": 1.902396335977754,
      "grad_norm": 12.238287925720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1356474906206131,
      "logits/rejected": -0.37541666626930237,
      "logps/chosen": -520.0164794921875,
      "logps/rejected": -683.745361328125,
      "loss": 0.6688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1182984113693237,
      "rewards/margins": 0.8172215819358826,
      "rewards/rejected": -1.9355201721191406,
      "step": 14538
    },
    {
      "epoch": 1.9025271939151058,
      "grad_norm": 8.742469787597656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10288234055042267,
      "logits/rejected": -0.13521131873130798,
      "logps/chosen": -415.53497314453125,
      "logps/rejected": -483.1618957519531,
      "loss": 0.2875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.233595848083496,
      "rewards/margins": 2.3714959621429443,
      "rewards/rejected": -3.6050918102264404,
      "step": 14539
    },
    {
      "epoch": 1.9026580518524576,
      "grad_norm": 10.57013988494873,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4099936783313751,
      "logits/rejected": -0.21882835030555725,
      "logps/chosen": -330.5249328613281,
      "logps/rejected": -414.53509521484375,
      "loss": 0.7155,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5974864959716797,
      "rewards/margins": 0.5191725492477417,
      "rewards/rejected": -2.116658926010132,
      "step": 14540
    },
    {
      "epoch": 1.9027889097898094,
      "grad_norm": 8.564867973327637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13108855485916138,
      "logits/rejected": -0.32768353819847107,
      "logps/chosen": -567.2310791015625,
      "logps/rejected": -479.4468994140625,
      "loss": 0.4677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7829669713973999,
      "rewards/margins": 1.3044054508209229,
      "rewards/rejected": -2.087372303009033,
      "step": 14541
    },
    {
      "epoch": 1.9029197677271612,
      "grad_norm": 7.080031394958496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5306440591812134,
      "logits/rejected": -0.49986544251441956,
      "logps/chosen": -459.67376708984375,
      "logps/rejected": -517.72705078125,
      "loss": 0.3089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7369608283042908,
      "rewards/margins": 1.9636110067367554,
      "rewards/rejected": -2.7005717754364014,
      "step": 14542
    },
    {
      "epoch": 1.903050625664513,
      "grad_norm": 9.145546913146973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21766841411590576,
      "logits/rejected": -0.2468276023864746,
      "logps/chosen": -705.0478515625,
      "logps/rejected": -665.2244873046875,
      "loss": 0.3444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5197935104370117,
      "rewards/margins": 1.557769536972046,
      "rewards/rejected": -2.0775630474090576,
      "step": 14543
    },
    {
      "epoch": 1.9031814836018648,
      "grad_norm": 10.200735092163086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2408028542995453,
      "logits/rejected": -0.37354084849357605,
      "logps/chosen": -451.51654052734375,
      "logps/rejected": -495.7447509765625,
      "loss": 0.4336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4361313581466675,
      "rewards/margins": 1.7327269315719604,
      "rewards/rejected": -2.168858289718628,
      "step": 14544
    },
    {
      "epoch": 1.9033123415392166,
      "grad_norm": 7.4601922035217285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4662576913833618,
      "logits/rejected": -0.6389965415000916,
      "logps/chosen": -721.752685546875,
      "logps/rejected": -496.729736328125,
      "loss": 0.3276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2755632698535919,
      "rewards/margins": 1.5545117855072021,
      "rewards/rejected": -1.8300750255584717,
      "step": 14545
    },
    {
      "epoch": 1.9034431994765684,
      "grad_norm": 10.226792335510254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43045881390571594,
      "logits/rejected": -0.2545194923877716,
      "logps/chosen": -426.641845703125,
      "logps/rejected": -497.8838806152344,
      "loss": 0.638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6241070032119751,
      "rewards/margins": 1.9097167253494263,
      "rewards/rejected": -2.5338237285614014,
      "step": 14546
    },
    {
      "epoch": 1.9035740574139202,
      "grad_norm": 7.649141788482666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0683760941028595,
      "logits/rejected": -0.41127485036849976,
      "logps/chosen": -370.6127624511719,
      "logps/rejected": -334.7957458496094,
      "loss": 0.3395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8140599131584167,
      "rewards/margins": 1.1495273113250732,
      "rewards/rejected": -1.9635874032974243,
      "step": 14547
    },
    {
      "epoch": 1.9037049153512717,
      "grad_norm": 12.682107925415039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5378737449645996,
      "logits/rejected": -0.2718445062637329,
      "logps/chosen": -496.0783996582031,
      "logps/rejected": -529.1227416992188,
      "loss": 0.6799,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.538433849811554,
      "rewards/margins": 0.7070587277412415,
      "rewards/rejected": -1.2454924583435059,
      "step": 14548
    },
    {
      "epoch": 1.9038357732886235,
      "grad_norm": 7.221687316894531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14208157360553741,
      "logits/rejected": 0.16580873727798462,
      "logps/chosen": -431.58551025390625,
      "logps/rejected": -361.39837646484375,
      "loss": 0.2225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7629847526550293,
      "rewards/margins": 2.1116838455200195,
      "rewards/rejected": -1.3486988544464111,
      "step": 14549
    },
    {
      "epoch": 1.9039666312259753,
      "grad_norm": 7.154809951782227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3134344816207886,
      "logits/rejected": -0.38846713304519653,
      "logps/chosen": -359.9609680175781,
      "logps/rejected": -532.2197265625,
      "loss": 0.2903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4630078971385956,
      "rewards/margins": 2.30502986907959,
      "rewards/rejected": -2.768037796020508,
      "step": 14550
    },
    {
      "epoch": 1.904097489163327,
      "grad_norm": 9.712920188903809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16713842749595642,
      "logits/rejected": -0.19635862112045288,
      "logps/chosen": -754.66357421875,
      "logps/rejected": -525.377685546875,
      "loss": 0.3148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5497051477432251,
      "rewards/margins": 1.9840412139892578,
      "rewards/rejected": -2.5337464809417725,
      "step": 14551
    },
    {
      "epoch": 1.9042283471006787,
      "grad_norm": 6.753545761108398,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19562533497810364,
      "logits/rejected": -0.5041301846504211,
      "logps/chosen": -443.4453430175781,
      "logps/rejected": -356.2178039550781,
      "loss": 0.2754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8721307516098022,
      "rewards/margins": 1.8868019580841064,
      "rewards/rejected": -2.7589328289031982,
      "step": 14552
    },
    {
      "epoch": 1.9043592050380305,
      "grad_norm": 7.2252326011657715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16910825669765472,
      "logits/rejected": -0.09040359407663345,
      "logps/chosen": -425.3106689453125,
      "logps/rejected": -390.343994140625,
      "loss": 0.3805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4235506057739258,
      "rewards/margins": 1.3688464164733887,
      "rewards/rejected": -1.792397141456604,
      "step": 14553
    },
    {
      "epoch": 1.9044900629753823,
      "grad_norm": 12.426995277404785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25648292899131775,
      "logits/rejected": -0.2131921648979187,
      "logps/chosen": -315.57666015625,
      "logps/rejected": -463.6683654785156,
      "loss": 0.4807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5839701294898987,
      "rewards/margins": 1.8128550052642822,
      "rewards/rejected": -2.396824836730957,
      "step": 14554
    },
    {
      "epoch": 1.904620920912734,
      "grad_norm": 10.592924118041992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2192719578742981,
      "logits/rejected": -0.3800986707210541,
      "logps/chosen": -456.3591613769531,
      "logps/rejected": -356.0890808105469,
      "loss": 0.4535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1595233827829361,
      "rewards/margins": 1.5927337408065796,
      "rewards/rejected": -1.752257227897644,
      "step": 14555
    },
    {
      "epoch": 1.9047517788500858,
      "grad_norm": 11.585677146911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15675324201583862,
      "logits/rejected": -0.3434349596500397,
      "logps/chosen": -388.2461853027344,
      "logps/rejected": -357.4910888671875,
      "loss": 0.5774,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6330186724662781,
      "rewards/margins": 1.2785999774932861,
      "rewards/rejected": -1.9116185903549194,
      "step": 14556
    },
    {
      "epoch": 1.9048826367874376,
      "grad_norm": 7.923202991485596,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030443498864769936,
      "logits/rejected": -0.20200026035308838,
      "logps/chosen": -563.2311401367188,
      "logps/rejected": -470.11041259765625,
      "loss": 0.2935,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3545650839805603,
      "rewards/margins": 1.9132733345031738,
      "rewards/rejected": -2.267838478088379,
      "step": 14557
    },
    {
      "epoch": 1.9050134947247894,
      "grad_norm": 4.6009721755981445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23927994072437286,
      "logits/rejected": -0.2719449996948242,
      "logps/chosen": -457.03778076171875,
      "logps/rejected": -422.4504699707031,
      "loss": 0.2047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25730571150779724,
      "rewards/margins": 1.8274685144424438,
      "rewards/rejected": -2.0847742557525635,
      "step": 14558
    },
    {
      "epoch": 1.9051443526621412,
      "grad_norm": 7.905684471130371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29990190267562866,
      "logits/rejected": -0.1825743019580841,
      "logps/chosen": -428.3561706542969,
      "logps/rejected": -292.885498046875,
      "loss": 0.4856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22580982744693756,
      "rewards/margins": 1.5647319555282593,
      "rewards/rejected": -1.7905418872833252,
      "step": 14559
    },
    {
      "epoch": 1.905275210599493,
      "grad_norm": 8.6466064453125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30407172441482544,
      "logits/rejected": -0.22722505033016205,
      "logps/chosen": -485.7996520996094,
      "logps/rejected": -392.50726318359375,
      "loss": 0.3122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01691320538520813,
      "rewards/margins": 2.4939968585968018,
      "rewards/rejected": -2.5109100341796875,
      "step": 14560
    },
    {
      "epoch": 1.9054060685368448,
      "grad_norm": 10.503923416137695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06506519019603729,
      "logits/rejected": -0.3282819390296936,
      "logps/chosen": -457.7184753417969,
      "logps/rejected": -434.9232482910156,
      "loss": 0.5337,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6004092693328857,
      "rewards/margins": 1.0192136764526367,
      "rewards/rejected": -2.6196229457855225,
      "step": 14561
    },
    {
      "epoch": 1.9055369264741966,
      "grad_norm": 9.653507232666016,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13426953554153442,
      "logits/rejected": -0.23762072622776031,
      "logps/chosen": -381.29986572265625,
      "logps/rejected": -355.49560546875,
      "loss": 0.3953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0871976613998413,
      "rewards/margins": 2.613140821456909,
      "rewards/rejected": -3.70033860206604,
      "step": 14562
    },
    {
      "epoch": 1.9056677844115482,
      "grad_norm": 12.781638145446777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12728549540042877,
      "logits/rejected": -0.07046819478273392,
      "logps/chosen": -457.1122741699219,
      "logps/rejected": -414.5050964355469,
      "loss": 0.9385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7479660511016846,
      "rewards/margins": 0.22150224447250366,
      "rewards/rejected": -1.969468116760254,
      "step": 14563
    },
    {
      "epoch": 1.9057986423489,
      "grad_norm": 7.280730247497559,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3359305262565613,
      "logits/rejected": -0.33858439326286316,
      "logps/chosen": -349.12677001953125,
      "logps/rejected": -366.7913818359375,
      "loss": 0.5055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6057554483413696,
      "rewards/margins": 1.2571640014648438,
      "rewards/rejected": -1.8629196882247925,
      "step": 14564
    },
    {
      "epoch": 1.9059295002862517,
      "grad_norm": 10.38742446899414,
      "learning_rate": 5e-06,
      "logits/chosen": 0.06565216183662415,
      "logits/rejected": 0.00453658401966095,
      "logps/chosen": -597.3538208007812,
      "logps/rejected": -718.2564086914062,
      "loss": 0.3875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06306934356689453,
      "rewards/margins": 1.2265249490737915,
      "rewards/rejected": -1.163455605506897,
      "step": 14565
    },
    {
      "epoch": 1.9060603582236035,
      "grad_norm": 6.882271766662598,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41242843866348267,
      "logits/rejected": -0.5158018469810486,
      "logps/chosen": -359.194580078125,
      "logps/rejected": -374.0531921386719,
      "loss": 0.3954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5380293726921082,
      "rewards/margins": 0.9782335758209229,
      "rewards/rejected": -1.5162630081176758,
      "step": 14566
    },
    {
      "epoch": 1.906191216160955,
      "grad_norm": 7.322561740875244,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27843576669692993,
      "logits/rejected": -0.28717613220214844,
      "logps/chosen": -479.1029968261719,
      "logps/rejected": -479.5594177246094,
      "loss": 0.343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3548870086669922,
      "rewards/margins": 1.4418597221374512,
      "rewards/rejected": -1.7967467308044434,
      "step": 14567
    },
    {
      "epoch": 1.906322074098307,
      "grad_norm": 3.285301685333252,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15179967880249023,
      "logits/rejected": -0.1827322393655777,
      "logps/chosen": -372.5696105957031,
      "logps/rejected": -516.5306396484375,
      "loss": 0.1301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36038458347320557,
      "rewards/margins": 3.00494647026062,
      "rewards/rejected": -3.3653311729431152,
      "step": 14568
    },
    {
      "epoch": 1.9064529320356587,
      "grad_norm": 10.52627182006836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4798702895641327,
      "logits/rejected": -0.3141286373138428,
      "logps/chosen": -448.73345947265625,
      "logps/rejected": -465.6114501953125,
      "loss": 0.5533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7930073142051697,
      "rewards/margins": 1.0131391286849976,
      "rewards/rejected": -1.8061463832855225,
      "step": 14569
    },
    {
      "epoch": 1.9065837899730105,
      "grad_norm": 7.729659080505371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33034783601760864,
      "logits/rejected": -0.3953014612197876,
      "logps/chosen": -539.517333984375,
      "logps/rejected": -536.3148803710938,
      "loss": 0.2169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09813191741704941,
      "rewards/margins": 2.960425615310669,
      "rewards/rejected": -2.862293243408203,
      "step": 14570
    },
    {
      "epoch": 1.9067146479103623,
      "grad_norm": 14.51368522644043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3562292754650116,
      "logits/rejected": -0.302722692489624,
      "logps/chosen": -428.97027587890625,
      "logps/rejected": -410.03436279296875,
      "loss": 0.8211,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8301613330841064,
      "rewards/margins": 0.5067571997642517,
      "rewards/rejected": -1.336918592453003,
      "step": 14571
    },
    {
      "epoch": 1.906845505847714,
      "grad_norm": 8.696313858032227,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13855580985546112,
      "logits/rejected": -0.2987099885940552,
      "logps/chosen": -345.74945068359375,
      "logps/rejected": -354.06329345703125,
      "loss": 0.5557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8331575989723206,
      "rewards/margins": 0.5660331845283508,
      "rewards/rejected": -1.3991907835006714,
      "step": 14572
    },
    {
      "epoch": 1.9069763637850659,
      "grad_norm": 9.77293586730957,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3272055983543396,
      "logits/rejected": -0.2978849411010742,
      "logps/chosen": -492.0863342285156,
      "logps/rejected": -553.0631103515625,
      "loss": 0.403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0379748344421387,
      "rewards/margins": 1.6728063821792603,
      "rewards/rejected": -3.7107810974121094,
      "step": 14573
    },
    {
      "epoch": 1.9071072217224176,
      "grad_norm": 17.572505950927734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41632166504859924,
      "logits/rejected": -0.3371846079826355,
      "logps/chosen": -521.6453247070312,
      "logps/rejected": -486.93505859375,
      "loss": 0.9431,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.360988140106201,
      "rewards/margins": 0.21458879113197327,
      "rewards/rejected": -2.5755772590637207,
      "step": 14574
    },
    {
      "epoch": 1.9072380796597694,
      "grad_norm": 19.415987014770508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37208282947540283,
      "logits/rejected": -0.30623579025268555,
      "logps/chosen": -501.4442138671875,
      "logps/rejected": -367.8995056152344,
      "loss": 0.8327,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9319570064544678,
      "rewards/margins": 0.7319860458374023,
      "rewards/rejected": -2.66394305229187,
      "step": 14575
    },
    {
      "epoch": 1.9073689375971212,
      "grad_norm": 8.158432960510254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.318220317363739,
      "logits/rejected": -0.30248719453811646,
      "logps/chosen": -613.9315185546875,
      "logps/rejected": -555.5065307617188,
      "loss": 0.3132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9926033020019531,
      "rewards/margins": 1.7141175270080566,
      "rewards/rejected": -2.7067208290100098,
      "step": 14576
    },
    {
      "epoch": 1.907499795534473,
      "grad_norm": 10.875385284423828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20145127177238464,
      "logits/rejected": -0.37239986658096313,
      "logps/chosen": -458.2819519042969,
      "logps/rejected": -389.6156311035156,
      "loss": 0.5919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0935828685760498,
      "rewards/margins": 0.8836473822593689,
      "rewards/rejected": -1.977230191230774,
      "step": 14577
    },
    {
      "epoch": 1.9076306534718248,
      "grad_norm": 8.411175727844238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41397231817245483,
      "logits/rejected": -0.6050916910171509,
      "logps/chosen": -427.56866455078125,
      "logps/rejected": -448.70367431640625,
      "loss": 0.3688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1545920372009277,
      "rewards/margins": 1.2041292190551758,
      "rewards/rejected": -2.3587212562561035,
      "step": 14578
    },
    {
      "epoch": 1.9077615114091764,
      "grad_norm": 12.788944244384766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2802961766719818,
      "logits/rejected": -0.265820175409317,
      "logps/chosen": -321.0245666503906,
      "logps/rejected": -390.28155517578125,
      "loss": 0.6928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.21897554397583,
      "rewards/margins": 1.5135884284973145,
      "rewards/rejected": -2.7325639724731445,
      "step": 14579
    },
    {
      "epoch": 1.9078923693465282,
      "grad_norm": 7.896624565124512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1632183939218521,
      "logits/rejected": -0.18156330287456512,
      "logps/chosen": -602.5888061523438,
      "logps/rejected": -473.74261474609375,
      "loss": 0.4488,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8969873189926147,
      "rewards/margins": 1.922404170036316,
      "rewards/rejected": -2.8193917274475098,
      "step": 14580
    },
    {
      "epoch": 1.90802322728388,
      "grad_norm": 13.020882606506348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3354794383049011,
      "logits/rejected": -0.4662740230560303,
      "logps/chosen": -531.3356323242188,
      "logps/rejected": -454.3929443359375,
      "loss": 0.7471,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7065784931182861,
      "rewards/margins": 0.4852733612060547,
      "rewards/rejected": -2.191851854324341,
      "step": 14581
    },
    {
      "epoch": 1.9081540852212318,
      "grad_norm": 5.066991806030273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21110187470912933,
      "logits/rejected": -0.3646027445793152,
      "logps/chosen": -476.77008056640625,
      "logps/rejected": -390.51593017578125,
      "loss": 0.1996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3572956323623657,
      "rewards/margins": 2.2798800468444824,
      "rewards/rejected": -2.6371757984161377,
      "step": 14582
    },
    {
      "epoch": 1.9082849431585833,
      "grad_norm": 10.874894142150879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07104306668043137,
      "logits/rejected": 0.006677590310573578,
      "logps/chosen": -615.810791015625,
      "logps/rejected": -665.2927856445312,
      "loss": 0.5339,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.677280068397522,
      "rewards/margins": 1.1617708206176758,
      "rewards/rejected": -1.8390508890151978,
      "step": 14583
    },
    {
      "epoch": 1.9084158010959351,
      "grad_norm": 4.609689235687256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028157778084278107,
      "logits/rejected": -0.317105770111084,
      "logps/chosen": -520.4541015625,
      "logps/rejected": -489.0413818359375,
      "loss": 0.2068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5451383590698242,
      "rewards/margins": 2.32175612449646,
      "rewards/rejected": -1.7766177654266357,
      "step": 14584
    },
    {
      "epoch": 1.908546659033287,
      "grad_norm": 11.184020042419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.134267657995224,
      "logits/rejected": -0.1926545649766922,
      "logps/chosen": -370.20294189453125,
      "logps/rejected": -354.2884826660156,
      "loss": 0.8428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40657734870910645,
      "rewards/margins": 1.0100103616714478,
      "rewards/rejected": -1.416587471961975,
      "step": 14585
    },
    {
      "epoch": 1.9086775169706387,
      "grad_norm": 11.789201736450195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35484474897384644,
      "logits/rejected": -0.4129446744918823,
      "logps/chosen": -579.7894897460938,
      "logps/rejected": -579.1296997070312,
      "loss": 0.4873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2995612621307373,
      "rewards/margins": 1.0328726768493652,
      "rewards/rejected": -2.3324339389801025,
      "step": 14586
    },
    {
      "epoch": 1.9088083749079905,
      "grad_norm": 7.206299781799316,
      "learning_rate": 5e-06,
      "logits/chosen": 0.13069669902324677,
      "logits/rejected": -0.225601464509964,
      "logps/chosen": -586.3170166015625,
      "logps/rejected": -394.33404541015625,
      "loss": 0.2878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03663790225982666,
      "rewards/margins": 2.230727195739746,
      "rewards/rejected": -2.19408917427063,
      "step": 14587
    },
    {
      "epoch": 1.9089392328453423,
      "grad_norm": 6.280185699462891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02353236824274063,
      "logits/rejected": -0.44449928402900696,
      "logps/chosen": -510.0628662109375,
      "logps/rejected": -542.2891235351562,
      "loss": 0.1956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9162240028381348,
      "rewards/margins": 3.0149993896484375,
      "rewards/rejected": -3.9312236309051514,
      "step": 14588
    },
    {
      "epoch": 1.909070090782694,
      "grad_norm": 13.402029037475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1984267234802246,
      "logits/rejected": -0.20159798860549927,
      "logps/chosen": -361.7617492675781,
      "logps/rejected": -447.04901123046875,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.01765775680542,
      "rewards/margins": 0.10358485579490662,
      "rewards/rejected": -1.1212425231933594,
      "step": 14589
    },
    {
      "epoch": 1.9092009487200459,
      "grad_norm": 7.914018154144287,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3679610788822174,
      "logits/rejected": -0.3213510513305664,
      "logps/chosen": -457.6751708984375,
      "logps/rejected": -430.6730651855469,
      "loss": 0.4005,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8812365531921387,
      "rewards/margins": 1.2660276889801025,
      "rewards/rejected": -3.1472647190093994,
      "step": 14590
    },
    {
      "epoch": 1.9093318066573977,
      "grad_norm": 12.626731872558594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42364293336868286,
      "logits/rejected": -0.4536595642566681,
      "logps/chosen": -300.0397033691406,
      "logps/rejected": -337.2833557128906,
      "loss": 0.7882,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0324480533599854,
      "rewards/margins": 0.8326382637023926,
      "rewards/rejected": -1.865086317062378,
      "step": 14591
    },
    {
      "epoch": 1.9094626645947494,
      "grad_norm": 5.607320785522461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.24193912744522095,
      "logits/rejected": 0.1944456845521927,
      "logps/chosen": -507.353759765625,
      "logps/rejected": -469.2922668457031,
      "loss": 0.2021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5887037515640259,
      "rewards/margins": 2.192732095718384,
      "rewards/rejected": -2.781435966491699,
      "step": 14592
    },
    {
      "epoch": 1.9095935225321012,
      "grad_norm": 7.80765962600708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2764066457748413,
      "logits/rejected": -0.4192855656147003,
      "logps/chosen": -330.3197937011719,
      "logps/rejected": -414.59765625,
      "loss": 0.3469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7543686628341675,
      "rewards/margins": 1.3434748649597168,
      "rewards/rejected": -2.0978434085845947,
      "step": 14593
    },
    {
      "epoch": 1.9097243804694528,
      "grad_norm": 9.21993350982666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14422041177749634,
      "logits/rejected": -0.27038297057151794,
      "logps/chosen": -401.96429443359375,
      "logps/rejected": -379.5824890136719,
      "loss": 0.3945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27543890476226807,
      "rewards/margins": 2.5171310901641846,
      "rewards/rejected": -2.792569875717163,
      "step": 14594
    },
    {
      "epoch": 1.9098552384068046,
      "grad_norm": 11.5076904296875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4002068042755127,
      "logits/rejected": -0.46651458740234375,
      "logps/chosen": -332.628173828125,
      "logps/rejected": -435.7104797363281,
      "loss": 0.5566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.091496229171753,
      "rewards/margins": 1.42940354347229,
      "rewards/rejected": -2.520900011062622,
      "step": 14595
    },
    {
      "epoch": 1.9099860963441564,
      "grad_norm": 32.91923904418945,
      "learning_rate": 5e-06,
      "logits/chosen": 0.153782919049263,
      "logits/rejected": -0.0563708171248436,
      "logps/chosen": -617.63134765625,
      "logps/rejected": -493.0216064453125,
      "loss": 0.9559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.749388575553894,
      "rewards/margins": 0.43479597568511963,
      "rewards/rejected": -1.1841844320297241,
      "step": 14596
    },
    {
      "epoch": 1.9101169542815082,
      "grad_norm": 8.198603630065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39670950174331665,
      "logits/rejected": -0.48563873767852783,
      "logps/chosen": -455.0042419433594,
      "logps/rejected": -463.8492431640625,
      "loss": 0.4015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5103384256362915,
      "rewards/margins": 1.9172230958938599,
      "rewards/rejected": -2.4275617599487305,
      "step": 14597
    },
    {
      "epoch": 1.9102478122188598,
      "grad_norm": 3.5127272605895996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35311850905418396,
      "logits/rejected": -0.3041677474975586,
      "logps/chosen": -597.35888671875,
      "logps/rejected": -468.60040283203125,
      "loss": 0.1475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5779560208320618,
      "rewards/margins": 2.596829891204834,
      "rewards/rejected": -2.018873929977417,
      "step": 14598
    },
    {
      "epoch": 1.9103786701562115,
      "grad_norm": 14.79150676727295,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09670490026473999,
      "logits/rejected": -0.0034122150391340256,
      "logps/chosen": -462.8725891113281,
      "logps/rejected": -458.5145263671875,
      "loss": 0.8952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7870063781738281,
      "rewards/margins": 0.3659750521183014,
      "rewards/rejected": -1.1529814004898071,
      "step": 14599
    },
    {
      "epoch": 1.9105095280935633,
      "grad_norm": 9.693982124328613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07866908609867096,
      "logits/rejected": -0.2411053329706192,
      "logps/chosen": -428.79150390625,
      "logps/rejected": -487.77984619140625,
      "loss": 0.3527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7577499151229858,
      "rewards/margins": 1.7209203243255615,
      "rewards/rejected": -2.478670120239258,
      "step": 14600
    },
    {
      "epoch": 1.9106403860309151,
      "grad_norm": 14.563301086425781,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24168667197227478,
      "logits/rejected": -0.11695091426372528,
      "logps/chosen": -620.216552734375,
      "logps/rejected": -657.91650390625,
      "loss": 1.1079,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9947699308395386,
      "rewards/margins": -0.08470021188259125,
      "rewards/rejected": -0.9100696444511414,
      "step": 14601
    },
    {
      "epoch": 1.910771243968267,
      "grad_norm": 4.5738019943237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20020174980163574,
      "logits/rejected": -0.454025536775589,
      "logps/chosen": -495.7506408691406,
      "logps/rejected": -416.210205078125,
      "loss": 0.1852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32718199491500854,
      "rewards/margins": 2.0479049682617188,
      "rewards/rejected": -1.7207229137420654,
      "step": 14602
    },
    {
      "epoch": 1.9109021019056187,
      "grad_norm": 3.342238426208496,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42091214656829834,
      "logits/rejected": -0.3152378499507904,
      "logps/chosen": -451.43243408203125,
      "logps/rejected": -489.2857971191406,
      "loss": 0.1282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13661880791187286,
      "rewards/margins": 2.554579973220825,
      "rewards/rejected": -2.6911985874176025,
      "step": 14603
    },
    {
      "epoch": 1.9110329598429705,
      "grad_norm": 17.149145126342773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2793526351451874,
      "logits/rejected": -0.4736132025718689,
      "logps/chosen": -541.7404174804688,
      "logps/rejected": -488.1791687011719,
      "loss": 1.2676,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.914963960647583,
      "rewards/margins": -0.498102605342865,
      "rewards/rejected": -0.416861355304718,
      "step": 14604
    },
    {
      "epoch": 1.9111638177803223,
      "grad_norm": 9.925559043884277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.60875403881073,
      "logits/rejected": -0.7268273830413818,
      "logps/chosen": -375.1395263671875,
      "logps/rejected": -417.88751220703125,
      "loss": 0.7752,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1065540313720703,
      "rewards/margins": 0.6393176913261414,
      "rewards/rejected": -1.745871901512146,
      "step": 14605
    },
    {
      "epoch": 1.911294675717674,
      "grad_norm": 12.031914710998535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22495603561401367,
      "logits/rejected": -0.24032188951969147,
      "logps/chosen": -691.3997802734375,
      "logps/rejected": -464.8656005859375,
      "loss": 0.475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.163517951965332,
      "rewards/margins": 1.1455795764923096,
      "rewards/rejected": -2.3090977668762207,
      "step": 14606
    },
    {
      "epoch": 1.9114255336550259,
      "grad_norm": 8.94698715209961,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33853644132614136,
      "logits/rejected": -0.15128755569458008,
      "logps/chosen": -342.7735595703125,
      "logps/rejected": -395.44000244140625,
      "loss": 0.5797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7147154211997986,
      "rewards/margins": 1.336015224456787,
      "rewards/rejected": -2.0507307052612305,
      "step": 14607
    },
    {
      "epoch": 1.9115563915923777,
      "grad_norm": 8.52197265625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3137539327144623,
      "logits/rejected": -0.1547134816646576,
      "logps/chosen": -312.4609375,
      "logps/rejected": -363.3826904296875,
      "loss": 0.5228,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3108697831630707,
      "rewards/margins": 1.9094913005828857,
      "rewards/rejected": -2.2203609943389893,
      "step": 14608
    },
    {
      "epoch": 1.9116872495297292,
      "grad_norm": 10.60756778717041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3107856810092926,
      "logits/rejected": -0.1242617666721344,
      "logps/chosen": -378.4786682128906,
      "logps/rejected": -323.71356201171875,
      "loss": 0.6104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1856298446655273,
      "rewards/margins": 1.377753496170044,
      "rewards/rejected": -2.5633835792541504,
      "step": 14609
    },
    {
      "epoch": 1.911818107467081,
      "grad_norm": 6.876828193664551,
      "learning_rate": 5e-06,
      "logits/chosen": 0.047145918011665344,
      "logits/rejected": -0.24604009091854095,
      "logps/chosen": -546.5189819335938,
      "logps/rejected": -400.951416015625,
      "loss": 0.3443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1495070457458496,
      "rewards/margins": 1.692545771598816,
      "rewards/rejected": -2.842052936553955,
      "step": 14610
    },
    {
      "epoch": 1.9119489654044328,
      "grad_norm": 8.114164352416992,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3754788935184479,
      "logits/rejected": -0.23022592067718506,
      "logps/chosen": -491.44378662109375,
      "logps/rejected": -569.071044921875,
      "loss": 0.3427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.320918083190918,
      "rewards/margins": 1.5767641067504883,
      "rewards/rejected": -2.8976824283599854,
      "step": 14611
    },
    {
      "epoch": 1.9120798233417846,
      "grad_norm": 13.474115371704102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7190407514572144,
      "logits/rejected": -0.4142920970916748,
      "logps/chosen": -315.971435546875,
      "logps/rejected": -543.6982421875,
      "loss": 0.504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.305802583694458,
      "rewards/margins": 0.5533161163330078,
      "rewards/rejected": -1.8591187000274658,
      "step": 14612
    },
    {
      "epoch": 1.9122106812791362,
      "grad_norm": 11.78431510925293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.249492347240448,
      "logits/rejected": -0.16058674454689026,
      "logps/chosen": -373.8039245605469,
      "logps/rejected": -350.55352783203125,
      "loss": 0.7976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.925789475440979,
      "rewards/margins": 0.42512989044189453,
      "rewards/rejected": -1.3509193658828735,
      "step": 14613
    },
    {
      "epoch": 1.912341539216488,
      "grad_norm": 3.2520933151245117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11580140888690948,
      "logits/rejected": -0.05292366445064545,
      "logps/chosen": -474.178466796875,
      "logps/rejected": -510.5314636230469,
      "loss": 0.108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14895039796829224,
      "rewards/margins": 3.139227867126465,
      "rewards/rejected": -2.9902775287628174,
      "step": 14614
    },
    {
      "epoch": 1.9124723971538398,
      "grad_norm": 8.166226387023926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18391777575016022,
      "logits/rejected": -0.17348456382751465,
      "logps/chosen": -301.0928649902344,
      "logps/rejected": -324.4795227050781,
      "loss": 0.461,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7197878360748291,
      "rewards/margins": 1.5661401748657227,
      "rewards/rejected": -2.2859277725219727,
      "step": 14615
    },
    {
      "epoch": 1.9126032550911916,
      "grad_norm": 12.279640197753906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3723052740097046,
      "logits/rejected": -0.37895169854164124,
      "logps/chosen": -579.5060424804688,
      "logps/rejected": -569.71875,
      "loss": 0.3195,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0453014373779297,
      "rewards/margins": 3.060574769973755,
      "rewards/rejected": -4.1058759689331055,
      "step": 14616
    },
    {
      "epoch": 1.9127341130285433,
      "grad_norm": 14.78798770904541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30475908517837524,
      "logits/rejected": -0.358972430229187,
      "logps/chosen": -426.56549072265625,
      "logps/rejected": -450.06170654296875,
      "loss": 0.8938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5079911947250366,
      "rewards/margins": 0.47620853781700134,
      "rewards/rejected": -1.9841997623443604,
      "step": 14617
    },
    {
      "epoch": 1.9128649709658951,
      "grad_norm": 1.913541555404663,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17881615459918976,
      "logits/rejected": -0.1448511779308319,
      "logps/chosen": -449.9095458984375,
      "logps/rejected": -554.9546508789062,
      "loss": 0.0575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25579798221588135,
      "rewards/margins": 4.182219505310059,
      "rewards/rejected": -4.438017845153809,
      "step": 14618
    },
    {
      "epoch": 1.912995828903247,
      "grad_norm": 14.451765060424805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35568466782569885,
      "logits/rejected": -0.3777662217617035,
      "logps/chosen": -534.0887451171875,
      "logps/rejected": -544.9718017578125,
      "loss": 0.9784,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.801483154296875,
      "rewards/margins": 0.2719927132129669,
      "rewards/rejected": -2.0734758377075195,
      "step": 14619
    },
    {
      "epoch": 1.9131266868405987,
      "grad_norm": 5.8927001953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19370320439338684,
      "logits/rejected": -0.3744492530822754,
      "logps/chosen": -582.5656127929688,
      "logps/rejected": -568.9439086914062,
      "loss": 0.2305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.375885009765625,
      "rewards/margins": 2.2373239994049072,
      "rewards/rejected": -2.6132090091705322,
      "step": 14620
    },
    {
      "epoch": 1.9132575447779505,
      "grad_norm": 7.896325588226318,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08574755489826202,
      "logits/rejected": -0.3643081486225128,
      "logps/chosen": -366.31744384765625,
      "logps/rejected": -361.91265869140625,
      "loss": 0.35,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8476428985595703,
      "rewards/margins": 1.2479127645492554,
      "rewards/rejected": -2.095555543899536,
      "step": 14621
    },
    {
      "epoch": 1.9133884027153023,
      "grad_norm": 13.398768424987793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7771922945976257,
      "logits/rejected": -0.5427935123443604,
      "logps/chosen": -520.2977294921875,
      "logps/rejected": -664.4501342773438,
      "loss": 0.6449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.417266309261322,
      "rewards/margins": 2.1688385009765625,
      "rewards/rejected": -2.5861048698425293,
      "step": 14622
    },
    {
      "epoch": 1.913519260652654,
      "grad_norm": 10.064879417419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2593068480491638,
      "logits/rejected": -0.3179243505001068,
      "logps/chosen": -447.28875732421875,
      "logps/rejected": -315.10638427734375,
      "loss": 0.3484,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6253402233123779,
      "rewards/margins": 2.8084568977355957,
      "rewards/rejected": -3.4337973594665527,
      "step": 14623
    },
    {
      "epoch": 1.9136501185900059,
      "grad_norm": 5.660781383514404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39630645513534546,
      "logits/rejected": -0.24639680981636047,
      "logps/chosen": -452.328369140625,
      "logps/rejected": -373.74029541015625,
      "loss": 0.209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04067838191986084,
      "rewards/margins": 2.0793709754943848,
      "rewards/rejected": -2.120049476623535,
      "step": 14624
    },
    {
      "epoch": 1.9137809765273575,
      "grad_norm": 4.988702297210693,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5087908506393433,
      "logits/rejected": -0.40559351444244385,
      "logps/chosen": -550.033935546875,
      "logps/rejected": -590.5553588867188,
      "loss": 0.201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21667462587356567,
      "rewards/margins": 2.6195175647735596,
      "rewards/rejected": -2.8361921310424805,
      "step": 14625
    },
    {
      "epoch": 1.9139118344647092,
      "grad_norm": 9.779434204101562,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37493896484375,
      "logits/rejected": -0.3395523726940155,
      "logps/chosen": -543.6070556640625,
      "logps/rejected": -477.65618896484375,
      "loss": 0.6035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9782423377037048,
      "rewards/margins": 1.0665900707244873,
      "rewards/rejected": -2.044832468032837,
      "step": 14626
    },
    {
      "epoch": 1.914042692402061,
      "grad_norm": 9.511463165283203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.039822403341531754,
      "logits/rejected": -0.023498637601733208,
      "logps/chosen": -610.3350219726562,
      "logps/rejected": -500.49859619140625,
      "loss": 0.4395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41480615735054016,
      "rewards/margins": 1.1965433359146118,
      "rewards/rejected": -1.6113494634628296,
      "step": 14627
    },
    {
      "epoch": 1.9141735503394126,
      "grad_norm": 8.133915901184082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3323204517364502,
      "logits/rejected": -0.08628064393997192,
      "logps/chosen": -324.136474609375,
      "logps/rejected": -350.03289794921875,
      "loss": 0.5349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8767822980880737,
      "rewards/margins": 1.2690140008926392,
      "rewards/rejected": -2.145796060562134,
      "step": 14628
    },
    {
      "epoch": 1.9143044082767644,
      "grad_norm": 9.320785522460938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3653714954853058,
      "logits/rejected": -0.27001193165779114,
      "logps/chosen": -476.74346923828125,
      "logps/rejected": -520.962646484375,
      "loss": 0.3315,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3100261688232422,
      "rewards/margins": 1.578705906867981,
      "rewards/rejected": -1.8887319564819336,
      "step": 14629
    },
    {
      "epoch": 1.9144352662141162,
      "grad_norm": 11.098142623901367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4677202105522156,
      "logits/rejected": -0.29014837741851807,
      "logps/chosen": -613.5432739257812,
      "logps/rejected": -632.17529296875,
      "loss": 0.4149,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9330118298530579,
      "rewards/margins": 1.1253935098648071,
      "rewards/rejected": -2.0584053993225098,
      "step": 14630
    },
    {
      "epoch": 1.914566124151468,
      "grad_norm": 6.598543643951416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1435386836528778,
      "logits/rejected": -0.37455758452415466,
      "logps/chosen": -470.5779113769531,
      "logps/rejected": -522.82177734375,
      "loss": 0.2609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21076661348342896,
      "rewards/margins": 1.9531642198562622,
      "rewards/rejected": -2.163930654525757,
      "step": 14631
    },
    {
      "epoch": 1.9146969820888198,
      "grad_norm": 12.996027946472168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20582079887390137,
      "logits/rejected": -0.15521195530891418,
      "logps/chosen": -376.2030029296875,
      "logps/rejected": -396.2307434082031,
      "loss": 0.6488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0401703119277954,
      "rewards/margins": 0.9345929622650146,
      "rewards/rejected": -1.97476327419281,
      "step": 14632
    },
    {
      "epoch": 1.9148278400261716,
      "grad_norm": 8.497915267944336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.405436247587204,
      "logits/rejected": -0.3402971625328064,
      "logps/chosen": -554.5699462890625,
      "logps/rejected": -437.661865234375,
      "loss": 0.3255,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.518915593624115,
      "rewards/margins": 1.8669638633728027,
      "rewards/rejected": -2.3858795166015625,
      "step": 14633
    },
    {
      "epoch": 1.9149586979635234,
      "grad_norm": 8.49697208404541,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39801689982414246,
      "logits/rejected": -0.2831898331642151,
      "logps/chosen": -278.0915832519531,
      "logps/rejected": -300.1903991699219,
      "loss": 0.5826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6846565008163452,
      "rewards/margins": 1.4002326726913452,
      "rewards/rejected": -2.0848891735076904,
      "step": 14634
    },
    {
      "epoch": 1.9150895559008752,
      "grad_norm": 11.78105640411377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46573054790496826,
      "logits/rejected": -0.42656466364860535,
      "logps/chosen": -591.37548828125,
      "logps/rejected": -477.0426025390625,
      "loss": 0.5146,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3320921659469604,
      "rewards/margins": 1.286688208580017,
      "rewards/rejected": -2.6187803745269775,
      "step": 14635
    },
    {
      "epoch": 1.915220413838227,
      "grad_norm": 5.88478946685791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.013927362859249115,
      "logits/rejected": -0.0020372681319713593,
      "logps/chosen": -379.43133544921875,
      "logps/rejected": -465.1397399902344,
      "loss": 0.2737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5035663843154907,
      "rewards/margins": 2.6670315265655518,
      "rewards/rejected": -3.170597791671753,
      "step": 14636
    },
    {
      "epoch": 1.9153512717755787,
      "grad_norm": 7.958607196807861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28935134410858154,
      "logits/rejected": -0.3581768274307251,
      "logps/chosen": -500.15924072265625,
      "logps/rejected": -429.738525390625,
      "loss": 0.4377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5794041156768799,
      "rewards/margins": 1.1856485605239868,
      "rewards/rejected": -1.7650526762008667,
      "step": 14637
    },
    {
      "epoch": 1.9154821297129305,
      "grad_norm": 11.938812255859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09381615370512009,
      "logits/rejected": -0.12907913327217102,
      "logps/chosen": -470.1121826171875,
      "logps/rejected": -336.4512023925781,
      "loss": 0.5018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7527191638946533,
      "rewards/margins": 1.0925654172897339,
      "rewards/rejected": -1.8452845811843872,
      "step": 14638
    },
    {
      "epoch": 1.9156129876502823,
      "grad_norm": 5.6486334800720215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23576714098453522,
      "logits/rejected": -0.24051617085933685,
      "logps/chosen": -597.6417236328125,
      "logps/rejected": -539.9583129882812,
      "loss": 0.2051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5625017285346985,
      "rewards/margins": 2.340059280395508,
      "rewards/rejected": -1.777557611465454,
      "step": 14639
    },
    {
      "epoch": 1.9157438455876339,
      "grad_norm": 12.138083457946777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20720800757408142,
      "logits/rejected": -0.09775761514902115,
      "logps/chosen": -403.1382751464844,
      "logps/rejected": -486.817626953125,
      "loss": 0.6614,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9207627773284912,
      "rewards/margins": 2.0479280948638916,
      "rewards/rejected": -2.9686906337738037,
      "step": 14640
    },
    {
      "epoch": 1.9158747035249857,
      "grad_norm": 9.496581077575684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20299969613552094,
      "logits/rejected": -0.294320672750473,
      "logps/chosen": -395.05963134765625,
      "logps/rejected": -380.46466064453125,
      "loss": 0.6046,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5322303771972656,
      "rewards/margins": 1.309235692024231,
      "rewards/rejected": -1.841465950012207,
      "step": 14641
    },
    {
      "epoch": 1.9160055614623375,
      "grad_norm": 8.399303436279297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31904345750808716,
      "logits/rejected": -0.2006201595067978,
      "logps/chosen": -390.7351989746094,
      "logps/rejected": -510.6170349121094,
      "loss": 0.4559,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0775177478790283,
      "rewards/margins": 1.2633103132247925,
      "rewards/rejected": -2.340827703475952,
      "step": 14642
    },
    {
      "epoch": 1.9161364193996893,
      "grad_norm": 11.98709774017334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07468529045581818,
      "logits/rejected": -0.31687361001968384,
      "logps/chosen": -478.58929443359375,
      "logps/rejected": -514.5108032226562,
      "loss": 0.6664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2114356905221939,
      "rewards/margins": 1.3491852283477783,
      "rewards/rejected": -1.560620903968811,
      "step": 14643
    },
    {
      "epoch": 1.9162672773370408,
      "grad_norm": 12.772226333618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22802017629146576,
      "logits/rejected": -0.15389341115951538,
      "logps/chosen": -543.8751220703125,
      "logps/rejected": -650.22412109375,
      "loss": 0.5441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.994855523109436,
      "rewards/margins": 0.8148193359375,
      "rewards/rejected": -1.809674859046936,
      "step": 14644
    },
    {
      "epoch": 1.9163981352743926,
      "grad_norm": 2.5549533367156982,
      "learning_rate": 5e-06,
      "logits/chosen": 0.12743619084358215,
      "logits/rejected": -0.1809173822402954,
      "logps/chosen": -737.3561401367188,
      "logps/rejected": -553.7793579101562,
      "loss": 0.0906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25787925720214844,
      "rewards/margins": 3.240565776824951,
      "rewards/rejected": -2.9826865196228027,
      "step": 14645
    },
    {
      "epoch": 1.9165289932117444,
      "grad_norm": 8.206559181213379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4754706919193268,
      "logits/rejected": -0.505975604057312,
      "logps/chosen": -519.6412963867188,
      "logps/rejected": -445.4116516113281,
      "loss": 0.3272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8584281802177429,
      "rewards/margins": 1.8171498775482178,
      "rewards/rejected": -2.6755778789520264,
      "step": 14646
    },
    {
      "epoch": 1.9166598511490962,
      "grad_norm": 10.166484832763672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07712370157241821,
      "logits/rejected": -0.3577972650527954,
      "logps/chosen": -472.8979797363281,
      "logps/rejected": -613.5633544921875,
      "loss": 0.6984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3466095924377441,
      "rewards/margins": 1.2800633907318115,
      "rewards/rejected": -2.6266727447509766,
      "step": 14647
    },
    {
      "epoch": 1.916790709086448,
      "grad_norm": 12.867680549621582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2425316870212555,
      "logits/rejected": -0.23259368538856506,
      "logps/chosen": -535.9215698242188,
      "logps/rejected": -829.8656005859375,
      "loss": 0.6133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.086812973022461,
      "rewards/margins": 1.845556616783142,
      "rewards/rejected": -2.9323694705963135,
      "step": 14648
    },
    {
      "epoch": 1.9169215670237998,
      "grad_norm": 9.62314224243164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1881324052810669,
      "logits/rejected": -0.16894881427288055,
      "logps/chosen": -444.7110595703125,
      "logps/rejected": -337.6446533203125,
      "loss": 0.5581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06023138761520386,
      "rewards/margins": 0.804372251033783,
      "rewards/rejected": -0.8646036982536316,
      "step": 14649
    },
    {
      "epoch": 1.9170524249611516,
      "grad_norm": 5.313241004943848,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2535514235496521,
      "logits/rejected": -0.3229304254055023,
      "logps/chosen": -314.5448303222656,
      "logps/rejected": -354.8960266113281,
      "loss": 0.2075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7944520115852356,
      "rewards/margins": 2.7035648822784424,
      "rewards/rejected": -3.4980170726776123,
      "step": 14650
    },
    {
      "epoch": 1.9171832828985034,
      "grad_norm": 7.160706996917725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21203161776065826,
      "logits/rejected": -0.3225601613521576,
      "logps/chosen": -512.0435791015625,
      "logps/rejected": -459.1309814453125,
      "loss": 0.3701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2130675315856934,
      "rewards/margins": 1.2496280670166016,
      "rewards/rejected": -3.462695360183716,
      "step": 14651
    },
    {
      "epoch": 1.9173141408358552,
      "grad_norm": 12.701011657714844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4135380983352661,
      "logits/rejected": -0.29417508840560913,
      "logps/chosen": -471.26873779296875,
      "logps/rejected": -543.7395629882812,
      "loss": 0.4655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5506048202514648,
      "rewards/margins": 1.5080305337905884,
      "rewards/rejected": -3.0586352348327637,
      "step": 14652
    },
    {
      "epoch": 1.917444998773207,
      "grad_norm": 4.5697784423828125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04951689764857292,
      "logits/rejected": -0.05926111340522766,
      "logps/chosen": -826.8923950195312,
      "logps/rejected": -573.93603515625,
      "loss": 0.205,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3471859097480774,
      "rewards/margins": 2.4466652870178223,
      "rewards/rejected": -2.0994794368743896,
      "step": 14653
    },
    {
      "epoch": 1.9175758567105587,
      "grad_norm": 3.6024398803710938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2537212073802948,
      "logits/rejected": -0.32831674814224243,
      "logps/chosen": -407.0544738769531,
      "logps/rejected": -519.879150390625,
      "loss": 0.1851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.023638129234313965,
      "rewards/margins": 2.83268666267395,
      "rewards/rejected": -2.8090481758117676,
      "step": 14654
    },
    {
      "epoch": 1.9177067146479103,
      "grad_norm": 12.30014705657959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.278403639793396,
      "logits/rejected": -0.39727094769477844,
      "logps/chosen": -689.60205078125,
      "logps/rejected": -440.60150146484375,
      "loss": 0.5321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.46930992603302,
      "rewards/margins": 1.0058343410491943,
      "rewards/rejected": -2.475144386291504,
      "step": 14655
    },
    {
      "epoch": 1.917837572585262,
      "grad_norm": 12.13818073272705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.14174015820026398,
      "logits/rejected": -0.12466252595186234,
      "logps/chosen": -312.5635070800781,
      "logps/rejected": -294.504638671875,
      "loss": 0.4585,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8234237432479858,
      "rewards/margins": 1.6514482498168945,
      "rewards/rejected": -2.47487211227417,
      "step": 14656
    },
    {
      "epoch": 1.917968430522614,
      "grad_norm": 16.769908905029297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3230537474155426,
      "logits/rejected": -0.43221354484558105,
      "logps/chosen": -355.5771789550781,
      "logps/rejected": -323.4495849609375,
      "loss": 1.1007,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.9565660953521729,
      "rewards/margins": -0.4321626126766205,
      "rewards/rejected": -1.5244033336639404,
      "step": 14657
    },
    {
      "epoch": 1.9180992884599657,
      "grad_norm": 9.731986999511719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2723925709724426,
      "logits/rejected": -0.21417547762393951,
      "logps/chosen": -546.13818359375,
      "logps/rejected": -652.9241333007812,
      "loss": 0.3379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.123307943344116,
      "rewards/margins": 1.140984296798706,
      "rewards/rejected": -3.2642924785614014,
      "step": 14658
    },
    {
      "epoch": 1.9182301463973173,
      "grad_norm": 7.795843601226807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27113640308380127,
      "logits/rejected": -0.34970369935035706,
      "logps/chosen": -566.4712524414062,
      "logps/rejected": -367.26104736328125,
      "loss": 0.3973,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6877157688140869,
      "rewards/margins": 1.3836915493011475,
      "rewards/rejected": -2.0714073181152344,
      "step": 14659
    },
    {
      "epoch": 1.918361004334669,
      "grad_norm": 14.379127502441406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3419856131076813,
      "logits/rejected": -0.31437405943870544,
      "logps/chosen": -379.7171936035156,
      "logps/rejected": -396.73876953125,
      "loss": 0.6914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9425057172775269,
      "rewards/margins": 0.535786509513855,
      "rewards/rejected": -1.4782921075820923,
      "step": 14660
    },
    {
      "epoch": 1.9184918622720208,
      "grad_norm": 4.217133045196533,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40521278977394104,
      "logits/rejected": -0.40205612778663635,
      "logps/chosen": -376.7095947265625,
      "logps/rejected": -349.16168212890625,
      "loss": 0.2424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8962960839271545,
      "rewards/margins": 1.9637823104858398,
      "rewards/rejected": -2.8600785732269287,
      "step": 14661
    },
    {
      "epoch": 1.9186227202093726,
      "grad_norm": 11.606216430664062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16468995809555054,
      "logits/rejected": -0.2400364726781845,
      "logps/chosen": -537.7677612304688,
      "logps/rejected": -570.1787719726562,
      "loss": 0.3783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.191666603088379,
      "rewards/margins": 1.6626899242401123,
      "rewards/rejected": -2.854356288909912,
      "step": 14662
    },
    {
      "epoch": 1.9187535781467244,
      "grad_norm": 11.259215354919434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40037059783935547,
      "logits/rejected": -0.38400301337242126,
      "logps/chosen": -478.1930847167969,
      "logps/rejected": -527.8782958984375,
      "loss": 0.4753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5208781957626343,
      "rewards/margins": 0.9670788049697876,
      "rewards/rejected": -1.4879570007324219,
      "step": 14663
    },
    {
      "epoch": 1.9188844360840762,
      "grad_norm": 3.795091390609741,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19980473816394806,
      "logits/rejected": 0.026087716221809387,
      "logps/chosen": -539.2156982421875,
      "logps/rejected": -469.7629699707031,
      "loss": 0.22,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04660951718688011,
      "rewards/margins": 1.8381202220916748,
      "rewards/rejected": -1.7915107011795044,
      "step": 14664
    },
    {
      "epoch": 1.919015294021428,
      "grad_norm": 4.93731164932251,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4691166877746582,
      "logits/rejected": -0.5806379318237305,
      "logps/chosen": -264.7229309082031,
      "logps/rejected": -396.48663330078125,
      "loss": 0.2689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5979152917861938,
      "rewards/margins": 2.0718958377838135,
      "rewards/rejected": -2.669811248779297,
      "step": 14665
    },
    {
      "epoch": 1.9191461519587798,
      "grad_norm": 8.060981750488281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.01474173553287983,
      "logits/rejected": -0.22910946607589722,
      "logps/chosen": -415.8048095703125,
      "logps/rejected": -324.03857421875,
      "loss": 0.4319,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3483314514160156,
      "rewards/margins": 1.5910930633544922,
      "rewards/rejected": -2.939424514770508,
      "step": 14666
    },
    {
      "epoch": 1.9192770098961316,
      "grad_norm": 11.24432373046875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23179955780506134,
      "logits/rejected": -0.13157828152179718,
      "logps/chosen": -311.1447448730469,
      "logps/rejected": -371.8589782714844,
      "loss": 0.6692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5724956393241882,
      "rewards/margins": 0.5047987699508667,
      "rewards/rejected": -1.0772943496704102,
      "step": 14667
    },
    {
      "epoch": 1.9194078678334834,
      "grad_norm": 5.68366003036499,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40855568647384644,
      "logits/rejected": -0.6074227690696716,
      "logps/chosen": -344.78900146484375,
      "logps/rejected": -344.7941589355469,
      "loss": 0.2731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4512174427509308,
      "rewards/margins": 1.4884319305419922,
      "rewards/rejected": -1.9396493434906006,
      "step": 14668
    },
    {
      "epoch": 1.9195387257708352,
      "grad_norm": 7.364633083343506,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26241225004196167,
      "logits/rejected": -0.26493051648139954,
      "logps/chosen": -363.9487609863281,
      "logps/rejected": -335.7377624511719,
      "loss": 0.48,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6245706677436829,
      "rewards/margins": 1.156873106956482,
      "rewards/rejected": -1.7814438343048096,
      "step": 14669
    },
    {
      "epoch": 1.919669583708187,
      "grad_norm": 6.080697059631348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2715624272823334,
      "logits/rejected": -0.19469796121120453,
      "logps/chosen": -547.8628540039062,
      "logps/rejected": -454.76434326171875,
      "loss": 0.3363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18804073333740234,
      "rewards/margins": 1.3466217517852783,
      "rewards/rejected": -1.158581018447876,
      "step": 14670
    },
    {
      "epoch": 1.9198004416455385,
      "grad_norm": 13.782288551330566,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35320085287094116,
      "logits/rejected": -0.3678983151912689,
      "logps/chosen": -699.81494140625,
      "logps/rejected": -486.3927917480469,
      "loss": 0.3305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.276219129562378,
      "rewards/margins": 1.4671316146850586,
      "rewards/rejected": -3.7433507442474365,
      "step": 14671
    },
    {
      "epoch": 1.9199312995828903,
      "grad_norm": 9.336884498596191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3826824724674225,
      "logits/rejected": -0.28360414505004883,
      "logps/chosen": -657.9168090820312,
      "logps/rejected": -456.0406494140625,
      "loss": 0.5212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8079850673675537,
      "rewards/margins": 0.8755080699920654,
      "rewards/rejected": -2.6834933757781982,
      "step": 14672
    },
    {
      "epoch": 1.9200621575202421,
      "grad_norm": 11.53859806060791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2647789418697357,
      "logits/rejected": -0.21806596219539642,
      "logps/chosen": -417.04449462890625,
      "logps/rejected": -361.5382080078125,
      "loss": 0.6117,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3105465173721313,
      "rewards/margins": 1.1270909309387207,
      "rewards/rejected": -2.4376375675201416,
      "step": 14673
    },
    {
      "epoch": 1.9201930154575937,
      "grad_norm": 13.374348640441895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15138643980026245,
      "logits/rejected": -0.4106050431728363,
      "logps/chosen": -756.6943969726562,
      "logps/rejected": -554.4969482421875,
      "loss": 0.8478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2891587018966675,
      "rewards/margins": 0.4311893582344055,
      "rewards/rejected": -1.7203480005264282,
      "step": 14674
    },
    {
      "epoch": 1.9203238733949455,
      "grad_norm": 4.693718433380127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2712119221687317,
      "logits/rejected": -0.15061944723129272,
      "logps/chosen": -546.510986328125,
      "logps/rejected": -541.4990234375,
      "loss": 0.1662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8697970509529114,
      "rewards/margins": 2.3493659496307373,
      "rewards/rejected": -3.2191624641418457,
      "step": 14675
    },
    {
      "epoch": 1.9204547313322973,
      "grad_norm": 2.7034711837768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18624651432037354,
      "logits/rejected": -0.3595817983150482,
      "logps/chosen": -512.6398315429688,
      "logps/rejected": -380.3541259765625,
      "loss": 0.0776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003328705206513405,
      "rewards/margins": 3.378894567489624,
      "rewards/rejected": -3.382223129272461,
      "step": 14676
    },
    {
      "epoch": 1.920585589269649,
      "grad_norm": 10.686270713806152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1321510523557663,
      "logits/rejected": -0.3360927999019623,
      "logps/chosen": -753.9693603515625,
      "logps/rejected": -475.3135070800781,
      "loss": 0.3808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6184770464897156,
      "rewards/margins": 1.327959418296814,
      "rewards/rejected": -1.9464362859725952,
      "step": 14677
    },
    {
      "epoch": 1.9207164472070009,
      "grad_norm": 7.445954322814941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5641000866889954,
      "logits/rejected": -0.5036441683769226,
      "logps/chosen": -465.2947998046875,
      "logps/rejected": -530.5753784179688,
      "loss": 0.3702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5110971927642822,
      "rewards/margins": 1.2518118619918823,
      "rewards/rejected": -2.762908935546875,
      "step": 14678
    },
    {
      "epoch": 1.9208473051443526,
      "grad_norm": 7.436558246612549,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26023730635643005,
      "logits/rejected": -0.4368540048599243,
      "logps/chosen": -404.6531066894531,
      "logps/rejected": -403.6062316894531,
      "loss": 0.231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6945685148239136,
      "rewards/margins": 1.7214683294296265,
      "rewards/rejected": -2.416036605834961,
      "step": 14679
    },
    {
      "epoch": 1.9209781630817044,
      "grad_norm": 4.508785724639893,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2120632380247116,
      "logits/rejected": -0.2545962929725647,
      "logps/chosen": -891.2634887695312,
      "logps/rejected": -688.6024780273438,
      "loss": 0.1666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24288177490234375,
      "rewards/margins": 2.4390664100646973,
      "rewards/rejected": -2.1961846351623535,
      "step": 14680
    },
    {
      "epoch": 1.9211090210190562,
      "grad_norm": 8.875548362731934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12203773111104965,
      "logits/rejected": -0.49440401792526245,
      "logps/chosen": -611.6552734375,
      "logps/rejected": -452.54571533203125,
      "loss": 0.4168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5252254605293274,
      "rewards/margins": 1.9550836086273193,
      "rewards/rejected": -2.480309247970581,
      "step": 14681
    },
    {
      "epoch": 1.921239878956408,
      "grad_norm": 9.39022445678711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06082363426685333,
      "logits/rejected": -0.2746767997741699,
      "logps/chosen": -410.7629699707031,
      "logps/rejected": -454.62384033203125,
      "loss": 0.4969,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8439562916755676,
      "rewards/margins": 1.1728912591934204,
      "rewards/rejected": -2.0168473720550537,
      "step": 14682
    },
    {
      "epoch": 1.9213707368937598,
      "grad_norm": 11.5038480758667,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3387017548084259,
      "logits/rejected": -0.5397226810455322,
      "logps/chosen": -345.880126953125,
      "logps/rejected": -271.927001953125,
      "loss": 0.6179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.10374116897583,
      "rewards/margins": 0.5347977876663208,
      "rewards/rejected": -1.6385388374328613,
      "step": 14683
    },
    {
      "epoch": 1.9215015948311116,
      "grad_norm": 9.696372032165527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22532865405082703,
      "logits/rejected": -0.45010852813720703,
      "logps/chosen": -596.3098754882812,
      "logps/rejected": -562.7932739257812,
      "loss": 0.4465,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6612033843994141,
      "rewards/margins": 1.3908436298370361,
      "rewards/rejected": -2.05204701423645,
      "step": 14684
    },
    {
      "epoch": 1.9216324527684634,
      "grad_norm": 11.710257530212402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22730591893196106,
      "logits/rejected": -0.11348164081573486,
      "logps/chosen": -431.62554931640625,
      "logps/rejected": -451.17193603515625,
      "loss": 0.3883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7323919534683228,
      "rewards/margins": 1.6952667236328125,
      "rewards/rejected": -2.4276585578918457,
      "step": 14685
    },
    {
      "epoch": 1.921763310705815,
      "grad_norm": 5.4532856941223145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45306748151779175,
      "logits/rejected": -0.5410513877868652,
      "logps/chosen": -399.58953857421875,
      "logps/rejected": -441.178955078125,
      "loss": 0.245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9368462562561035,
      "rewards/margins": 2.2611467838287354,
      "rewards/rejected": -3.1979928016662598,
      "step": 14686
    },
    {
      "epoch": 1.9218941686431668,
      "grad_norm": 5.077823162078857,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005552023649215698,
      "logits/rejected": -0.22286203503608704,
      "logps/chosen": -682.8989868164062,
      "logps/rejected": -467.25390625,
      "loss": 0.2508,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13606470823287964,
      "rewards/margins": 1.933064579963684,
      "rewards/rejected": -1.7970000505447388,
      "step": 14687
    },
    {
      "epoch": 1.9220250265805185,
      "grad_norm": 5.945013523101807,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29236990213394165,
      "logits/rejected": -0.4979434609413147,
      "logps/chosen": -436.16131591796875,
      "logps/rejected": -436.16851806640625,
      "loss": 0.3366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4062727093696594,
      "rewards/margins": 1.2712774276733398,
      "rewards/rejected": -1.6775500774383545,
      "step": 14688
    },
    {
      "epoch": 1.9221558845178703,
      "grad_norm": 9.278658866882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.019138289615511894,
      "logits/rejected": -0.1801156997680664,
      "logps/chosen": -366.1209411621094,
      "logps/rejected": -383.8758544921875,
      "loss": 0.5582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6339820623397827,
      "rewards/margins": 1.3855268955230713,
      "rewards/rejected": -2.0195088386535645,
      "step": 14689
    },
    {
      "epoch": 1.922286742455222,
      "grad_norm": 12.922751426696777,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02351272851228714,
      "logits/rejected": -0.10995793342590332,
      "logps/chosen": -359.125244140625,
      "logps/rejected": -427.4322509765625,
      "loss": 0.4739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.291426420211792,
      "rewards/margins": 1.1219336986541748,
      "rewards/rejected": -2.4133598804473877,
      "step": 14690
    },
    {
      "epoch": 1.9224176003925737,
      "grad_norm": 14.75668716430664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4801490604877472,
      "logits/rejected": -0.4457908272743225,
      "logps/chosen": -629.6890258789062,
      "logps/rejected": -478.3367004394531,
      "loss": 0.7392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2738059759140015,
      "rewards/margins": 1.4686310291290283,
      "rewards/rejected": -2.7424371242523193,
      "step": 14691
    },
    {
      "epoch": 1.9225484583299255,
      "grad_norm": 14.411162376403809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2869623005390167,
      "logits/rejected": -0.4709717631340027,
      "logps/chosen": -381.5647888183594,
      "logps/rejected": -430.5945739746094,
      "loss": 0.7757,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.95437490940094,
      "rewards/margins": 0.6020780801773071,
      "rewards/rejected": -2.556453227996826,
      "step": 14692
    },
    {
      "epoch": 1.9226793162672773,
      "grad_norm": 9.561802864074707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.260760098695755,
      "logits/rejected": -0.46588677167892456,
      "logps/chosen": -578.3118896484375,
      "logps/rejected": -439.63916015625,
      "loss": 0.5491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.42336615920066833,
      "rewards/margins": 1.0639595985412598,
      "rewards/rejected": -1.4873257875442505,
      "step": 14693
    },
    {
      "epoch": 1.922810174204629,
      "grad_norm": 11.220113754272461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.687584400177002,
      "logits/rejected": -0.6399692296981812,
      "logps/chosen": -396.956298828125,
      "logps/rejected": -488.71533203125,
      "loss": 0.4919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.153157949447632,
      "rewards/margins": 1.2916337251663208,
      "rewards/rejected": -3.4447920322418213,
      "step": 14694
    },
    {
      "epoch": 1.9229410321419809,
      "grad_norm": 6.718522548675537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15423649549484253,
      "logits/rejected": -0.11584638804197311,
      "logps/chosen": -549.1162109375,
      "logps/rejected": -416.9877014160156,
      "loss": 0.3906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22644005715847015,
      "rewards/margins": 2.1901445388793945,
      "rewards/rejected": -2.4165844917297363,
      "step": 14695
    },
    {
      "epoch": 1.9230718900793327,
      "grad_norm": 4.317182540893555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09488515555858612,
      "logits/rejected": -0.5092142820358276,
      "logps/chosen": -381.1036682128906,
      "logps/rejected": -293.3535461425781,
      "loss": 0.1654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12463721632957458,
      "rewards/margins": 2.292092800140381,
      "rewards/rejected": -2.4167299270629883,
      "step": 14696
    },
    {
      "epoch": 1.9232027480166844,
      "grad_norm": 8.610443115234375,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07026483863592148,
      "logits/rejected": -0.009804636240005493,
      "logps/chosen": -622.0532836914062,
      "logps/rejected": -473.1868896484375,
      "loss": 0.4155,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7000586986541748,
      "rewards/margins": 1.1546568870544434,
      "rewards/rejected": -1.8547155857086182,
      "step": 14697
    },
    {
      "epoch": 1.9233336059540362,
      "grad_norm": 10.340843200683594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3211723268032074,
      "logits/rejected": -0.19381189346313477,
      "logps/chosen": -369.80853271484375,
      "logps/rejected": -388.1829528808594,
      "loss": 0.6911,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9784822463989258,
      "rewards/margins": 0.7078619003295898,
      "rewards/rejected": -1.6863442659378052,
      "step": 14698
    },
    {
      "epoch": 1.923464463891388,
      "grad_norm": 7.521302223205566,
      "learning_rate": 5e-06,
      "logits/chosen": 0.2912457585334778,
      "logits/rejected": 0.27373409271240234,
      "logps/chosen": -425.1695556640625,
      "logps/rejected": -499.8349914550781,
      "loss": 0.3135,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5649406909942627,
      "rewards/margins": 1.5683581829071045,
      "rewards/rejected": -3.1332991123199463,
      "step": 14699
    },
    {
      "epoch": 1.9235953218287398,
      "grad_norm": 8.892584800720215,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0049088746309280396,
      "logits/rejected": -0.38023388385772705,
      "logps/chosen": -525.4942016601562,
      "logps/rejected": -393.8360290527344,
      "loss": 0.3865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15751135349273682,
      "rewards/margins": 1.6309928894042969,
      "rewards/rejected": -1.7885042428970337,
      "step": 14700
    },
    {
      "epoch": 1.9237261797660914,
      "grad_norm": 10.792713165283203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3959120810031891,
      "logits/rejected": -0.06951864808797836,
      "logps/chosen": -460.9318542480469,
      "logps/rejected": -491.38555908203125,
      "loss": 0.5977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4569474458694458,
      "rewards/margins": 0.5949627161026001,
      "rewards/rejected": -2.0519096851348877,
      "step": 14701
    },
    {
      "epoch": 1.9238570377034432,
      "grad_norm": 13.376798629760742,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3797919750213623,
      "logits/rejected": -0.25824642181396484,
      "logps/chosen": -552.0388793945312,
      "logps/rejected": -461.7232360839844,
      "loss": 0.8983,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.282881736755371,
      "rewards/margins": 1.1976011991500854,
      "rewards/rejected": -2.480482816696167,
      "step": 14702
    },
    {
      "epoch": 1.923987895640795,
      "grad_norm": 12.457331657409668,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04711414873600006,
      "logits/rejected": -0.1943090260028839,
      "logps/chosen": -440.8997802734375,
      "logps/rejected": -432.101806640625,
      "loss": 0.7379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2918432652950287,
      "rewards/margins": 0.9883931279182434,
      "rewards/rejected": -1.2802364826202393,
      "step": 14703
    },
    {
      "epoch": 1.9241187535781468,
      "grad_norm": 5.877878189086914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0236373133957386,
      "logits/rejected": -0.22226670384407043,
      "logps/chosen": -516.3232421875,
      "logps/rejected": -551.4846801757812,
      "loss": 0.2354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5872301459312439,
      "rewards/margins": 2.2364346981048584,
      "rewards/rejected": -2.823664903640747,
      "step": 14704
    },
    {
      "epoch": 1.9242496115154983,
      "grad_norm": 8.27827262878418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10677345842123032,
      "logits/rejected": -0.2245243787765503,
      "logps/chosen": -418.4835205078125,
      "logps/rejected": -447.1002502441406,
      "loss": 0.501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2242530584335327,
      "rewards/margins": 0.8888974189758301,
      "rewards/rejected": -2.1131505966186523,
      "step": 14705
    },
    {
      "epoch": 1.9243804694528501,
      "grad_norm": 9.504355430603027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10516613721847534,
      "logits/rejected": 0.009152647107839584,
      "logps/chosen": -413.2779541015625,
      "logps/rejected": -387.63336181640625,
      "loss": 0.4788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8881601095199585,
      "rewards/margins": 1.1294771432876587,
      "rewards/rejected": -2.017637252807617,
      "step": 14706
    },
    {
      "epoch": 1.924511327390202,
      "grad_norm": 9.289165496826172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4053857922554016,
      "logits/rejected": -0.3415767252445221,
      "logps/chosen": -397.27020263671875,
      "logps/rejected": -388.1046142578125,
      "loss": 0.5923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9051827192306519,
      "rewards/margins": 0.45957237482070923,
      "rewards/rejected": -2.364755153656006,
      "step": 14707
    },
    {
      "epoch": 1.9246421853275537,
      "grad_norm": 10.865693092346191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43132174015045166,
      "logits/rejected": -0.3262431025505066,
      "logps/chosen": -397.944091796875,
      "logps/rejected": -375.560546875,
      "loss": 0.6202,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.446465015411377,
      "rewards/margins": 1.4086863994598389,
      "rewards/rejected": -2.8551511764526367,
      "step": 14708
    },
    {
      "epoch": 1.9247730432649055,
      "grad_norm": 12.696473121643066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11135923117399216,
      "logits/rejected": -0.14141449332237244,
      "logps/chosen": -477.1098937988281,
      "logps/rejected": -455.9889831542969,
      "loss": 0.7187,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8208677172660828,
      "rewards/margins": 0.5143821239471436,
      "rewards/rejected": -1.3352500200271606,
      "step": 14709
    },
    {
      "epoch": 1.9249039012022573,
      "grad_norm": 5.670882225036621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10293742269277573,
      "logits/rejected": -0.4220026135444641,
      "logps/chosen": -549.20751953125,
      "logps/rejected": -477.78253173828125,
      "loss": 0.3569,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14339064061641693,
      "rewards/margins": 1.995063066482544,
      "rewards/rejected": -1.8516724109649658,
      "step": 14710
    },
    {
      "epoch": 1.925034759139609,
      "grad_norm": 9.385436058044434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23920442163944244,
      "logits/rejected": -0.26852843165397644,
      "logps/chosen": -332.66375732421875,
      "logps/rejected": -497.10302734375,
      "loss": 0.367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0957906246185303,
      "rewards/margins": 1.3569360971450806,
      "rewards/rejected": -2.4527268409729004,
      "step": 14711
    },
    {
      "epoch": 1.9251656170769609,
      "grad_norm": 9.617453575134277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2015828788280487,
      "logits/rejected": -0.38867926597595215,
      "logps/chosen": -485.4429016113281,
      "logps/rejected": -366.22454833984375,
      "loss": 0.4772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1150381565093994,
      "rewards/margins": 1.3507741689682007,
      "rewards/rejected": -2.4658124446868896,
      "step": 14712
    },
    {
      "epoch": 1.9252964750143127,
      "grad_norm": 6.521022796630859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18952277302742004,
      "logits/rejected": -0.2791522145271301,
      "logps/chosen": -705.3485717773438,
      "logps/rejected": -433.979248046875,
      "loss": 0.2777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20838947594165802,
      "rewards/margins": 2.5380756855010986,
      "rewards/rejected": -2.7464652061462402,
      "step": 14713
    },
    {
      "epoch": 1.9254273329516645,
      "grad_norm": 9.711189270019531,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15253470838069916,
      "logits/rejected": 0.10121460258960724,
      "logps/chosen": -588.75244140625,
      "logps/rejected": -519.9638061523438,
      "loss": 0.5933,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6506883502006531,
      "rewards/margins": 0.7031852602958679,
      "rewards/rejected": -1.3538737297058105,
      "step": 14714
    },
    {
      "epoch": 1.9255581908890163,
      "grad_norm": 9.148865699768066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11089947819709778,
      "logits/rejected": -0.24483567476272583,
      "logps/chosen": -495.2062683105469,
      "logps/rejected": -677.318359375,
      "loss": 0.4226,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6709851026535034,
      "rewards/margins": 1.675686001777649,
      "rewards/rejected": -2.3466711044311523,
      "step": 14715
    },
    {
      "epoch": 1.9256890488263678,
      "grad_norm": 4.94929838180542,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028735749423503876,
      "logits/rejected": -0.07142258435487747,
      "logps/chosen": -377.8846130371094,
      "logps/rejected": -375.6064453125,
      "loss": 0.2043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5580343008041382,
      "rewards/margins": 1.9473233222961426,
      "rewards/rejected": -2.505357503890991,
      "step": 14716
    },
    {
      "epoch": 1.9258199067637196,
      "grad_norm": 8.161121368408203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32443538308143616,
      "logits/rejected": -0.45958632230758667,
      "logps/chosen": -491.3724365234375,
      "logps/rejected": -415.87237548828125,
      "loss": 0.4356,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9897491931915283,
      "rewards/margins": 2.095212459564209,
      "rewards/rejected": -3.0849616527557373,
      "step": 14717
    },
    {
      "epoch": 1.9259507647010714,
      "grad_norm": 8.677833557128906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20849815011024475,
      "logits/rejected": -0.32478487491607666,
      "logps/chosen": -524.0576171875,
      "logps/rejected": -484.59478759765625,
      "loss": 0.4348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.532887876033783,
      "rewards/margins": 1.0161303281784058,
      "rewards/rejected": -1.5490180253982544,
      "step": 14718
    },
    {
      "epoch": 1.9260816226384232,
      "grad_norm": 7.783034324645996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11344259977340698,
      "logits/rejected": -0.2553364634513855,
      "logps/chosen": -493.8067626953125,
      "logps/rejected": -429.13623046875,
      "loss": 0.3708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1836755871772766,
      "rewards/margins": 1.8232276439666748,
      "rewards/rejected": -2.0069031715393066,
      "step": 14719
    },
    {
      "epoch": 1.9262124805757748,
      "grad_norm": 8.529886245727539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22392934560775757,
      "logits/rejected": -0.3416101932525635,
      "logps/chosen": -629.6696166992188,
      "logps/rejected": -449.140869140625,
      "loss": 0.3424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7354457974433899,
      "rewards/margins": 1.344749093055725,
      "rewards/rejected": -2.0801949501037598,
      "step": 14720
    },
    {
      "epoch": 1.9263433385131266,
      "grad_norm": 4.271259784698486,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2921699285507202,
      "logits/rejected": -0.19644209742546082,
      "logps/chosen": -555.5523681640625,
      "logps/rejected": -578.7326049804688,
      "loss": 0.1859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0783313512802124,
      "rewards/margins": 2.100799560546875,
      "rewards/rejected": -3.179130792617798,
      "step": 14721
    },
    {
      "epoch": 1.9264741964504783,
      "grad_norm": 10.693836212158203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34889736771583557,
      "logits/rejected": -0.5466780662536621,
      "logps/chosen": -445.4366149902344,
      "logps/rejected": -480.5059814453125,
      "loss": 0.523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9600169658660889,
      "rewards/margins": 0.7586986422538757,
      "rewards/rejected": -1.718715786933899,
      "step": 14722
    },
    {
      "epoch": 1.9266050543878301,
      "grad_norm": 6.904438495635986,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03141070157289505,
      "logits/rejected": -0.11958431452512741,
      "logps/chosen": -430.0486755371094,
      "logps/rejected": -536.0825805664062,
      "loss": 0.3897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4846944808959961,
      "rewards/margins": 1.7769155502319336,
      "rewards/rejected": -2.2616100311279297,
      "step": 14723
    },
    {
      "epoch": 1.926735912325182,
      "grad_norm": 10.725330352783203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36845073103904724,
      "logits/rejected": -0.4651365280151367,
      "logps/chosen": -352.8260498046875,
      "logps/rejected": -381.0296936035156,
      "loss": 0.5948,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.663061499595642,
      "rewards/margins": 0.8726524114608765,
      "rewards/rejected": -2.5357136726379395,
      "step": 14724
    },
    {
      "epoch": 1.9268667702625337,
      "grad_norm": 6.822666168212891,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05890895053744316,
      "logits/rejected": -0.11013894528150558,
      "logps/chosen": -503.9810791015625,
      "logps/rejected": -380.61334228515625,
      "loss": 0.285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6414174437522888,
      "rewards/margins": 1.683792233467102,
      "rewards/rejected": -2.325209617614746,
      "step": 14725
    },
    {
      "epoch": 1.9269976281998855,
      "grad_norm": 11.931021690368652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3165510296821594,
      "logits/rejected": -0.24621692299842834,
      "logps/chosen": -321.9443054199219,
      "logps/rejected": -416.08514404296875,
      "loss": 1.0371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6328648328781128,
      "rewards/margins": 0.4874702990055084,
      "rewards/rejected": -1.1203352212905884,
      "step": 14726
    },
    {
      "epoch": 1.9271284861372373,
      "grad_norm": 10.82071590423584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2009112387895584,
      "logits/rejected": -0.15932147204875946,
      "logps/chosen": -627.29052734375,
      "logps/rejected": -433.79742431640625,
      "loss": 0.4895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.088154673576355,
      "rewards/margins": 0.9528185725212097,
      "rewards/rejected": -2.040973424911499,
      "step": 14727
    },
    {
      "epoch": 1.927259344074589,
      "grad_norm": 7.440184593200684,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08021939545869827,
      "logits/rejected": -0.15953652560710907,
      "logps/chosen": -551.4091796875,
      "logps/rejected": -571.1788940429688,
      "loss": 0.3169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7738279104232788,
      "rewards/margins": 1.5378495454788208,
      "rewards/rejected": -2.3116776943206787,
      "step": 14728
    },
    {
      "epoch": 1.927390202011941,
      "grad_norm": 11.759108543395996,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19881673157215118,
      "logits/rejected": -0.05563900247216225,
      "logps/chosen": -444.40576171875,
      "logps/rejected": -410.4552307128906,
      "loss": 0.6507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9629275798797607,
      "rewards/margins": 0.6445493698120117,
      "rewards/rejected": -2.6074767112731934,
      "step": 14729
    },
    {
      "epoch": 1.9275210599492927,
      "grad_norm": 6.3204145431518555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5362701416015625,
      "logits/rejected": -0.2932736873626709,
      "logps/chosen": -410.35968017578125,
      "logps/rejected": -408.8751525878906,
      "loss": 0.2621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7321226596832275,
      "rewards/margins": 1.4171204566955566,
      "rewards/rejected": -2.149243116378784,
      "step": 14730
    },
    {
      "epoch": 1.9276519178866445,
      "grad_norm": 6.374972820281982,
      "learning_rate": 5e-06,
      "logits/chosen": -0.028552040457725525,
      "logits/rejected": -0.20836764574050903,
      "logps/chosen": -667.822509765625,
      "logps/rejected": -688.3792724609375,
      "loss": 0.1802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04487761855125427,
      "rewards/margins": 3.146803140640259,
      "rewards/rejected": -3.191680908203125,
      "step": 14731
    },
    {
      "epoch": 1.927782775823996,
      "grad_norm": 6.166441917419434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.112500861287117,
      "logits/rejected": -0.3916843831539154,
      "logps/chosen": -525.8118896484375,
      "logps/rejected": -364.86370849609375,
      "loss": 0.3424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20619335770606995,
      "rewards/margins": 1.8394643068313599,
      "rewards/rejected": -2.0456578731536865,
      "step": 14732
    },
    {
      "epoch": 1.9279136337613478,
      "grad_norm": 7.272904396057129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.023851055651903152,
      "logits/rejected": 0.09661225229501724,
      "logps/chosen": -531.2019653320312,
      "logps/rejected": -502.64410400390625,
      "loss": 0.2373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6908499002456665,
      "rewards/margins": 1.9205559492111206,
      "rewards/rejected": -2.611405849456787,
      "step": 14733
    },
    {
      "epoch": 1.9280444916986996,
      "grad_norm": 8.711753845214844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28249379992485046,
      "logits/rejected": -0.37412193417549133,
      "logps/chosen": -458.4527587890625,
      "logps/rejected": -366.14031982421875,
      "loss": 0.3598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4042943716049194,
      "rewards/margins": 2.076507568359375,
      "rewards/rejected": -3.480802059173584,
      "step": 14734
    },
    {
      "epoch": 1.9281753496360514,
      "grad_norm": 10.169661521911621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3080577850341797,
      "logits/rejected": -0.25976139307022095,
      "logps/chosen": -514.9215087890625,
      "logps/rejected": -521.3789672851562,
      "loss": 0.3827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.945625901222229,
      "rewards/margins": 1.689383625984192,
      "rewards/rejected": -2.635009288787842,
      "step": 14735
    },
    {
      "epoch": 1.928306207573403,
      "grad_norm": 4.261481761932373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1409231573343277,
      "logits/rejected": -0.027340620756149292,
      "logps/chosen": -413.8740539550781,
      "logps/rejected": -380.2349853515625,
      "loss": 0.1892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8465690612792969,
      "rewards/margins": 2.2230498790740967,
      "rewards/rejected": -3.0696189403533936,
      "step": 14736
    },
    {
      "epoch": 1.9284370655107548,
      "grad_norm": 13.742417335510254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15582937002182007,
      "logits/rejected": -0.1645599901676178,
      "logps/chosen": -721.7799682617188,
      "logps/rejected": -467.0682373046875,
      "loss": 0.5816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11948129534721375,
      "rewards/margins": 0.8204506039619446,
      "rewards/rejected": -0.9399318695068359,
      "step": 14737
    },
    {
      "epoch": 1.9285679234481066,
      "grad_norm": 3.9218251705169678,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043301526457071304,
      "logits/rejected": -0.26783862709999084,
      "logps/chosen": -403.289794921875,
      "logps/rejected": -315.43896484375,
      "loss": 0.1834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7052659392356873,
      "rewards/margins": 2.6679742336273193,
      "rewards/rejected": -3.3732399940490723,
      "step": 14738
    },
    {
      "epoch": 1.9286987813854584,
      "grad_norm": 11.637101173400879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.354638010263443,
      "logits/rejected": -0.2746131420135498,
      "logps/chosen": -418.5167236328125,
      "logps/rejected": -462.0496520996094,
      "loss": 0.6275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.706324577331543,
      "rewards/margins": 1.0394299030303955,
      "rewards/rejected": -1.7457544803619385,
      "step": 14739
    },
    {
      "epoch": 1.9288296393228102,
      "grad_norm": 6.170796871185303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18217329680919647,
      "logits/rejected": -0.35140538215637207,
      "logps/chosen": -314.856689453125,
      "logps/rejected": -422.4715270996094,
      "loss": 0.266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7761052846908569,
      "rewards/margins": 2.291618585586548,
      "rewards/rejected": -3.0677239894866943,
      "step": 14740
    },
    {
      "epoch": 1.928960497260162,
      "grad_norm": 11.094812393188477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18746469914913177,
      "logits/rejected": -0.2941700518131256,
      "logps/chosen": -466.8038635253906,
      "logps/rejected": -426.8487548828125,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9983580112457275,
      "rewards/margins": 0.2605733871459961,
      "rewards/rejected": -1.2589313983917236,
      "step": 14741
    },
    {
      "epoch": 1.9290913551975137,
      "grad_norm": 9.167337417602539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2816867232322693,
      "logits/rejected": -0.20119741559028625,
      "logps/chosen": -432.7132568359375,
      "logps/rejected": -454.1187744140625,
      "loss": 0.4847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9470430612564087,
      "rewards/margins": 1.1963598728179932,
      "rewards/rejected": -2.1434028148651123,
      "step": 14742
    },
    {
      "epoch": 1.9292222131348655,
      "grad_norm": 7.9761247634887695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22956165671348572,
      "logits/rejected": -0.406477153301239,
      "logps/chosen": -657.951416015625,
      "logps/rejected": -657.2818603515625,
      "loss": 0.3631,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4833993911743164,
      "rewards/margins": 1.6864897012710571,
      "rewards/rejected": -2.169889211654663,
      "step": 14743
    },
    {
      "epoch": 1.9293530710722173,
      "grad_norm": 6.548792839050293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3499235510826111,
      "logits/rejected": -0.4825063943862915,
      "logps/chosen": -805.159423828125,
      "logps/rejected": -561.03173828125,
      "loss": 0.2523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06557254493236542,
      "rewards/margins": 1.8113932609558105,
      "rewards/rejected": -1.876965880393982,
      "step": 14744
    },
    {
      "epoch": 1.929483929009569,
      "grad_norm": 5.577779769897461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.436531662940979,
      "logits/rejected": -0.5015188455581665,
      "logps/chosen": -368.9049987792969,
      "logps/rejected": -352.08367919921875,
      "loss": 0.2803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3750100135803223,
      "rewards/margins": 2.148023843765259,
      "rewards/rejected": -3.523033857345581,
      "step": 14745
    },
    {
      "epoch": 1.929614786946921,
      "grad_norm": 10.637619972229004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2268863320350647,
      "logits/rejected": -0.3595786690711975,
      "logps/chosen": -424.2886047363281,
      "logps/rejected": -511.18133544921875,
      "loss": 0.4333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9727113842964172,
      "rewards/margins": 1.0996156930923462,
      "rewards/rejected": -2.072327136993408,
      "step": 14746
    },
    {
      "epoch": 1.9297456448842725,
      "grad_norm": 4.122369766235352,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31652623414993286,
      "logits/rejected": -0.5244765281677246,
      "logps/chosen": -368.0998840332031,
      "logps/rejected": -381.3138122558594,
      "loss": 0.1909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6667632460594177,
      "rewards/margins": 2.1249377727508545,
      "rewards/rejected": -2.791701316833496,
      "step": 14747
    },
    {
      "epoch": 1.9298765028216243,
      "grad_norm": 6.939834117889404,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060169704258441925,
      "logits/rejected": -0.021203845739364624,
      "logps/chosen": -481.7757263183594,
      "logps/rejected": -433.4386291503906,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09064312279224396,
      "rewards/margins": 1.801020860671997,
      "rewards/rejected": -1.8916640281677246,
      "step": 14748
    },
    {
      "epoch": 1.930007360758976,
      "grad_norm": 10.463481903076172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.184993714094162,
      "logits/rejected": -0.13310155272483826,
      "logps/chosen": -515.0658569335938,
      "logps/rejected": -338.46392822265625,
      "loss": 0.2906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4286775588989258,
      "rewards/margins": 2.1719465255737305,
      "rewards/rejected": -3.6006240844726562,
      "step": 14749
    },
    {
      "epoch": 1.9301382186963278,
      "grad_norm": 8.455924034118652,
      "learning_rate": 5e-06,
      "logits/chosen": 0.029926370829343796,
      "logits/rejected": 0.03982701897621155,
      "logps/chosen": -597.4530639648438,
      "logps/rejected": -609.526611328125,
      "loss": 0.3565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5244272947311401,
      "rewards/margins": 1.156905174255371,
      "rewards/rejected": -2.6813323497772217,
      "step": 14750
    },
    {
      "epoch": 1.9302690766336794,
      "grad_norm": 15.25151252746582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3973780870437622,
      "logits/rejected": -0.47878342866897583,
      "logps/chosen": -471.17254638671875,
      "logps/rejected": -494.1887512207031,
      "loss": 1.0363,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.9707846641540527,
      "rewards/margins": -0.23583713173866272,
      "rewards/rejected": -1.7349472045898438,
      "step": 14751
    },
    {
      "epoch": 1.9303999345710312,
      "grad_norm": 9.983565330505371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3730166554450989,
      "logits/rejected": -0.5575970411300659,
      "logps/chosen": -482.1446533203125,
      "logps/rejected": -468.905029296875,
      "loss": 0.3856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.526951014995575,
      "rewards/margins": 1.093286156654358,
      "rewards/rejected": -1.620237112045288,
      "step": 14752
    },
    {
      "epoch": 1.930530792508383,
      "grad_norm": 12.658387184143066,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10378091782331467,
      "logits/rejected": -0.2647646963596344,
      "logps/chosen": -393.7986145019531,
      "logps/rejected": -359.7613525390625,
      "loss": 0.7917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0674874782562256,
      "rewards/margins": 0.4747534990310669,
      "rewards/rejected": -1.542241096496582,
      "step": 14753
    },
    {
      "epoch": 1.9306616504457348,
      "grad_norm": 10.163052558898926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3883121609687805,
      "logits/rejected": -0.4346713423728943,
      "logps/chosen": -526.9407958984375,
      "logps/rejected": -503.08868408203125,
      "loss": 0.408,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4669134616851807,
      "rewards/margins": 1.5225529670715332,
      "rewards/rejected": -2.989466667175293,
      "step": 14754
    },
    {
      "epoch": 1.9307925083830866,
      "grad_norm": 9.332633018493652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39114922285079956,
      "logits/rejected": -0.31119269132614136,
      "logps/chosen": -431.5928649902344,
      "logps/rejected": -291.4835205078125,
      "loss": 0.4471,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44478338956832886,
      "rewards/margins": 1.1691325902938843,
      "rewards/rejected": -1.6139161586761475,
      "step": 14755
    },
    {
      "epoch": 1.9309233663204384,
      "grad_norm": 7.7292962074279785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19120880961418152,
      "logits/rejected": -0.18913060426712036,
      "logps/chosen": -485.2027282714844,
      "logps/rejected": -477.89764404296875,
      "loss": 0.3361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9715803861618042,
      "rewards/margins": 1.4313305616378784,
      "rewards/rejected": -2.4029109477996826,
      "step": 14756
    },
    {
      "epoch": 1.9310542242577902,
      "grad_norm": 9.173383712768555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03976533189415932,
      "logits/rejected": -0.20822517573833466,
      "logps/chosen": -450.70831298828125,
      "logps/rejected": -338.27447509765625,
      "loss": 0.4429,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.537013053894043,
      "rewards/margins": 1.226205587387085,
      "rewards/rejected": -2.763218879699707,
      "step": 14757
    },
    {
      "epoch": 1.931185082195142,
      "grad_norm": 6.891988754272461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16890794038772583,
      "logits/rejected": -0.4140227138996124,
      "logps/chosen": -793.3955078125,
      "logps/rejected": -484.61676025390625,
      "loss": 0.3468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7244491577148438,
      "rewards/margins": 1.7214173078536987,
      "rewards/rejected": -2.445866346359253,
      "step": 14758
    },
    {
      "epoch": 1.9313159401324937,
      "grad_norm": 10.160917282104492,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40614765882492065,
      "logits/rejected": -0.2460421621799469,
      "logps/chosen": -425.67974853515625,
      "logps/rejected": -558.5288696289062,
      "loss": 0.5553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1406278610229492,
      "rewards/margins": 1.0132509469985962,
      "rewards/rejected": -2.153878927230835,
      "step": 14759
    },
    {
      "epoch": 1.9314467980698455,
      "grad_norm": 7.916110515594482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26511329412460327,
      "logits/rejected": 0.010842043906450272,
      "logps/chosen": -626.8441772460938,
      "logps/rejected": -561.455810546875,
      "loss": 0.3056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8482823371887207,
      "rewards/margins": 1.9296717643737793,
      "rewards/rejected": -2.7779541015625,
      "step": 14760
    },
    {
      "epoch": 1.9315776560071973,
      "grad_norm": 11.109046936035156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20038548111915588,
      "logits/rejected": -0.41162657737731934,
      "logps/chosen": -459.775634765625,
      "logps/rejected": -591.4169311523438,
      "loss": 0.5047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6314388513565063,
      "rewards/margins": 1.6753727197647095,
      "rewards/rejected": -3.306811571121216,
      "step": 14761
    },
    {
      "epoch": 1.931708513944549,
      "grad_norm": 4.833070278167725,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030103325843811035,
      "logits/rejected": -0.17647014558315277,
      "logps/chosen": -444.0522766113281,
      "logps/rejected": -457.38482666015625,
      "loss": 0.1556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4758850634098053,
      "rewards/margins": 2.3023579120635986,
      "rewards/rejected": -2.778243064880371,
      "step": 14762
    },
    {
      "epoch": 1.9318393718819007,
      "grad_norm": 8.411535263061523,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02193874679505825,
      "logits/rejected": -0.20630814135074615,
      "logps/chosen": -344.39453125,
      "logps/rejected": -350.1953125,
      "loss": 0.3592,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3884494304656982,
      "rewards/margins": 1.529571533203125,
      "rewards/rejected": -2.9180212020874023,
      "step": 14763
    },
    {
      "epoch": 1.9319702298192525,
      "grad_norm": 9.309516906738281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15297862887382507,
      "logits/rejected": -0.39093732833862305,
      "logps/chosen": -361.51318359375,
      "logps/rejected": -323.8931579589844,
      "loss": 0.4966,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8492317795753479,
      "rewards/margins": 1.3331947326660156,
      "rewards/rejected": -2.1824264526367188,
      "step": 14764
    },
    {
      "epoch": 1.9321010877566043,
      "grad_norm": 6.33702278137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10021507740020752,
      "logits/rejected": -0.1934823989868164,
      "logps/chosen": -391.81097412109375,
      "logps/rejected": -329.77545166015625,
      "loss": 0.4501,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0008951425552368,
      "rewards/margins": 0.764930248260498,
      "rewards/rejected": -1.7658252716064453,
      "step": 14765
    },
    {
      "epoch": 1.9322319456939558,
      "grad_norm": 6.188359260559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33202505111694336,
      "logits/rejected": -0.3220190107822418,
      "logps/chosen": -385.896484375,
      "logps/rejected": -584.2752075195312,
      "loss": 0.2337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8491884469985962,
      "rewards/margins": 2.8560264110565186,
      "rewards/rejected": -3.705214500427246,
      "step": 14766
    },
    {
      "epoch": 1.9323628036313076,
      "grad_norm": 4.99847936630249,
      "learning_rate": 5e-06,
      "logits/chosen": -0.367620050907135,
      "logits/rejected": -0.15350350737571716,
      "logps/chosen": -490.2440490722656,
      "logps/rejected": -384.50244140625,
      "loss": 0.1964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0208765268325806,
      "rewards/margins": 2.1417887210845947,
      "rewards/rejected": -3.162665605545044,
      "step": 14767
    },
    {
      "epoch": 1.9324936615686594,
      "grad_norm": 5.715097904205322,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25291645526885986,
      "logits/rejected": -0.32889771461486816,
      "logps/chosen": -387.0074768066406,
      "logps/rejected": -336.79327392578125,
      "loss": 0.3405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.686877965927124,
      "rewards/margins": 1.2862727642059326,
      "rewards/rejected": -2.9731507301330566,
      "step": 14768
    },
    {
      "epoch": 1.9326245195060112,
      "grad_norm": 10.264533996582031,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01756707951426506,
      "logits/rejected": -0.17477165162563324,
      "logps/chosen": -526.2081298828125,
      "logps/rejected": -504.4210510253906,
      "loss": 0.3077,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2410848140716553,
      "rewards/margins": 1.8145846128463745,
      "rewards/rejected": -3.0556693077087402,
      "step": 14769
    },
    {
      "epoch": 1.932755377443363,
      "grad_norm": 6.990040302276611,
      "learning_rate": 5e-06,
      "logits/chosen": 0.15529197454452515,
      "logits/rejected": -0.05071381479501724,
      "logps/chosen": -354.61358642578125,
      "logps/rejected": -497.366455078125,
      "loss": 0.4898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.268384575843811,
      "rewards/margins": 1.8408491611480713,
      "rewards/rejected": -3.1092336177825928,
      "step": 14770
    },
    {
      "epoch": 1.9328862353807148,
      "grad_norm": 11.495593070983887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3272014856338501,
      "logits/rejected": -0.3174285888671875,
      "logps/chosen": -496.26275634765625,
      "logps/rejected": -432.46417236328125,
      "loss": 0.5411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7368242740631104,
      "rewards/margins": 1.1315805912017822,
      "rewards/rejected": -1.868404746055603,
      "step": 14771
    },
    {
      "epoch": 1.9330170933180666,
      "grad_norm": 6.59488582611084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03852629289031029,
      "logits/rejected": -0.24799947440624237,
      "logps/chosen": -380.64306640625,
      "logps/rejected": -405.16448974609375,
      "loss": 0.2376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16963598132133484,
      "rewards/margins": 1.9899126291275024,
      "rewards/rejected": -2.15954852104187,
      "step": 14772
    },
    {
      "epoch": 1.9331479512554184,
      "grad_norm": 4.468404293060303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19052475690841675,
      "logits/rejected": -0.2168312966823578,
      "logps/chosen": -493.17791748046875,
      "logps/rejected": -377.7718505859375,
      "loss": 0.2029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5415364503860474,
      "rewards/margins": 2.053164005279541,
      "rewards/rejected": -2.594700336456299,
      "step": 14773
    },
    {
      "epoch": 1.9332788091927702,
      "grad_norm": 8.101936340332031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24011890590190887,
      "logits/rejected": -0.595198392868042,
      "logps/chosen": -292.5021057128906,
      "logps/rejected": -271.04730224609375,
      "loss": 0.4906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.955296516418457,
      "rewards/margins": 1.0543403625488281,
      "rewards/rejected": -2.009636878967285,
      "step": 14774
    },
    {
      "epoch": 1.933409667130122,
      "grad_norm": 5.505444526672363,
      "learning_rate": 5e-06,
      "logits/chosen": 0.021880831569433212,
      "logits/rejected": -0.14308376610279083,
      "logps/chosen": -542.5001220703125,
      "logps/rejected": -475.84173583984375,
      "loss": 0.28,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6468309164047241,
      "rewards/margins": 2.4619669914245605,
      "rewards/rejected": -3.108797788619995,
      "step": 14775
    },
    {
      "epoch": 1.9335405250674738,
      "grad_norm": 7.859689235687256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07630114257335663,
      "logits/rejected": -0.11999396979808807,
      "logps/chosen": -433.99237060546875,
      "logps/rejected": -432.6080017089844,
      "loss": 0.3089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10467356443405151,
      "rewards/margins": 2.712171792984009,
      "rewards/rejected": -2.816844940185547,
      "step": 14776
    },
    {
      "epoch": 1.9336713830048256,
      "grad_norm": 11.095808029174805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1122806966304779,
      "logits/rejected": -0.31215405464172363,
      "logps/chosen": -504.9309387207031,
      "logps/rejected": -601.2945556640625,
      "loss": 0.5437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5436053276062012,
      "rewards/margins": 1.3499236106872559,
      "rewards/rejected": -2.893528938293457,
      "step": 14777
    },
    {
      "epoch": 1.9338022409421771,
      "grad_norm": 9.18989086151123,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6608081459999084,
      "logits/rejected": -0.6905325651168823,
      "logps/chosen": -355.63360595703125,
      "logps/rejected": -380.5068664550781,
      "loss": 0.7069,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2413603067398071,
      "rewards/margins": 0.174118772149086,
      "rewards/rejected": -1.415479063987732,
      "step": 14778
    },
    {
      "epoch": 1.933933098879529,
      "grad_norm": 8.318060874938965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1955314725637436,
      "logits/rejected": -0.121272012591362,
      "logps/chosen": -420.4510498046875,
      "logps/rejected": -512.0887451171875,
      "loss": 0.3866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006403543055057526,
      "rewards/margins": 1.5925439596176147,
      "rewards/rejected": -1.5861403942108154,
      "step": 14779
    },
    {
      "epoch": 1.9340639568168807,
      "grad_norm": 7.4491963386535645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6497134566307068,
      "logits/rejected": -0.32091647386550903,
      "logps/chosen": -440.8157653808594,
      "logps/rejected": -760.1976928710938,
      "loss": 0.2962,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8954399228096008,
      "rewards/margins": 2.227511405944824,
      "rewards/rejected": -3.122951030731201,
      "step": 14780
    },
    {
      "epoch": 1.9341948147542323,
      "grad_norm": 5.552928924560547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28471478819847107,
      "logits/rejected": -0.3892841935157776,
      "logps/chosen": -443.20074462890625,
      "logps/rejected": -485.2970275878906,
      "loss": 0.2294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.572886347770691,
      "rewards/margins": 1.6547465324401855,
      "rewards/rejected": -3.227633237838745,
      "step": 14781
    },
    {
      "epoch": 1.934325672691584,
      "grad_norm": 9.690781593322754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2111484557390213,
      "logits/rejected": -0.2621898651123047,
      "logps/chosen": -429.36932373046875,
      "logps/rejected": -421.2135009765625,
      "loss": 0.4822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6479451656341553,
      "rewards/margins": 0.7107513546943665,
      "rewards/rejected": -2.358696460723877,
      "step": 14782
    },
    {
      "epoch": 1.9344565306289359,
      "grad_norm": 7.442964553833008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06656729429960251,
      "logits/rejected": -0.21746791899204254,
      "logps/chosen": -525.9812622070312,
      "logps/rejected": -450.86773681640625,
      "loss": 0.3655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8518526554107666,
      "rewards/margins": 1.5218377113342285,
      "rewards/rejected": -2.373690366744995,
      "step": 14783
    },
    {
      "epoch": 1.9345873885662876,
      "grad_norm": 9.264501571655273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23941025137901306,
      "logits/rejected": -0.14327096939086914,
      "logps/chosen": -521.0499877929688,
      "logps/rejected": -433.70849609375,
      "loss": 0.3912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.088695764541626,
      "rewards/margins": 2.161273717880249,
      "rewards/rejected": -3.249969482421875,
      "step": 14784
    },
    {
      "epoch": 1.9347182465036394,
      "grad_norm": 12.30979061126709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18719013035297394,
      "logits/rejected": -0.20961381494998932,
      "logps/chosen": -461.55010986328125,
      "logps/rejected": -412.05322265625,
      "loss": 0.684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26710277795791626,
      "rewards/margins": 1.1780059337615967,
      "rewards/rejected": -1.4451087713241577,
      "step": 14785
    },
    {
      "epoch": 1.9348491044409912,
      "grad_norm": 8.755428314208984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04076934978365898,
      "logits/rejected": 0.17637057602405548,
      "logps/chosen": -476.5783996582031,
      "logps/rejected": -605.7493896484375,
      "loss": 0.3976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6264562606811523,
      "rewards/margins": 2.6295323371887207,
      "rewards/rejected": -4.255988597869873,
      "step": 14786
    },
    {
      "epoch": 1.934979962378343,
      "grad_norm": 13.597246170043945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20614677667617798,
      "logits/rejected": -0.5004740357398987,
      "logps/chosen": -467.07757568359375,
      "logps/rejected": -586.545166015625,
      "loss": 0.4626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6227761507034302,
      "rewards/margins": 2.3030099868774414,
      "rewards/rejected": -3.925786256790161,
      "step": 14787
    },
    {
      "epoch": 1.9351108203156948,
      "grad_norm": 5.290597438812256,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27657783031463623,
      "logits/rejected": -0.4474475681781769,
      "logps/chosen": -661.9554443359375,
      "logps/rejected": -381.7462158203125,
      "loss": 0.2558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.93071049451828,
      "rewards/margins": 2.355405807495117,
      "rewards/rejected": -3.286116361618042,
      "step": 14788
    },
    {
      "epoch": 1.9352416782530466,
      "grad_norm": 6.479827880859375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27870720624923706,
      "logits/rejected": -0.527029275894165,
      "logps/chosen": -347.9195556640625,
      "logps/rejected": -375.19219970703125,
      "loss": 0.2998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.105757713317871,
      "rewards/margins": 1.7096571922302246,
      "rewards/rejected": -2.8154149055480957,
      "step": 14789
    },
    {
      "epoch": 1.9353725361903984,
      "grad_norm": 10.540634155273438,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18902671337127686,
      "logits/rejected": -0.1786072850227356,
      "logps/chosen": -370.2248229980469,
      "logps/rejected": -537.3447875976562,
      "loss": 0.5425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5046372413635254,
      "rewards/margins": 1.830054759979248,
      "rewards/rejected": -3.3346920013427734,
      "step": 14790
    },
    {
      "epoch": 1.9355033941277502,
      "grad_norm": 13.16061782836914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5892196893692017,
      "logits/rejected": -0.6749282479286194,
      "logps/chosen": -405.9127502441406,
      "logps/rejected": -424.799560546875,
      "loss": 0.8688,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -2.130615711212158,
      "rewards/margins": 0.01822642982006073,
      "rewards/rejected": -2.1488423347473145,
      "step": 14791
    },
    {
      "epoch": 1.935634252065102,
      "grad_norm": 15.15782356262207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5126347541809082,
      "logits/rejected": -0.433724969625473,
      "logps/chosen": -632.5165405273438,
      "logps/rejected": -456.2615966796875,
      "loss": 0.6175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2962112426757812,
      "rewards/margins": 1.3105218410491943,
      "rewards/rejected": -3.606733560562134,
      "step": 14792
    },
    {
      "epoch": 1.9357651100024535,
      "grad_norm": 11.350947380065918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31306612491607666,
      "logits/rejected": -0.2691202163696289,
      "logps/chosen": -679.4769287109375,
      "logps/rejected": -533.8021850585938,
      "loss": 0.4563,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.692833662033081,
      "rewards/margins": 2.514631986618042,
      "rewards/rejected": -3.207465887069702,
      "step": 14793
    },
    {
      "epoch": 1.9358959679398053,
      "grad_norm": 8.763506889343262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20996984839439392,
      "logits/rejected": -0.3617889881134033,
      "logps/chosen": -623.6036376953125,
      "logps/rejected": -431.3398132324219,
      "loss": 0.3599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1959315538406372,
      "rewards/margins": 2.1060895919799805,
      "rewards/rejected": -3.302021026611328,
      "step": 14794
    },
    {
      "epoch": 1.9360268258771571,
      "grad_norm": 8.331518173217773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10448460280895233,
      "logits/rejected": 0.006337612867355347,
      "logps/chosen": -351.37823486328125,
      "logps/rejected": -369.55859375,
      "loss": 0.3487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4784659147262573,
      "rewards/margins": 1.7161445617675781,
      "rewards/rejected": -3.194610357284546,
      "step": 14795
    },
    {
      "epoch": 1.936157683814509,
      "grad_norm": 4.689083576202393,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38281095027923584,
      "logits/rejected": -0.4149387776851654,
      "logps/chosen": -361.6198425292969,
      "logps/rejected": -317.4349670410156,
      "loss": 0.173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5687321424484253,
      "rewards/margins": 1.898403525352478,
      "rewards/rejected": -2.4671356678009033,
      "step": 14796
    },
    {
      "epoch": 1.9362885417518605,
      "grad_norm": 12.036471366882324,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29197731614112854,
      "logits/rejected": -0.2458721101284027,
      "logps/chosen": -600.453125,
      "logps/rejected": -595.94970703125,
      "loss": 0.7848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1159435510635376,
      "rewards/margins": 0.8497591018676758,
      "rewards/rejected": -1.965702772140503,
      "step": 14797
    },
    {
      "epoch": 1.9364193996892123,
      "grad_norm": 15.113072395324707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3456195592880249,
      "logits/rejected": -0.3980933427810669,
      "logps/chosen": -396.5743713378906,
      "logps/rejected": -493.7956237792969,
      "loss": 0.5909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7380416989326477,
      "rewards/margins": 1.4880726337432861,
      "rewards/rejected": -2.226114511489868,
      "step": 14798
    },
    {
      "epoch": 1.936550257626564,
      "grad_norm": 14.272379875183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3613540530204773,
      "logits/rejected": -0.20580093562602997,
      "logps/chosen": -435.705810546875,
      "logps/rejected": -484.6535339355469,
      "loss": 0.7605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5367788076400757,
      "rewards/margins": 1.1810446977615356,
      "rewards/rejected": -2.7178235054016113,
      "step": 14799
    },
    {
      "epoch": 1.9366811155639159,
      "grad_norm": 7.840482711791992,
      "learning_rate": 5e-06,
      "logits/chosen": 0.016405470669269562,
      "logits/rejected": -0.18568141758441925,
      "logps/chosen": -482.5450439453125,
      "logps/rejected": -530.2962646484375,
      "loss": 0.2256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9998728036880493,
      "rewards/margins": 3.2792391777038574,
      "rewards/rejected": -4.279111862182617,
      "step": 14800
    },
    {
      "epoch": 1.9368119735012677,
      "grad_norm": 3.0521490573883057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45445552468299866,
      "logits/rejected": -0.347807914018631,
      "logps/chosen": -745.3296508789062,
      "logps/rejected": -466.28350830078125,
      "loss": 0.0948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5735126733779907,
      "rewards/margins": 3.1492228507995605,
      "rewards/rejected": -3.7227354049682617,
      "step": 14801
    },
    {
      "epoch": 1.9369428314386195,
      "grad_norm": 5.888744831085205,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05566930025815964,
      "logits/rejected": -0.2159651517868042,
      "logps/chosen": -556.0873413085938,
      "logps/rejected": -570.76025390625,
      "loss": 0.1506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3765518069267273,
      "rewards/margins": 2.94101619720459,
      "rewards/rejected": -2.564464569091797,
      "step": 14802
    },
    {
      "epoch": 1.9370736893759712,
      "grad_norm": 7.242635726928711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03456743806600571,
      "logits/rejected": -0.256273090839386,
      "logps/chosen": -529.8903198242188,
      "logps/rejected": -329.867431640625,
      "loss": 0.2915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4262123107910156,
      "rewards/margins": 2.544769287109375,
      "rewards/rejected": -2.9709818363189697,
      "step": 14803
    },
    {
      "epoch": 1.937204547313323,
      "grad_norm": 3.42655348777771,
      "learning_rate": 5e-06,
      "logits/chosen": 0.054788898676633835,
      "logits/rejected": -0.06169569492340088,
      "logps/chosen": -526.1005249023438,
      "logps/rejected": -507.42376708984375,
      "loss": 0.1215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17633819580078125,
      "rewards/margins": 2.9047484397888184,
      "rewards/rejected": -3.0810866355895996,
      "step": 14804
    },
    {
      "epoch": 1.9373354052506748,
      "grad_norm": 5.672080993652344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11561558395624161,
      "logits/rejected": -0.1785372793674469,
      "logps/chosen": -418.8699951171875,
      "logps/rejected": -541.650634765625,
      "loss": 0.2464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08634796738624573,
      "rewards/margins": 1.9243324995040894,
      "rewards/rejected": -2.0106801986694336,
      "step": 14805
    },
    {
      "epoch": 1.9374662631880266,
      "grad_norm": 15.203523635864258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21829383075237274,
      "logits/rejected": -0.40697064995765686,
      "logps/chosen": -590.9376220703125,
      "logps/rejected": -648.470703125,
      "loss": 0.6857,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.6624467372894287,
      "rewards/margins": 1.2593296766281128,
      "rewards/rejected": -3.921776294708252,
      "step": 14806
    },
    {
      "epoch": 1.9375971211253784,
      "grad_norm": 9.165188789367676,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16811850666999817,
      "logits/rejected": -0.21972471475601196,
      "logps/chosen": -630.1519775390625,
      "logps/rejected": -509.1710510253906,
      "loss": 0.3491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2409416437149048,
      "rewards/margins": 1.663116693496704,
      "rewards/rejected": -2.9040582180023193,
      "step": 14807
    },
    {
      "epoch": 1.93772797906273,
      "grad_norm": 9.50487995147705,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0454319566488266,
      "logits/rejected": 0.060937583446502686,
      "logps/chosen": -401.067138671875,
      "logps/rejected": -524.95849609375,
      "loss": 0.4268,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12446920573711395,
      "rewards/margins": 2.6814346313476562,
      "rewards/rejected": -2.805903911590576,
      "step": 14808
    },
    {
      "epoch": 1.9378588370000818,
      "grad_norm": 9.809236526489258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27425286173820496,
      "logits/rejected": -0.19094130396842957,
      "logps/chosen": -368.3302307128906,
      "logps/rejected": -381.3875427246094,
      "loss": 0.608,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5409324765205383,
      "rewards/margins": 1.4606096744537354,
      "rewards/rejected": -2.001542091369629,
      "step": 14809
    },
    {
      "epoch": 1.9379896949374336,
      "grad_norm": 9.715021133422852,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1883068084716797,
      "logits/rejected": -0.1655384600162506,
      "logps/chosen": -449.87371826171875,
      "logps/rejected": -472.3919677734375,
      "loss": 0.4741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.397676706314087,
      "rewards/margins": 1.2187156677246094,
      "rewards/rejected": -2.6163923740386963,
      "step": 14810
    },
    {
      "epoch": 1.9381205528747854,
      "grad_norm": 4.793546199798584,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3646795153617859,
      "logits/rejected": -0.6038407683372498,
      "logps/chosen": -570.4327392578125,
      "logps/rejected": -495.92938232421875,
      "loss": 0.227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16808584332466125,
      "rewards/margins": 2.3543694019317627,
      "rewards/rejected": -2.5224552154541016,
      "step": 14811
    },
    {
      "epoch": 1.938251410812137,
      "grad_norm": 10.140091896057129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19160164892673492,
      "logits/rejected": -0.22535237669944763,
      "logps/chosen": -491.1175231933594,
      "logps/rejected": -420.1717834472656,
      "loss": 0.4721,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3935739994049072,
      "rewards/margins": 1.6198763847351074,
      "rewards/rejected": -3.0134503841400146,
      "step": 14812
    },
    {
      "epoch": 1.9383822687494887,
      "grad_norm": 11.908557891845703,
      "learning_rate": 5e-06,
      "logits/chosen": -0.047175969928503036,
      "logits/rejected": -0.28058508038520813,
      "logps/chosen": -550.8225708007812,
      "logps/rejected": -469.89544677734375,
      "loss": 0.7861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4511738121509552,
      "rewards/margins": 0.7816852331161499,
      "rewards/rejected": -1.2328591346740723,
      "step": 14813
    },
    {
      "epoch": 1.9385131266868405,
      "grad_norm": 7.1080145835876465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5894055366516113,
      "logits/rejected": -0.6585742831230164,
      "logps/chosen": -373.8045959472656,
      "logps/rejected": -364.24169921875,
      "loss": 0.2438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6859890222549438,
      "rewards/margins": 1.9490143060684204,
      "rewards/rejected": -3.6350035667419434,
      "step": 14814
    },
    {
      "epoch": 1.9386439846241923,
      "grad_norm": 4.357081413269043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0074562132358551025,
      "logits/rejected": -0.1272178292274475,
      "logps/chosen": -447.9436340332031,
      "logps/rejected": -369.56158447265625,
      "loss": 0.1999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0989245176315308,
      "rewards/margins": 2.1635022163391113,
      "rewards/rejected": -3.2624266147613525,
      "step": 14815
    },
    {
      "epoch": 1.938774842561544,
      "grad_norm": 9.299798965454102,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09179256856441498,
      "logits/rejected": 0.09376958012580872,
      "logps/chosen": -440.71478271484375,
      "logps/rejected": -522.37890625,
      "loss": 0.563,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7932466864585876,
      "rewards/margins": 1.3903216123580933,
      "rewards/rejected": -2.183568239212036,
      "step": 14816
    },
    {
      "epoch": 1.9389057004988959,
      "grad_norm": 7.162217617034912,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3126715421676636,
      "logits/rejected": -0.37032994627952576,
      "logps/chosen": -443.8163146972656,
      "logps/rejected": -354.16558837890625,
      "loss": 0.3254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.33291053771972656,
      "rewards/margins": 1.4731377363204956,
      "rewards/rejected": -1.8060482740402222,
      "step": 14817
    },
    {
      "epoch": 1.9390365584362477,
      "grad_norm": 5.157691955566406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17530782520771027,
      "logits/rejected": -0.12183539569377899,
      "logps/chosen": -538.9174194335938,
      "logps/rejected": -540.896728515625,
      "loss": 0.2639,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.486788034439087,
      "rewards/margins": 2.290933132171631,
      "rewards/rejected": -3.777721405029297,
      "step": 14818
    },
    {
      "epoch": 1.9391674163735995,
      "grad_norm": 8.182753562927246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.46207770705223083,
      "logits/rejected": -0.3870636522769928,
      "logps/chosen": -384.4693603515625,
      "logps/rejected": -418.98321533203125,
      "loss": 0.4744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.229697585105896,
      "rewards/margins": 0.798099160194397,
      "rewards/rejected": -2.027796745300293,
      "step": 14819
    },
    {
      "epoch": 1.9392982743109513,
      "grad_norm": 4.597191333770752,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15879659354686737,
      "logits/rejected": -0.2778467833995819,
      "logps/chosen": -470.82952880859375,
      "logps/rejected": -467.31231689453125,
      "loss": 0.2033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47994881868362427,
      "rewards/margins": 1.6987833976745605,
      "rewards/rejected": -2.17873215675354,
      "step": 14820
    },
    {
      "epoch": 1.939429132248303,
      "grad_norm": 11.951495170593262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24012774229049683,
      "logits/rejected": -0.18578144907951355,
      "logps/chosen": -458.55731201171875,
      "logps/rejected": -605.0692749023438,
      "loss": 0.5447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6769050359725952,
      "rewards/margins": 0.840854287147522,
      "rewards/rejected": -2.517759323120117,
      "step": 14821
    },
    {
      "epoch": 1.9395599901856548,
      "grad_norm": 12.180516242980957,
      "learning_rate": 5e-06,
      "logits/chosen": 0.04627927392721176,
      "logits/rejected": -0.03906559571623802,
      "logps/chosen": -325.163330078125,
      "logps/rejected": -393.32489013671875,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9236477613449097,
      "rewards/margins": 0.7533156871795654,
      "rewards/rejected": -1.676963448524475,
      "step": 14822
    },
    {
      "epoch": 1.9396908481230064,
      "grad_norm": 6.520656108856201,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38171422481536865,
      "logits/rejected": -0.37136292457580566,
      "logps/chosen": -471.7164611816406,
      "logps/rejected": -343.9721374511719,
      "loss": 0.4014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.659389078617096,
      "rewards/margins": 0.9003904461860657,
      "rewards/rejected": -1.5597796440124512,
      "step": 14823
    },
    {
      "epoch": 1.9398217060603582,
      "grad_norm": 8.290669441223145,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10887054353952408,
      "logits/rejected": 0.02060447633266449,
      "logps/chosen": -518.9667358398438,
      "logps/rejected": -610.2628784179688,
      "loss": 0.2702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4388045072555542,
      "rewards/margins": 1.5156152248382568,
      "rewards/rejected": -2.9544196128845215,
      "step": 14824
    },
    {
      "epoch": 1.93995256399771,
      "grad_norm": 15.000969886779785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42116233706474304,
      "logits/rejected": -0.44641169905662537,
      "logps/chosen": -461.07806396484375,
      "logps/rejected": -542.103515625,
      "loss": 0.5607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2245597839355469,
      "rewards/margins": 1.1080102920532227,
      "rewards/rejected": -2.3325703144073486,
      "step": 14825
    },
    {
      "epoch": 1.9400834219350618,
      "grad_norm": 6.503255367279053,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4414396584033966,
      "logits/rejected": -0.5023989677429199,
      "logps/chosen": -326.662841796875,
      "logps/rejected": -392.6184387207031,
      "loss": 0.3695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7563722133636475,
      "rewards/margins": 1.2748968601226807,
      "rewards/rejected": -2.031269073486328,
      "step": 14826
    },
    {
      "epoch": 1.9402142798724133,
      "grad_norm": 6.501021862030029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21488814055919647,
      "logits/rejected": -0.3024895191192627,
      "logps/chosen": -468.564697265625,
      "logps/rejected": -429.3564758300781,
      "loss": 0.1762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9229848980903625,
      "rewards/margins": 2.2366397380828857,
      "rewards/rejected": -3.1596250534057617,
      "step": 14827
    },
    {
      "epoch": 1.9403451378097651,
      "grad_norm": 5.196409702301025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5205919146537781,
      "logits/rejected": -0.47387346625328064,
      "logps/chosen": -534.0819702148438,
      "logps/rejected": -416.0607604980469,
      "loss": 0.3022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1363015174865723,
      "rewards/margins": 1.7499483823776245,
      "rewards/rejected": -2.8862500190734863,
      "step": 14828
    },
    {
      "epoch": 1.940475995747117,
      "grad_norm": 10.340721130371094,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45454761385917664,
      "logits/rejected": -0.513510525226593,
      "logps/chosen": -328.6021728515625,
      "logps/rejected": -371.4141845703125,
      "loss": 0.3936,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0019255876541138,
      "rewards/margins": 1.8906556367874146,
      "rewards/rejected": -2.8925814628601074,
      "step": 14829
    },
    {
      "epoch": 1.9406068536844687,
      "grad_norm": 8.937376976013184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5655440092086792,
      "logits/rejected": -0.4361363351345062,
      "logps/chosen": -424.21630859375,
      "logps/rejected": -692.701171875,
      "loss": 0.4233,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.114070177078247,
      "rewards/margins": 1.4876139163970947,
      "rewards/rejected": -3.601684331893921,
      "step": 14830
    },
    {
      "epoch": 1.9407377116218205,
      "grad_norm": 13.752108573913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4593580365180969,
      "logits/rejected": -0.3269578814506531,
      "logps/chosen": -388.81646728515625,
      "logps/rejected": -300.7398681640625,
      "loss": 0.4361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6129318475723267,
      "rewards/margins": 1.5717597007751465,
      "rewards/rejected": -3.1846916675567627,
      "step": 14831
    },
    {
      "epoch": 1.9408685695591723,
      "grad_norm": 9.558518409729004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04718884453177452,
      "logits/rejected": 0.0563410259783268,
      "logps/chosen": -469.73199462890625,
      "logps/rejected": -544.62890625,
      "loss": 0.5021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0932741165161133,
      "rewards/margins": 0.9480795860290527,
      "rewards/rejected": -2.041353702545166,
      "step": 14832
    },
    {
      "epoch": 1.940999427496524,
      "grad_norm": 7.987039566040039,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24807783961296082,
      "logits/rejected": -0.09279739856719971,
      "logps/chosen": -409.80938720703125,
      "logps/rejected": -509.9766845703125,
      "loss": 0.4236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4657502174377441,
      "rewards/margins": 1.085942029953003,
      "rewards/rejected": -2.551692247390747,
      "step": 14833
    },
    {
      "epoch": 1.941130285433876,
      "grad_norm": 9.408177375793457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4958024024963379,
      "logits/rejected": -0.7243800759315491,
      "logps/chosen": -363.3106384277344,
      "logps/rejected": -369.3797302246094,
      "loss": 0.4013,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.934784173965454,
      "rewards/margins": 1.540207862854004,
      "rewards/rejected": -3.474992036819458,
      "step": 14834
    },
    {
      "epoch": 1.9412611433712277,
      "grad_norm": 7.245809555053711,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2998284101486206,
      "logits/rejected": -0.435483455657959,
      "logps/chosen": -497.8937683105469,
      "logps/rejected": -424.149169921875,
      "loss": 0.3375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4166979193687439,
      "rewards/margins": 1.740465760231018,
      "rewards/rejected": -2.1571638584136963,
      "step": 14835
    },
    {
      "epoch": 1.9413920013085795,
      "grad_norm": 4.253312110900879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2819252908229828,
      "logits/rejected": -0.5215181708335876,
      "logps/chosen": -414.74945068359375,
      "logps/rejected": -521.5933837890625,
      "loss": 0.1807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9502505660057068,
      "rewards/margins": 2.111067771911621,
      "rewards/rejected": -3.0613183975219727,
      "step": 14836
    },
    {
      "epoch": 1.9415228592459313,
      "grad_norm": 11.967823028564453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3017333745956421,
      "logits/rejected": -0.39351677894592285,
      "logps/chosen": -337.5862121582031,
      "logps/rejected": -319.743408203125,
      "loss": 1.0758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.323372483253479,
      "rewards/margins": -0.05156897008419037,
      "rewards/rejected": -1.2718034982681274,
      "step": 14837
    },
    {
      "epoch": 1.941653717183283,
      "grad_norm": 6.032360076904297,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4632350504398346,
      "logits/rejected": -0.42421168088912964,
      "logps/chosen": -529.8024291992188,
      "logps/rejected": -499.03485107421875,
      "loss": 0.2953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8391826152801514,
      "rewards/margins": 1.5996081829071045,
      "rewards/rejected": -3.438790798187256,
      "step": 14838
    },
    {
      "epoch": 1.9417845751206346,
      "grad_norm": 6.532111167907715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5782860517501831,
      "logits/rejected": -0.7342824935913086,
      "logps/chosen": -435.1194152832031,
      "logps/rejected": -497.8302917480469,
      "loss": 0.2711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7137978076934814,
      "rewards/margins": 1.7913984060287476,
      "rewards/rejected": -2.5051960945129395,
      "step": 14839
    },
    {
      "epoch": 1.9419154330579864,
      "grad_norm": 10.79383659362793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11287915706634521,
      "logits/rejected": -0.36498531699180603,
      "logps/chosen": -604.9779052734375,
      "logps/rejected": -602.990478515625,
      "loss": 0.527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9010826945304871,
      "rewards/margins": 1.4804260730743408,
      "rewards/rejected": -2.3815088272094727,
      "step": 14840
    },
    {
      "epoch": 1.9420462909953382,
      "grad_norm": 14.285758018493652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40466809272766113,
      "logits/rejected": -0.6765051484107971,
      "logps/chosen": -469.381591796875,
      "logps/rejected": -421.6669006347656,
      "loss": 0.4478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.082411289215088,
      "rewards/margins": 1.7859104871749878,
      "rewards/rejected": -3.868321657180786,
      "step": 14841
    },
    {
      "epoch": 1.94217714893269,
      "grad_norm": 8.19948959350586,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08887012302875519,
      "logits/rejected": 0.016265161335468292,
      "logps/chosen": -553.0797119140625,
      "logps/rejected": -421.2467041015625,
      "loss": 0.3219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3679187297821045,
      "rewards/margins": 1.87160062789917,
      "rewards/rejected": -2.2395193576812744,
      "step": 14842
    },
    {
      "epoch": 1.9423080068700416,
      "grad_norm": 8.903319358825684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11588680744171143,
      "logits/rejected": -0.19685117900371552,
      "logps/chosen": -491.4920654296875,
      "logps/rejected": -417.06219482421875,
      "loss": 0.3941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1286280155181885,
      "rewards/margins": 1.0967798233032227,
      "rewards/rejected": -2.225407838821411,
      "step": 14843
    },
    {
      "epoch": 1.9424388648073934,
      "grad_norm": 9.981755256652832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.02517441287636757,
      "logits/rejected": 0.0024309977889060974,
      "logps/chosen": -453.8625793457031,
      "logps/rejected": -519.6552734375,
      "loss": 0.5162,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7479130029678345,
      "rewards/margins": 0.902038037776947,
      "rewards/rejected": -1.6499511003494263,
      "step": 14844
    },
    {
      "epoch": 1.9425697227447452,
      "grad_norm": 12.911152839660645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4296766519546509,
      "logits/rejected": -0.36693650484085083,
      "logps/chosen": -563.3912353515625,
      "logps/rejected": -343.5495300292969,
      "loss": 0.558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.131645917892456,
      "rewards/margins": 1.047545075416565,
      "rewards/rejected": -3.1791911125183105,
      "step": 14845
    },
    {
      "epoch": 1.942700580682097,
      "grad_norm": 15.756065368652344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19133245944976807,
      "logits/rejected": -0.4059729278087616,
      "logps/chosen": -580.1640625,
      "logps/rejected": -517.2695922851562,
      "loss": 0.7551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9353795051574707,
      "rewards/margins": 0.6197893619537354,
      "rewards/rejected": -1.555168867111206,
      "step": 14846
    },
    {
      "epoch": 1.9428314386194487,
      "grad_norm": 13.031120300292969,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18081019818782806,
      "logits/rejected": -0.3250930607318878,
      "logps/chosen": -532.21630859375,
      "logps/rejected": -423.8083190917969,
      "loss": 0.8015,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6097217798233032,
      "rewards/margins": 1.052893042564392,
      "rewards/rejected": -2.6626148223876953,
      "step": 14847
    },
    {
      "epoch": 1.9429622965568005,
      "grad_norm": 5.949339389801025,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26901179552078247,
      "logits/rejected": -0.40205246210098267,
      "logps/chosen": -476.7096252441406,
      "logps/rejected": -323.3022155761719,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0004866123199463,
      "rewards/margins": 2.143131732940674,
      "rewards/rejected": -3.143618583679199,
      "step": 14848
    },
    {
      "epoch": 1.9430931544941523,
      "grad_norm": 7.628336429595947,
      "learning_rate": 5e-06,
      "logits/chosen": 0.030699245631694794,
      "logits/rejected": -0.5010116100311279,
      "logps/chosen": -721.5303955078125,
      "logps/rejected": -543.441650390625,
      "loss": 0.2759,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.967897355556488,
      "rewards/margins": 1.5461665391921997,
      "rewards/rejected": -2.514064073562622,
      "step": 14849
    },
    {
      "epoch": 1.943224012431504,
      "grad_norm": 7.519665241241455,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2791478931903839,
      "logits/rejected": -0.34728574752807617,
      "logps/chosen": -408.5748291015625,
      "logps/rejected": -361.17047119140625,
      "loss": 0.3746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1472225189208984,
      "rewards/margins": 1.7968981266021729,
      "rewards/rejected": -2.9441208839416504,
      "step": 14850
    },
    {
      "epoch": 1.943354870368856,
      "grad_norm": 7.042099952697754,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05554581061005592,
      "logits/rejected": -0.2663871943950653,
      "logps/chosen": -538.6968383789062,
      "logps/rejected": -387.646484375,
      "loss": 0.4472,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5978229641914368,
      "rewards/margins": 1.1475900411605835,
      "rewards/rejected": -1.7454129457473755,
      "step": 14851
    },
    {
      "epoch": 1.9434857283062077,
      "grad_norm": 7.3898725509643555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13798396289348602,
      "logits/rejected": -0.06201154738664627,
      "logps/chosen": -426.20953369140625,
      "logps/rejected": -505.50115966796875,
      "loss": 0.3117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9473075866699219,
      "rewards/margins": 1.6857895851135254,
      "rewards/rejected": -2.633096933364868,
      "step": 14852
    },
    {
      "epoch": 1.9436165862435595,
      "grad_norm": 9.971047401428223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.345873087644577,
      "logits/rejected": -0.10385101288557053,
      "logps/chosen": -505.0246276855469,
      "logps/rejected": -531.8186645507812,
      "loss": 0.4264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6832523941993713,
      "rewards/margins": 1.484640121459961,
      "rewards/rejected": -2.1678924560546875,
      "step": 14853
    },
    {
      "epoch": 1.943747444180911,
      "grad_norm": 8.159387588500977,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12895597517490387,
      "logits/rejected": -0.29802730679512024,
      "logps/chosen": -415.4390869140625,
      "logps/rejected": -394.08514404296875,
      "loss": 0.2363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27871087193489075,
      "rewards/margins": 2.406304121017456,
      "rewards/rejected": -2.6850152015686035,
      "step": 14854
    },
    {
      "epoch": 1.9438783021182628,
      "grad_norm": 5.097631454467773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1852584034204483,
      "logits/rejected": -0.1713181734085083,
      "logps/chosen": -533.9904174804688,
      "logps/rejected": -491.65216064453125,
      "loss": 0.2428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6571356058120728,
      "rewards/margins": 1.8714256286621094,
      "rewards/rejected": -2.5285611152648926,
      "step": 14855
    },
    {
      "epoch": 1.9440091600556146,
      "grad_norm": 9.773946762084961,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07407934218645096,
      "logits/rejected": 0.10991416871547699,
      "logps/chosen": -474.8139953613281,
      "logps/rejected": -554.736572265625,
      "loss": 0.5167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2011572122573853,
      "rewards/margins": 1.8822797536849976,
      "rewards/rejected": -3.083437204360962,
      "step": 14856
    },
    {
      "epoch": 1.9441400179929664,
      "grad_norm": 13.60827922821045,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30925172567367554,
      "logits/rejected": -0.42831602692604065,
      "logps/chosen": -541.609619140625,
      "logps/rejected": -468.41107177734375,
      "loss": 0.7412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9865063428878784,
      "rewards/margins": 0.9085404872894287,
      "rewards/rejected": -2.8950467109680176,
      "step": 14857
    },
    {
      "epoch": 1.944270875930318,
      "grad_norm": 6.620357513427734,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2807532548904419,
      "logits/rejected": -0.7096335887908936,
      "logps/chosen": -330.3233642578125,
      "logps/rejected": -372.1461486816406,
      "loss": 0.2036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0341734886169434,
      "rewards/margins": 2.540409564971924,
      "rewards/rejected": -3.5745832920074463,
      "step": 14858
    },
    {
      "epoch": 1.9444017338676698,
      "grad_norm": 10.745375633239746,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13447287678718567,
      "logits/rejected": -0.22393743693828583,
      "logps/chosen": -476.55230712890625,
      "logps/rejected": -474.2509765625,
      "loss": 0.4707,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8831353187561035,
      "rewards/margins": 0.956571102142334,
      "rewards/rejected": -1.8397064208984375,
      "step": 14859
    },
    {
      "epoch": 1.9445325918050216,
      "grad_norm": 16.497312545776367,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37097036838531494,
      "logits/rejected": -0.44402098655700684,
      "logps/chosen": -490.548583984375,
      "logps/rejected": -506.60186767578125,
      "loss": 0.758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.676798939704895,
      "rewards/margins": 1.7443982362747192,
      "rewards/rejected": -3.4211976528167725,
      "step": 14860
    },
    {
      "epoch": 1.9446634497423734,
      "grad_norm": 10.124716758728027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33512550592422485,
      "logits/rejected": -0.4256289005279541,
      "logps/chosen": -457.44757080078125,
      "logps/rejected": -498.4727783203125,
      "loss": 0.598,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5262184143066406,
      "rewards/margins": 0.6477905511856079,
      "rewards/rejected": -1.1740089654922485,
      "step": 14861
    },
    {
      "epoch": 1.9447943076797252,
      "grad_norm": 10.335670471191406,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40035128593444824,
      "logits/rejected": -0.20984713733196259,
      "logps/chosen": -506.64617919921875,
      "logps/rejected": -507.6274719238281,
      "loss": 0.4944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3857053518295288,
      "rewards/margins": 1.6452090740203857,
      "rewards/rejected": -3.030914306640625,
      "step": 14862
    },
    {
      "epoch": 1.944925165617077,
      "grad_norm": 6.547967910766602,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2870665192604065,
      "logits/rejected": -0.3720577657222748,
      "logps/chosen": -505.46173095703125,
      "logps/rejected": -587.9168090820312,
      "loss": 0.2753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8606696128845215,
      "rewards/margins": 1.9082008600234985,
      "rewards/rejected": -2.7688705921173096,
      "step": 14863
    },
    {
      "epoch": 1.9450560235544287,
      "grad_norm": 13.823638916015625,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19288863241672516,
      "logits/rejected": -0.36332598328590393,
      "logps/chosen": -349.2148132324219,
      "logps/rejected": -363.8249206542969,
      "loss": 0.7554,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1648969650268555,
      "rewards/margins": 0.1660483479499817,
      "rewards/rejected": -1.3309452533721924,
      "step": 14864
    },
    {
      "epoch": 1.9451868814917805,
      "grad_norm": 7.621646404266357,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07262483239173889,
      "logits/rejected": -0.2659299969673157,
      "logps/chosen": -622.1953735351562,
      "logps/rejected": -426.5266418457031,
      "loss": 0.2807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6004344820976257,
      "rewards/margins": 2.1934425830841064,
      "rewards/rejected": -2.793876886367798,
      "step": 14865
    },
    {
      "epoch": 1.9453177394291323,
      "grad_norm": 11.077458381652832,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16540110111236572,
      "logits/rejected": -0.004385910928249359,
      "logps/chosen": -441.7908935546875,
      "logps/rejected": -406.228759765625,
      "loss": 0.7068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.686257243156433,
      "rewards/margins": 0.6400707364082336,
      "rewards/rejected": -2.3263278007507324,
      "step": 14866
    },
    {
      "epoch": 1.9454485973664841,
      "grad_norm": 11.631501197814941,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3320949077606201,
      "logits/rejected": -0.5863617658615112,
      "logps/chosen": -530.8588256835938,
      "logps/rejected": -362.7724304199219,
      "loss": 0.5558,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3443281650543213,
      "rewards/margins": 1.457313060760498,
      "rewards/rejected": -2.8016412258148193,
      "step": 14867
    },
    {
      "epoch": 1.945579455303836,
      "grad_norm": 11.623451232910156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3559754490852356,
      "logits/rejected": -0.3337244391441345,
      "logps/chosen": -432.98822021484375,
      "logps/rejected": -608.4970092773438,
      "loss": 0.5528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1907849311828613,
      "rewards/margins": 1.257196068763733,
      "rewards/rejected": -2.4479808807373047,
      "step": 14868
    },
    {
      "epoch": 1.9457103132411875,
      "grad_norm": 9.897717475891113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2649537920951843,
      "logits/rejected": -0.4838075041770935,
      "logps/chosen": -592.275146484375,
      "logps/rejected": -578.025390625,
      "loss": 0.4723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5651326179504395,
      "rewards/margins": 1.0482995510101318,
      "rewards/rejected": -3.6134321689605713,
      "step": 14869
    },
    {
      "epoch": 1.9458411711785393,
      "grad_norm": 16.319408416748047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22019940614700317,
      "logits/rejected": -0.16351373493671417,
      "logps/chosen": -449.2425537109375,
      "logps/rejected": -495.81829833984375,
      "loss": 1.2286,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.131955862045288,
      "rewards/margins": -0.008204072713851929,
      "rewards/rejected": -2.123751640319824,
      "step": 14870
    },
    {
      "epoch": 1.945972029115891,
      "grad_norm": 8.96288776397705,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23619191348552704,
      "logits/rejected": -0.31653642654418945,
      "logps/chosen": -302.35443115234375,
      "logps/rejected": -316.69952392578125,
      "loss": 0.5965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9902211427688599,
      "rewards/margins": 0.6723035573959351,
      "rewards/rejected": -1.662524700164795,
      "step": 14871
    },
    {
      "epoch": 1.9461028870532429,
      "grad_norm": 15.674112319946289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.00792022980749607,
      "logits/rejected": -0.27838096022605896,
      "logps/chosen": -452.0494384765625,
      "logps/rejected": -396.08648681640625,
      "loss": 0.7717,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8666188716888428,
      "rewards/margins": 0.9401523470878601,
      "rewards/rejected": -2.8067715167999268,
      "step": 14872
    },
    {
      "epoch": 1.9462337449905944,
      "grad_norm": 5.7591962814331055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6467821598052979,
      "logits/rejected": -0.5346800684928894,
      "logps/chosen": -356.2344970703125,
      "logps/rejected": -384.2876281738281,
      "loss": 0.3386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5312421321868896,
      "rewards/margins": 1.2665061950683594,
      "rewards/rejected": -2.797748565673828,
      "step": 14873
    },
    {
      "epoch": 1.9463646029279462,
      "grad_norm": 9.723310470581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34399500489234924,
      "logits/rejected": -0.4583475589752197,
      "logps/chosen": -455.80853271484375,
      "logps/rejected": -416.6866760253906,
      "loss": 0.4513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3956737518310547,
      "rewards/margins": 1.279491662979126,
      "rewards/rejected": -2.6751654148101807,
      "step": 14874
    },
    {
      "epoch": 1.946495460865298,
      "grad_norm": 12.316210746765137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2902640700340271,
      "logits/rejected": -0.44536489248275757,
      "logps/chosen": -500.9385986328125,
      "logps/rejected": -437.65667724609375,
      "loss": 0.5022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.517398715019226,
      "rewards/margins": 0.7377461791038513,
      "rewards/rejected": -2.2551448345184326,
      "step": 14875
    },
    {
      "epoch": 1.9466263188026498,
      "grad_norm": 7.38297700881958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30373433232307434,
      "logits/rejected": -0.4171101748943329,
      "logps/chosen": -506.31964111328125,
      "logps/rejected": -448.73272705078125,
      "loss": 0.2743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3048352897167206,
      "rewards/margins": 2.3648343086242676,
      "rewards/rejected": -2.6696693897247314,
      "step": 14876
    },
    {
      "epoch": 1.9467571767400016,
      "grad_norm": 9.580022811889648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.631247341632843,
      "logits/rejected": -0.7298694849014282,
      "logps/chosen": -450.8493957519531,
      "logps/rejected": -361.7356262207031,
      "loss": 0.3117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.946195125579834,
      "rewards/margins": 2.4098551273345947,
      "rewards/rejected": -4.356050491333008,
      "step": 14877
    },
    {
      "epoch": 1.9468880346773534,
      "grad_norm": 9.985567092895508,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1198749840259552,
      "logits/rejected": -0.07139887660741806,
      "logps/chosen": -516.2491455078125,
      "logps/rejected": -610.744873046875,
      "loss": 0.4688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1834874153137207,
      "rewards/margins": 1.1885414123535156,
      "rewards/rejected": -2.3720285892486572,
      "step": 14878
    },
    {
      "epoch": 1.9470188926147052,
      "grad_norm": 6.254091262817383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2999372184276581,
      "logits/rejected": -0.351590633392334,
      "logps/chosen": -479.95916748046875,
      "logps/rejected": -392.6163635253906,
      "loss": 0.3811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8949601650238037,
      "rewards/margins": 1.1356005668640137,
      "rewards/rejected": -2.0305607318878174,
      "step": 14879
    },
    {
      "epoch": 1.947149750552057,
      "grad_norm": 7.600088596343994,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31537696719169617,
      "logits/rejected": -0.4757842421531677,
      "logps/chosen": -442.97039794921875,
      "logps/rejected": -351.7434997558594,
      "loss": 0.4054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4285783767700195,
      "rewards/margins": 0.8744497299194336,
      "rewards/rejected": -2.3030283451080322,
      "step": 14880
    },
    {
      "epoch": 1.9472806084894088,
      "grad_norm": 6.995810031890869,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5561337471008301,
      "logits/rejected": -0.45886334776878357,
      "logps/chosen": -400.7982177734375,
      "logps/rejected": -438.7559814453125,
      "loss": 0.4107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7429600954055786,
      "rewards/margins": 1.7357985973358154,
      "rewards/rejected": -2.4787588119506836,
      "step": 14881
    },
    {
      "epoch": 1.9474114664267606,
      "grad_norm": 5.2167649269104,
      "learning_rate": 5e-06,
      "logits/chosen": -0.30518943071365356,
      "logits/rejected": -0.1306585967540741,
      "logps/chosen": -380.09930419921875,
      "logps/rejected": -418.381591796875,
      "loss": 0.2336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8852729797363281,
      "rewards/margins": 1.6790496110916138,
      "rewards/rejected": -2.5643224716186523,
      "step": 14882
    },
    {
      "epoch": 1.9475423243641123,
      "grad_norm": 4.3432536125183105,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23028433322906494,
      "logits/rejected": -0.4590193033218384,
      "logps/chosen": -299.4672546386719,
      "logps/rejected": -417.349609375,
      "loss": 0.2,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3992211818695068,
      "rewards/margins": 1.6999847888946533,
      "rewards/rejected": -3.09920597076416,
      "step": 14883
    },
    {
      "epoch": 1.9476731823014641,
      "grad_norm": 4.774012565612793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44027817249298096,
      "logits/rejected": -0.3486589789390564,
      "logps/chosen": -343.0692138671875,
      "logps/rejected": -393.18450927734375,
      "loss": 0.1901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5675377249717712,
      "rewards/margins": 2.006397247314453,
      "rewards/rejected": -2.5739352703094482,
      "step": 14884
    },
    {
      "epoch": 1.9478040402388157,
      "grad_norm": 6.364482879638672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.657654881477356,
      "logits/rejected": -0.4368407726287842,
      "logps/chosen": -509.12469482421875,
      "logps/rejected": -439.79791259765625,
      "loss": 0.3557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0048483610153198,
      "rewards/margins": 1.6691946983337402,
      "rewards/rejected": -2.6740431785583496,
      "step": 14885
    },
    {
      "epoch": 1.9479348981761675,
      "grad_norm": 8.334173202514648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7222525477409363,
      "logits/rejected": -0.43215522170066833,
      "logps/chosen": -515.1372680664062,
      "logps/rejected": -523.3256225585938,
      "loss": 0.384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8513271808624268,
      "rewards/margins": 1.7927067279815674,
      "rewards/rejected": -2.644033908843994,
      "step": 14886
    },
    {
      "epoch": 1.9480657561135193,
      "grad_norm": 10.381866455078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4517979919910431,
      "logits/rejected": -0.40366482734680176,
      "logps/chosen": -619.3736572265625,
      "logps/rejected": -449.1958312988281,
      "loss": 0.4772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1466670036315918,
      "rewards/margins": 0.7634690999984741,
      "rewards/rejected": -1.9101362228393555,
      "step": 14887
    },
    {
      "epoch": 1.948196614050871,
      "grad_norm": 11.967451095581055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4075063169002533,
      "logits/rejected": -0.41250675916671753,
      "logps/chosen": -308.2342529296875,
      "logps/rejected": -330.1070556640625,
      "loss": 0.4543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.597346305847168,
      "rewards/margins": 2.3718695640563965,
      "rewards/rejected": -2.9692158699035645,
      "step": 14888
    },
    {
      "epoch": 1.9483274719882226,
      "grad_norm": 10.176580429077148,
      "learning_rate": 5e-06,
      "logits/chosen": -0.054465726017951965,
      "logits/rejected": 0.04588979482650757,
      "logps/chosen": -306.4367370605469,
      "logps/rejected": -362.423828125,
      "loss": 0.6466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6702306270599365,
      "rewards/margins": 1.089921474456787,
      "rewards/rejected": -2.7601521015167236,
      "step": 14889
    },
    {
      "epoch": 1.9484583299255744,
      "grad_norm": 9.847614288330078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5731701850891113,
      "logits/rejected": -0.4729370176792145,
      "logps/chosen": -505.3493957519531,
      "logps/rejected": -593.427978515625,
      "loss": 0.3347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6595866680145264,
      "rewards/margins": 1.5021175146102905,
      "rewards/rejected": -2.1617043018341064,
      "step": 14890
    },
    {
      "epoch": 1.9485891878629262,
      "grad_norm": 3.906937837600708,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21127763390541077,
      "logits/rejected": -0.196548193693161,
      "logps/chosen": -459.46502685546875,
      "logps/rejected": -496.4947204589844,
      "loss": 0.168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9269399642944336,
      "rewards/margins": 2.237532377243042,
      "rewards/rejected": -3.1644723415374756,
      "step": 14891
    },
    {
      "epoch": 1.948720045800278,
      "grad_norm": 4.373399257659912,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2771276831626892,
      "logits/rejected": -0.4386288523674011,
      "logps/chosen": -410.1474609375,
      "logps/rejected": -444.41729736328125,
      "loss": 0.1736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42114412784576416,
      "rewards/margins": 2.7912068367004395,
      "rewards/rejected": -3.212351083755493,
      "step": 14892
    },
    {
      "epoch": 1.9488509037376298,
      "grad_norm": 17.645538330078125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2761619985103607,
      "logits/rejected": -0.5096999406814575,
      "logps/chosen": -608.4645385742188,
      "logps/rejected": -444.7026672363281,
      "loss": 1.2361,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7420510053634644,
      "rewards/margins": -0.5113510489463806,
      "rewards/rejected": -1.230699896812439,
      "step": 14893
    },
    {
      "epoch": 1.9489817616749816,
      "grad_norm": 8.81348705291748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3384318947792053,
      "logits/rejected": -0.5007939338684082,
      "logps/chosen": -523.916259765625,
      "logps/rejected": -348.0794982910156,
      "loss": 0.2582,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6033545732498169,
      "rewards/margins": 1.8980581760406494,
      "rewards/rejected": -2.501412868499756,
      "step": 14894
    },
    {
      "epoch": 1.9491126196123334,
      "grad_norm": 8.620148658752441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.47050178050994873,
      "logits/rejected": -0.30762675404548645,
      "logps/chosen": -493.22943115234375,
      "logps/rejected": -542.2802734375,
      "loss": 0.2388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.706145703792572,
      "rewards/margins": 2.5846452713012695,
      "rewards/rejected": -3.2907910346984863,
      "step": 14895
    },
    {
      "epoch": 1.9492434775496852,
      "grad_norm": 7.192814826965332,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22554215788841248,
      "logits/rejected": -0.5227062106132507,
      "logps/chosen": -371.486328125,
      "logps/rejected": -392.27960205078125,
      "loss": 0.3636,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.769155740737915,
      "rewards/margins": 1.3730285167694092,
      "rewards/rejected": -2.142184257507324,
      "step": 14896
    },
    {
      "epoch": 1.949374335487037,
      "grad_norm": 13.066852569580078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.505034327507019,
      "logits/rejected": -0.33229687809944153,
      "logps/chosen": -412.30450439453125,
      "logps/rejected": -499.1702880859375,
      "loss": 0.7515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9602260589599609,
      "rewards/margins": 0.974292516708374,
      "rewards/rejected": -1.9345186948776245,
      "step": 14897
    },
    {
      "epoch": 1.9495051934243888,
      "grad_norm": 8.508824348449707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4040859341621399,
      "logits/rejected": -0.44400298595428467,
      "logps/chosen": -385.94964599609375,
      "logps/rejected": -568.4188232421875,
      "loss": 0.3571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6649271249771118,
      "rewards/margins": 1.5142321586608887,
      "rewards/rejected": -3.179159164428711,
      "step": 14898
    },
    {
      "epoch": 1.9496360513617406,
      "grad_norm": 11.619422912597656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21890482306480408,
      "logits/rejected": -0.3450953960418701,
      "logps/chosen": -618.0186767578125,
      "logps/rejected": -594.4523315429688,
      "loss": 0.8663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9577344655990601,
      "rewards/margins": 0.7666879296302795,
      "rewards/rejected": -1.7244224548339844,
      "step": 14899
    },
    {
      "epoch": 1.9497669092990921,
      "grad_norm": 13.460638046264648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21927975118160248,
      "logits/rejected": -0.29173076152801514,
      "logps/chosen": -508.2110595703125,
      "logps/rejected": -308.8404541015625,
      "loss": 0.5446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3219216763973236,
      "rewards/margins": 1.707788348197937,
      "rewards/rejected": -2.029710054397583,
      "step": 14900
    },
    {
      "epoch": 1.949897767236444,
      "grad_norm": 19.010723114013672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5524470806121826,
      "logits/rejected": -0.21379858255386353,
      "logps/chosen": -606.2272338867188,
      "logps/rejected": -431.05255126953125,
      "loss": 1.2242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9080026149749756,
      "rewards/margins": -0.0912211537361145,
      "rewards/rejected": -1.8167815208435059,
      "step": 14901
    },
    {
      "epoch": 1.9500286251737957,
      "grad_norm": 11.37627124786377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09392234683036804,
      "logits/rejected": -0.31246981024742126,
      "logps/chosen": -472.67669677734375,
      "logps/rejected": -403.0570373535156,
      "loss": 0.604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8439035415649414,
      "rewards/margins": 1.4821555614471436,
      "rewards/rejected": -2.326058864593506,
      "step": 14902
    },
    {
      "epoch": 1.9501594831111475,
      "grad_norm": 10.373772621154785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6296782493591309,
      "logits/rejected": -0.7143679261207581,
      "logps/chosen": -340.11187744140625,
      "logps/rejected": -591.1807250976562,
      "loss": 0.6104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0927855968475342,
      "rewards/margins": 1.5912824869155884,
      "rewards/rejected": -2.684068202972412,
      "step": 14903
    },
    {
      "epoch": 1.950290341048499,
      "grad_norm": 7.9955668449401855,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3568567633628845,
      "logits/rejected": -0.34695860743522644,
      "logps/chosen": -443.99249267578125,
      "logps/rejected": -640.09423828125,
      "loss": 0.3993,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3546743392944336,
      "rewards/margins": 1.64626145362854,
      "rewards/rejected": -3.0009357929229736,
      "step": 14904
    },
    {
      "epoch": 1.9504211989858509,
      "grad_norm": 9.364543914794922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.052547529339790344,
      "logits/rejected": -0.15090495347976685,
      "logps/chosen": -495.92755126953125,
      "logps/rejected": -364.2489318847656,
      "loss": 0.4013,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.089233636856079,
      "rewards/margins": 1.5228744745254517,
      "rewards/rejected": -2.612107992172241,
      "step": 14905
    },
    {
      "epoch": 1.9505520569232027,
      "grad_norm": 7.371047496795654,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15668177604675293,
      "logits/rejected": -0.3104963004589081,
      "logps/chosen": -617.8004150390625,
      "logps/rejected": -440.59307861328125,
      "loss": 0.2316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45531120896339417,
      "rewards/margins": 2.1281142234802246,
      "rewards/rejected": -2.583425521850586,
      "step": 14906
    },
    {
      "epoch": 1.9506829148605545,
      "grad_norm": 15.797006607055664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16118475794792175,
      "logits/rejected": -0.32983487844467163,
      "logps/chosen": -415.14013671875,
      "logps/rejected": -323.5675354003906,
      "loss": 0.8787,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1572500467300415,
      "rewards/margins": 0.34456098079681396,
      "rewards/rejected": -1.5018110275268555,
      "step": 14907
    },
    {
      "epoch": 1.9508137727979062,
      "grad_norm": 10.429085731506348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33729690313339233,
      "logits/rejected": -0.32824647426605225,
      "logps/chosen": -413.01312255859375,
      "logps/rejected": -421.369140625,
      "loss": 0.719,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7040801644325256,
      "rewards/margins": 0.6401876211166382,
      "rewards/rejected": -1.3442678451538086,
      "step": 14908
    },
    {
      "epoch": 1.950944630735258,
      "grad_norm": 2.4018023014068604,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2535364031791687,
      "logits/rejected": -0.4980645775794983,
      "logps/chosen": -523.3838500976562,
      "logps/rejected": -444.3806457519531,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1856735497713089,
      "rewards/margins": 2.6270556449890137,
      "rewards/rejected": -2.8127293586730957,
      "step": 14909
    },
    {
      "epoch": 1.9510754886726098,
      "grad_norm": 11.351171493530273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07536572217941284,
      "logits/rejected": -0.18840292096138,
      "logps/chosen": -502.2403869628906,
      "logps/rejected": -597.4302978515625,
      "loss": 0.3485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35201799869537354,
      "rewards/margins": 2.0951693058013916,
      "rewards/rejected": -1.7431514263153076,
      "step": 14910
    },
    {
      "epoch": 1.9512063466099616,
      "grad_norm": 12.035938262939453,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17216423153877258,
      "logits/rejected": -0.12760674953460693,
      "logps/chosen": -465.10400390625,
      "logps/rejected": -449.2154846191406,
      "loss": 0.6572,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8834614753723145,
      "rewards/margins": 1.0916115045547485,
      "rewards/rejected": -1.9750728607177734,
      "step": 14911
    },
    {
      "epoch": 1.9513372045473134,
      "grad_norm": 9.045050621032715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17391960322856903,
      "logits/rejected": -0.23489443957805634,
      "logps/chosen": -473.1649475097656,
      "logps/rejected": -372.5880126953125,
      "loss": 0.4858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.439461588859558,
      "rewards/margins": 0.9103704690933228,
      "rewards/rejected": -2.3498318195343018,
      "step": 14912
    },
    {
      "epoch": 1.9514680624846652,
      "grad_norm": 6.431005477905273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.488481730222702,
      "logits/rejected": -0.3521156311035156,
      "logps/chosen": -351.8599853515625,
      "logps/rejected": -443.4930419921875,
      "loss": 0.3371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6563335657119751,
      "rewards/margins": 1.6780712604522705,
      "rewards/rejected": -2.334404945373535,
      "step": 14913
    },
    {
      "epoch": 1.951598920422017,
      "grad_norm": 9.421015739440918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0734449177980423,
      "logits/rejected": -0.006044261157512665,
      "logps/chosen": -552.8994750976562,
      "logps/rejected": -437.12103271484375,
      "loss": 0.3901,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15456180274486542,
      "rewards/margins": 1.1231586933135986,
      "rewards/rejected": -1.2777204513549805,
      "step": 14914
    },
    {
      "epoch": 1.9517297783593686,
      "grad_norm": 7.782721042633057,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5062313079833984,
      "logits/rejected": -0.387094110250473,
      "logps/chosen": -519.7581787109375,
      "logps/rejected": -413.912841796875,
      "loss": 0.4069,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1754474639892578,
      "rewards/margins": 1.4489531517028809,
      "rewards/rejected": -1.6244004964828491,
      "step": 14915
    },
    {
      "epoch": 1.9518606362967204,
      "grad_norm": 7.975254058837891,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12195815145969391,
      "logits/rejected": -0.28465184569358826,
      "logps/chosen": -398.2845153808594,
      "logps/rejected": -344.2771911621094,
      "loss": 0.4483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4195557832717896,
      "rewards/margins": 1.2340104579925537,
      "rewards/rejected": -2.6535661220550537,
      "step": 14916
    },
    {
      "epoch": 1.9519914942340721,
      "grad_norm": 9.490832328796387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14134159684181213,
      "logits/rejected": -0.100095734000206,
      "logps/chosen": -403.37860107421875,
      "logps/rejected": -578.679443359375,
      "loss": 0.4251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9544086456298828,
      "rewards/margins": 1.603902816772461,
      "rewards/rejected": -2.558311700820923,
      "step": 14917
    },
    {
      "epoch": 1.952122352171424,
      "grad_norm": 9.537665367126465,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31530290842056274,
      "logits/rejected": -0.5237188935279846,
      "logps/chosen": -452.612060546875,
      "logps/rejected": -623.185302734375,
      "loss": 0.4387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7470853924751282,
      "rewards/margins": 0.9546613693237305,
      "rewards/rejected": -1.7017468214035034,
      "step": 14918
    },
    {
      "epoch": 1.9522532101087755,
      "grad_norm": 5.537647724151611,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15509729087352753,
      "logits/rejected": -0.16750089824199677,
      "logps/chosen": -419.2373046875,
      "logps/rejected": -431.4173583984375,
      "loss": 0.1838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8452695608139038,
      "rewards/margins": 2.393049955368042,
      "rewards/rejected": -3.2383196353912354,
      "step": 14919
    },
    {
      "epoch": 1.9523840680461273,
      "grad_norm": 13.425675392150879,
      "learning_rate": 5e-06,
      "logits/chosen": -0.393329381942749,
      "logits/rejected": -0.35794883966445923,
      "logps/chosen": -364.91510009765625,
      "logps/rejected": -372.0619201660156,
      "loss": 0.5214,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3733394145965576,
      "rewards/margins": 1.4222419261932373,
      "rewards/rejected": -2.795581340789795,
      "step": 14920
    },
    {
      "epoch": 1.952514925983479,
      "grad_norm": 8.953384399414062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2734535336494446,
      "logits/rejected": -0.26508188247680664,
      "logps/chosen": -342.787841796875,
      "logps/rejected": -382.9256896972656,
      "loss": 0.4942,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1610811948776245,
      "rewards/margins": 0.5218778252601624,
      "rewards/rejected": -1.682958960533142,
      "step": 14921
    },
    {
      "epoch": 1.9526457839208309,
      "grad_norm": 11.339240074157715,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2412726730108261,
      "logits/rejected": -0.3315873146057129,
      "logps/chosen": -351.3294982910156,
      "logps/rejected": -332.4154968261719,
      "loss": 0.6505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4784553050994873,
      "rewards/margins": 0.9708113670349121,
      "rewards/rejected": -2.4492669105529785,
      "step": 14922
    },
    {
      "epoch": 1.9527766418581827,
      "grad_norm": 12.523933410644531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3490932285785675,
      "logits/rejected": -0.3904764652252197,
      "logps/chosen": -402.9107360839844,
      "logps/rejected": -394.5461120605469,
      "loss": 0.4683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1605281829833984,
      "rewards/margins": 1.8642299175262451,
      "rewards/rejected": -4.0247578620910645,
      "step": 14923
    },
    {
      "epoch": 1.9529074997955345,
      "grad_norm": 9.954051971435547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42433762550354004,
      "logits/rejected": -0.39140379428863525,
      "logps/chosen": -515.1146240234375,
      "logps/rejected": -410.8543701171875,
      "loss": 0.5934,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.266592025756836,
      "rewards/margins": 0.7143646478652954,
      "rewards/rejected": -1.9809565544128418,
      "step": 14924
    },
    {
      "epoch": 1.9530383577328863,
      "grad_norm": 7.21891975402832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11273666471242905,
      "logits/rejected": -0.1765747368335724,
      "logps/chosen": -425.1148681640625,
      "logps/rejected": -630.8475952148438,
      "loss": 0.2884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3454822301864624,
      "rewards/margins": 2.0230958461761475,
      "rewards/rejected": -3.3685781955718994,
      "step": 14925
    },
    {
      "epoch": 1.953169215670238,
      "grad_norm": 8.25745677947998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4743380844593048,
      "logits/rejected": -0.5951864123344421,
      "logps/chosen": -499.88397216796875,
      "logps/rejected": -425.7701721191406,
      "loss": 0.3165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8701423406600952,
      "rewards/margins": 1.4134138822555542,
      "rewards/rejected": -2.2835562229156494,
      "step": 14926
    },
    {
      "epoch": 1.9533000736075898,
      "grad_norm": 14.569263458251953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05368378013372421,
      "logits/rejected": -0.22258591651916504,
      "logps/chosen": -493.1604919433594,
      "logps/rejected": -392.5527648925781,
      "loss": 0.4635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8050804138183594,
      "rewards/margins": 1.0011379718780518,
      "rewards/rejected": -1.8062183856964111,
      "step": 14927
    },
    {
      "epoch": 1.9534309315449416,
      "grad_norm": 8.322997093200684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4112356901168823,
      "logits/rejected": -0.15168872475624084,
      "logps/chosen": -304.66680908203125,
      "logps/rejected": -379.71484375,
      "loss": 0.5704,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.456360787153244,
      "rewards/margins": 1.1367497444152832,
      "rewards/rejected": -1.5931103229522705,
      "step": 14928
    },
    {
      "epoch": 1.9535617894822934,
      "grad_norm": 6.387545108795166,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24931754171848297,
      "logits/rejected": -0.2108413726091385,
      "logps/chosen": -328.089111328125,
      "logps/rejected": -370.8348083496094,
      "loss": 0.2402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25790196657180786,
      "rewards/margins": 1.4448432922363281,
      "rewards/rejected": -1.7027451992034912,
      "step": 14929
    },
    {
      "epoch": 1.9536926474196452,
      "grad_norm": 14.353063583374023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3645205497741699,
      "logits/rejected": -0.41419318318367004,
      "logps/chosen": -736.412353515625,
      "logps/rejected": -530.0707397460938,
      "loss": 0.828,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0525575876235962,
      "rewards/margins": 0.21499767899513245,
      "rewards/rejected": -1.2675552368164062,
      "step": 14930
    },
    {
      "epoch": 1.9538235053569968,
      "grad_norm": 16.104015350341797,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03504139184951782,
      "logits/rejected": -0.1374177634716034,
      "logps/chosen": -569.58203125,
      "logps/rejected": -532.2054443359375,
      "loss": 0.4745,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9024688005447388,
      "rewards/margins": 0.779998779296875,
      "rewards/rejected": -1.6824675798416138,
      "step": 14931
    },
    {
      "epoch": 1.9539543632943486,
      "grad_norm": 8.320401191711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05875002592802048,
      "logits/rejected": -0.3548080623149872,
      "logps/chosen": -401.5611572265625,
      "logps/rejected": -434.11517333984375,
      "loss": 0.3735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06819742918014526,
      "rewards/margins": 1.9740233421325684,
      "rewards/rejected": -2.0422205924987793,
      "step": 14932
    },
    {
      "epoch": 1.9540852212317004,
      "grad_norm": 11.895630836486816,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2942869961261749,
      "logits/rejected": -0.6783890724182129,
      "logps/chosen": -575.267578125,
      "logps/rejected": -383.9168701171875,
      "loss": 0.4957,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5836631655693054,
      "rewards/margins": 1.5189310312271118,
      "rewards/rejected": -2.1025943756103516,
      "step": 14933
    },
    {
      "epoch": 1.954216079169052,
      "grad_norm": 6.914787292480469,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07015896588563919,
      "logits/rejected": -0.36478158831596375,
      "logps/chosen": -456.39324951171875,
      "logps/rejected": -396.7547912597656,
      "loss": 0.2618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0859915018081665,
      "rewards/margins": 2.243797779083252,
      "rewards/rejected": -3.329789161682129,
      "step": 14934
    },
    {
      "epoch": 1.9543469371064037,
      "grad_norm": 9.079322814941406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1328434944152832,
      "logits/rejected": -0.15036433935165405,
      "logps/chosen": -464.2921447753906,
      "logps/rejected": -322.2073669433594,
      "loss": 0.4843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9360843896865845,
      "rewards/margins": 1.0958691835403442,
      "rewards/rejected": -2.0319535732269287,
      "step": 14935
    },
    {
      "epoch": 1.9544777950437555,
      "grad_norm": 5.144931316375732,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2530553340911865,
      "logits/rejected": -0.0450332872569561,
      "logps/chosen": -624.4383544921875,
      "logps/rejected": -613.10400390625,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06735438108444214,
      "rewards/margins": 2.831188440322876,
      "rewards/rejected": -2.898542642593384,
      "step": 14936
    },
    {
      "epoch": 1.9546086529811073,
      "grad_norm": 11.88476848602295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41105759143829346,
      "logits/rejected": -0.3507784307003021,
      "logps/chosen": -543.094482421875,
      "logps/rejected": -533.6009521484375,
      "loss": 0.6812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2780555486679077,
      "rewards/margins": 0.6165163516998291,
      "rewards/rejected": -1.8945717811584473,
      "step": 14937
    },
    {
      "epoch": 1.954739510918459,
      "grad_norm": 14.07645320892334,
      "learning_rate": 5e-06,
      "logits/chosen": -0.28296372294425964,
      "logits/rejected": -0.34154826402664185,
      "logps/chosen": -717.2388916015625,
      "logps/rejected": -474.9028625488281,
      "loss": 0.6227,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8287856578826904,
      "rewards/margins": 0.8287918567657471,
      "rewards/rejected": -1.657577633857727,
      "step": 14938
    },
    {
      "epoch": 1.954870368855811,
      "grad_norm": 8.404640197753906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6107608079910278,
      "logits/rejected": -0.3517528772354126,
      "logps/chosen": -379.03924560546875,
      "logps/rejected": -375.08380126953125,
      "loss": 0.4221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.008312702178955,
      "rewards/margins": 1.1105358600616455,
      "rewards/rejected": -2.1188485622406006,
      "step": 14939
    },
    {
      "epoch": 1.9550012267931627,
      "grad_norm": 6.362077236175537,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15299460291862488,
      "logits/rejected": -0.27688536047935486,
      "logps/chosen": -441.5472412109375,
      "logps/rejected": -471.8500061035156,
      "loss": 0.2498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1962796449661255,
      "rewards/margins": 2.1373729705810547,
      "rewards/rejected": -3.3336527347564697,
      "step": 14940
    },
    {
      "epoch": 1.9551320847305145,
      "grad_norm": 8.964582443237305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09703388065099716,
      "logits/rejected": -0.21783509850502014,
      "logps/chosen": -336.0142822265625,
      "logps/rejected": -322.81866455078125,
      "loss": 0.5113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18770748376846313,
      "rewards/margins": 0.924626350402832,
      "rewards/rejected": -1.1123337745666504,
      "step": 14941
    },
    {
      "epoch": 1.9552629426678663,
      "grad_norm": 6.481982707977295,
      "learning_rate": 5e-06,
      "logits/chosen": -0.19914495944976807,
      "logits/rejected": -0.27648019790649414,
      "logps/chosen": -313.4638366699219,
      "logps/rejected": -406.271484375,
      "loss": 0.4115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1107542514801025,
      "rewards/margins": 1.1195249557495117,
      "rewards/rejected": -2.2302792072296143,
      "step": 14942
    },
    {
      "epoch": 1.955393800605218,
      "grad_norm": 8.432637214660645,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4113115072250366,
      "logits/rejected": -0.419981449842453,
      "logps/chosen": -220.09307861328125,
      "logps/rejected": -309.58251953125,
      "loss": 0.5104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3868693113327026,
      "rewards/margins": 1.9086698293685913,
      "rewards/rejected": -3.295539379119873,
      "step": 14943
    },
    {
      "epoch": 1.9555246585425698,
      "grad_norm": 8.363127708435059,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09057451784610748,
      "logits/rejected": -0.14513681828975677,
      "logps/chosen": -485.82025146484375,
      "logps/rejected": -341.09735107421875,
      "loss": 0.2305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6364988088607788,
      "rewards/margins": 2.1724178791046143,
      "rewards/rejected": -2.8089168071746826,
      "step": 14944
    },
    {
      "epoch": 1.9556555164799216,
      "grad_norm": 6.026478290557861,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18001192808151245,
      "logits/rejected": -0.21549449861049652,
      "logps/chosen": -338.3243408203125,
      "logps/rejected": -366.0466003417969,
      "loss": 0.2347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7051727175712585,
      "rewards/margins": 1.5241765975952148,
      "rewards/rejected": -2.229349374771118,
      "step": 14945
    },
    {
      "epoch": 1.9557863744172732,
      "grad_norm": 12.078364372253418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4286295473575592,
      "logits/rejected": -0.5269184112548828,
      "logps/chosen": -562.77392578125,
      "logps/rejected": -441.88433837890625,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4148157835006714,
      "rewards/margins": 2.801429033279419,
      "rewards/rejected": -4.216244697570801,
      "step": 14946
    },
    {
      "epoch": 1.955917232354625,
      "grad_norm": 8.682324409484863,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4556913375854492,
      "logits/rejected": -0.6619875431060791,
      "logps/chosen": -312.1038818359375,
      "logps/rejected": -341.4947814941406,
      "loss": 0.3014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8637970685958862,
      "rewards/margins": 1.9291870594024658,
      "rewards/rejected": -3.7929844856262207,
      "step": 14947
    },
    {
      "epoch": 1.9560480902919768,
      "grad_norm": 8.561044692993164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0937761664390564,
      "logits/rejected": -0.17923536896705627,
      "logps/chosen": -502.7218017578125,
      "logps/rejected": -473.4124755859375,
      "loss": 0.3314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7702186703681946,
      "rewards/margins": 1.4256234169006348,
      "rewards/rejected": -2.1958420276641846,
      "step": 14948
    },
    {
      "epoch": 1.9561789482293286,
      "grad_norm": 14.762239456176758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3295997977256775,
      "logits/rejected": -0.24206110835075378,
      "logps/chosen": -542.1793823242188,
      "logps/rejected": -473.9468994140625,
      "loss": 0.9719,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6389899253845215,
      "rewards/margins": 0.26877307891845703,
      "rewards/rejected": -0.9077629446983337,
      "step": 14949
    },
    {
      "epoch": 1.9563098061666802,
      "grad_norm": 10.421810150146484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14404641091823578,
      "logits/rejected": -0.34212225675582886,
      "logps/chosen": -521.7665405273438,
      "logps/rejected": -473.95849609375,
      "loss": 0.6517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.859593391418457,
      "rewards/margins": 0.46914416551589966,
      "rewards/rejected": -1.328737497329712,
      "step": 14950
    },
    {
      "epoch": 1.956440664104032,
      "grad_norm": 9.322566986083984,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2691611647605896,
      "logits/rejected": -0.4530576467514038,
      "logps/chosen": -520.944580078125,
      "logps/rejected": -459.5492858886719,
      "loss": 0.3905,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3805603981018066,
      "rewards/margins": 1.1992745399475098,
      "rewards/rejected": -2.5798349380493164,
      "step": 14951
    },
    {
      "epoch": 1.9565715220413837,
      "grad_norm": 7.7742486000061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37407195568084717,
      "logits/rejected": -0.6008605360984802,
      "logps/chosen": -409.9168701171875,
      "logps/rejected": -365.0686950683594,
      "loss": 0.2876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.487145185470581,
      "rewards/margins": 2.231661081314087,
      "rewards/rejected": -3.718806505203247,
      "step": 14952
    },
    {
      "epoch": 1.9567023799787355,
      "grad_norm": 8.703380584716797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04511445388197899,
      "logits/rejected": -0.1436297595500946,
      "logps/chosen": -338.4414978027344,
      "logps/rejected": -366.70306396484375,
      "loss": 0.499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42983803153038025,
      "rewards/margins": 1.0346794128417969,
      "rewards/rejected": -1.4645174741744995,
      "step": 14953
    },
    {
      "epoch": 1.9568332379160873,
      "grad_norm": 4.504928112030029,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49168428778648376,
      "logits/rejected": -0.5204790234565735,
      "logps/chosen": -261.9164733886719,
      "logps/rejected": -294.90380859375,
      "loss": 0.2424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5326986312866211,
      "rewards/margins": 2.0582101345062256,
      "rewards/rejected": -2.5909085273742676,
      "step": 14954
    },
    {
      "epoch": 1.9569640958534391,
      "grad_norm": 7.737887859344482,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4786107540130615,
      "logits/rejected": -0.5790220499038696,
      "logps/chosen": -525.4189453125,
      "logps/rejected": -561.1690673828125,
      "loss": 0.4234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9330177307128906,
      "rewards/margins": 1.9583755731582642,
      "rewards/rejected": -3.8913934230804443,
      "step": 14955
    },
    {
      "epoch": 1.957094953790791,
      "grad_norm": 3.903890609741211,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3331022560596466,
      "logits/rejected": -0.6613597869873047,
      "logps/chosen": -444.19659423828125,
      "logps/rejected": -362.447998046875,
      "loss": 0.1247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5984781384468079,
      "rewards/margins": 2.604048728942871,
      "rewards/rejected": -3.2025270462036133,
      "step": 14956
    },
    {
      "epoch": 1.9572258117281427,
      "grad_norm": 6.116054058074951,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2520125210285187,
      "logits/rejected": -0.3552183210849762,
      "logps/chosen": -359.8343200683594,
      "logps/rejected": -321.24920654296875,
      "loss": 0.3767,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7858843803405762,
      "rewards/margins": 1.565281867980957,
      "rewards/rejected": -3.351166248321533,
      "step": 14957
    },
    {
      "epoch": 1.9573566696654945,
      "grad_norm": 12.14476203918457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2886506915092468,
      "logits/rejected": -0.317240446805954,
      "logps/chosen": -363.0366516113281,
      "logps/rejected": -436.59564208984375,
      "loss": 0.5085,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6477493047714233,
      "rewards/margins": 1.0360560417175293,
      "rewards/rejected": -2.683804988861084,
      "step": 14958
    },
    {
      "epoch": 1.9574875276028463,
      "grad_norm": 6.4862470626831055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24190814793109894,
      "logits/rejected": -0.5526699423789978,
      "logps/chosen": -532.42724609375,
      "logps/rejected": -587.86865234375,
      "loss": 0.2537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4073282480239868,
      "rewards/margins": 2.044858694076538,
      "rewards/rejected": -3.4521868228912354,
      "step": 14959
    },
    {
      "epoch": 1.957618385540198,
      "grad_norm": 6.0475053787231445,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1852121502161026,
      "logits/rejected": -0.17544542253017426,
      "logps/chosen": -429.5229797363281,
      "logps/rejected": -397.8016357421875,
      "loss": 0.3075,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7927456498146057,
      "rewards/margins": 1.3577216863632202,
      "rewards/rejected": -2.1504673957824707,
      "step": 14960
    },
    {
      "epoch": 1.9577492434775496,
      "grad_norm": 6.111482620239258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3482491374015808,
      "logits/rejected": -0.3049309253692627,
      "logps/chosen": -337.0885925292969,
      "logps/rejected": -385.4080505371094,
      "loss": 0.2882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.172281265258789,
      "rewards/margins": 1.4261877536773682,
      "rewards/rejected": -2.5984692573547363,
      "step": 14961
    },
    {
      "epoch": 1.9578801014149014,
      "grad_norm": 6.613376617431641,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29431384801864624,
      "logits/rejected": -0.3062947988510132,
      "logps/chosen": -424.11212158203125,
      "logps/rejected": -409.18646240234375,
      "loss": 0.2426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.442439079284668,
      "rewards/margins": 1.6984602212905884,
      "rewards/rejected": -3.140899181365967,
      "step": 14962
    },
    {
      "epoch": 1.9580109593522532,
      "grad_norm": 7.480698585510254,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6047780513763428,
      "logits/rejected": -0.5412057638168335,
      "logps/chosen": -342.6700134277344,
      "logps/rejected": -467.2822570800781,
      "loss": 0.298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0532103776931763,
      "rewards/margins": 2.497546434402466,
      "rewards/rejected": -3.5507569313049316,
      "step": 14963
    },
    {
      "epoch": 1.958141817289605,
      "grad_norm": 4.331813812255859,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17072264850139618,
      "logits/rejected": -0.22946704924106598,
      "logps/chosen": -367.7039794921875,
      "logps/rejected": -310.8877868652344,
      "loss": 0.2635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6218870282173157,
      "rewards/margins": 2.1735153198242188,
      "rewards/rejected": -2.7954025268554688,
      "step": 14964
    },
    {
      "epoch": 1.9582726752269566,
      "grad_norm": 6.978176116943359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2455049753189087,
      "logits/rejected": -0.4828418493270874,
      "logps/chosen": -419.9999694824219,
      "logps/rejected": -597.2239379882812,
      "loss": 0.2332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3058819770812988,
      "rewards/margins": 2.312868595123291,
      "rewards/rejected": -3.6187503337860107,
      "step": 14965
    },
    {
      "epoch": 1.9584035331643084,
      "grad_norm": 11.74717903137207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5076278448104858,
      "logits/rejected": -0.38469597697257996,
      "logps/chosen": -451.3079833984375,
      "logps/rejected": -487.7203369140625,
      "loss": 0.381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9233266711235046,
      "rewards/margins": 2.442812919616699,
      "rewards/rejected": -3.3661394119262695,
      "step": 14966
    },
    {
      "epoch": 1.9585343911016602,
      "grad_norm": 2.715489149093628,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004772700369358063,
      "logits/rejected": -0.23488467931747437,
      "logps/chosen": -493.99493408203125,
      "logps/rejected": -562.4755249023438,
      "loss": 0.087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4103453755378723,
      "rewards/margins": 2.6546621322631836,
      "rewards/rejected": -3.065007448196411,
      "step": 14967
    },
    {
      "epoch": 1.958665249039012,
      "grad_norm": 10.234349250793457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40276429057121277,
      "logits/rejected": -0.6868301630020142,
      "logps/chosen": -316.00067138671875,
      "logps/rejected": -563.06787109375,
      "loss": 0.6421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.291985034942627,
      "rewards/margins": 2.384381055831909,
      "rewards/rejected": -3.676366090774536,
      "step": 14968
    },
    {
      "epoch": 1.9587961069763637,
      "grad_norm": 7.5099778175354,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13090020418167114,
      "logits/rejected": -0.3148135542869568,
      "logps/chosen": -399.9234619140625,
      "logps/rejected": -446.28143310546875,
      "loss": 0.3512,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8976492881774902,
      "rewards/margins": 1.7187888622283936,
      "rewards/rejected": -3.6164379119873047,
      "step": 14969
    },
    {
      "epoch": 1.9589269649137155,
      "grad_norm": 9.814468383789062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2610797584056854,
      "logits/rejected": -0.49696457386016846,
      "logps/chosen": -305.2043762207031,
      "logps/rejected": -532.7842407226562,
      "loss": 0.4598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0920010805130005,
      "rewards/margins": 2.0753211975097656,
      "rewards/rejected": -3.1673219203948975,
      "step": 14970
    },
    {
      "epoch": 1.9590578228510673,
      "grad_norm": 5.533992290496826,
      "learning_rate": 5e-06,
      "logits/chosen": -0.327359676361084,
      "logits/rejected": -0.7149718999862671,
      "logps/chosen": -502.1034240722656,
      "logps/rejected": -485.73394775390625,
      "loss": 0.2114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8793973922729492,
      "rewards/margins": 1.8643035888671875,
      "rewards/rejected": -3.743701219558716,
      "step": 14971
    },
    {
      "epoch": 1.9591886807884191,
      "grad_norm": 9.326903343200684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07686661183834076,
      "logits/rejected": -0.09936223179101944,
      "logps/chosen": -373.77191162109375,
      "logps/rejected": -321.9213562011719,
      "loss": 0.5262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6446475982666016,
      "rewards/margins": 0.5161741375923157,
      "rewards/rejected": -2.1608216762542725,
      "step": 14972
    },
    {
      "epoch": 1.959319538725771,
      "grad_norm": 7.622227668762207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03779499977827072,
      "logits/rejected": -0.09954062849283218,
      "logps/chosen": -584.6271362304688,
      "logps/rejected": -530.5125732421875,
      "loss": 0.3181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2420109212398529,
      "rewards/margins": 1.4656352996826172,
      "rewards/rejected": -1.707646131515503,
      "step": 14973
    },
    {
      "epoch": 1.9594503966631227,
      "grad_norm": 8.046180725097656,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16110140085220337,
      "logits/rejected": -0.36551597714424133,
      "logps/chosen": -370.47552490234375,
      "logps/rejected": -506.19854736328125,
      "loss": 0.2132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0843762457370758,
      "rewards/margins": 2.5436618328094482,
      "rewards/rejected": -2.628037929534912,
      "step": 14974
    },
    {
      "epoch": 1.9595812546004745,
      "grad_norm": 3.591670513153076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3650543689727783,
      "logits/rejected": -0.45321106910705566,
      "logps/chosen": -339.6153564453125,
      "logps/rejected": -457.43341064453125,
      "loss": 0.1023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23518216609954834,
      "rewards/margins": 3.239581823348999,
      "rewards/rejected": -3.4747636318206787,
      "step": 14975
    },
    {
      "epoch": 1.959712112537826,
      "grad_norm": 7.263967514038086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3772784471511841,
      "logits/rejected": -0.4418337047100067,
      "logps/chosen": -446.7328186035156,
      "logps/rejected": -357.5277099609375,
      "loss": 0.4418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6165077686309814,
      "rewards/margins": 1.2450480461120605,
      "rewards/rejected": -2.861555814743042,
      "step": 14976
    },
    {
      "epoch": 1.9598429704751779,
      "grad_norm": 12.593047142028809,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2748018205165863,
      "logits/rejected": -0.21431148052215576,
      "logps/chosen": -337.0623474121094,
      "logps/rejected": -361.96075439453125,
      "loss": 0.7107,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5366328954696655,
      "rewards/margins": 0.4007682800292969,
      "rewards/rejected": -1.937401294708252,
      "step": 14977
    },
    {
      "epoch": 1.9599738284125297,
      "grad_norm": 6.292354583740234,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03750868886709213,
      "logits/rejected": -0.25739404559135437,
      "logps/chosen": -596.4951782226562,
      "logps/rejected": -558.1657104492188,
      "loss": 0.2525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.800864815711975,
      "rewards/margins": 1.770491123199463,
      "rewards/rejected": -3.5713560581207275,
      "step": 14978
    },
    {
      "epoch": 1.9601046863498814,
      "grad_norm": 12.254754066467285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13602913916110992,
      "logits/rejected": -0.179925337433815,
      "logps/chosen": -603.6002807617188,
      "logps/rejected": -714.90185546875,
      "loss": 0.5841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.25333833694458,
      "rewards/margins": 0.8606448173522949,
      "rewards/rejected": -2.113983154296875,
      "step": 14979
    },
    {
      "epoch": 1.960235544287233,
      "grad_norm": 18.24898910522461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.380008727312088,
      "logits/rejected": -0.43185168504714966,
      "logps/chosen": -464.8495178222656,
      "logps/rejected": -428.29827880859375,
      "loss": 0.5145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4104702472686768,
      "rewards/margins": 1.4550879001617432,
      "rewards/rejected": -2.865558624267578,
      "step": 14980
    },
    {
      "epoch": 1.9603664022245848,
      "grad_norm": 9.009114265441895,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18605096638202667,
      "logits/rejected": -0.17339783906936646,
      "logps/chosen": -437.7554626464844,
      "logps/rejected": -340.1011962890625,
      "loss": 0.4158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.519604504108429,
      "rewards/margins": 1.779136300086975,
      "rewards/rejected": -2.298740863800049,
      "step": 14981
    },
    {
      "epoch": 1.9604972601619366,
      "grad_norm": 10.382355690002441,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5146063566207886,
      "logits/rejected": -0.2698473632335663,
      "logps/chosen": -529.0657958984375,
      "logps/rejected": -513.2205810546875,
      "loss": 0.5324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4113420248031616,
      "rewards/margins": 1.8866485357284546,
      "rewards/rejected": -3.297990322113037,
      "step": 14982
    },
    {
      "epoch": 1.9606281180992884,
      "grad_norm": 9.792113304138184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2467871606349945,
      "logits/rejected": -0.2692262530326843,
      "logps/chosen": -332.47320556640625,
      "logps/rejected": -655.7000122070312,
      "loss": 0.1992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3873411417007446,
      "rewards/margins": 2.325981378555298,
      "rewards/rejected": -3.713322401046753,
      "step": 14983
    },
    {
      "epoch": 1.9607589760366402,
      "grad_norm": 13.128594398498535,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5216891169548035,
      "logits/rejected": -0.5386430621147156,
      "logps/chosen": -432.90289306640625,
      "logps/rejected": -577.2408447265625,
      "loss": 0.5009,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5243360996246338,
      "rewards/margins": 1.1705718040466309,
      "rewards/rejected": -2.6949079036712646,
      "step": 14984
    },
    {
      "epoch": 1.960889833973992,
      "grad_norm": 6.06540584564209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27354615926742554,
      "logits/rejected": -0.2610296607017517,
      "logps/chosen": -454.22576904296875,
      "logps/rejected": -482.9234619140625,
      "loss": 0.2677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2208313941955566,
      "rewards/margins": 1.977028727531433,
      "rewards/rejected": -3.1978602409362793,
      "step": 14985
    },
    {
      "epoch": 1.9610206919113438,
      "grad_norm": 7.0715227127075195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22944849729537964,
      "logits/rejected": -0.4084964692592621,
      "logps/chosen": -508.9727478027344,
      "logps/rejected": -553.8403930664062,
      "loss": 0.2363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9666409492492676,
      "rewards/margins": 2.3654422760009766,
      "rewards/rejected": -3.332082986831665,
      "step": 14986
    },
    {
      "epoch": 1.9611515498486956,
      "grad_norm": 12.84475326538086,
      "learning_rate": 5e-06,
      "logits/chosen": -0.48569661378860474,
      "logits/rejected": -0.4174143671989441,
      "logps/chosen": -336.3587341308594,
      "logps/rejected": -340.0126953125,
      "loss": 0.6817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.345374345779419,
      "rewards/margins": 0.49778157472610474,
      "rewards/rejected": -2.843155860900879,
      "step": 14987
    },
    {
      "epoch": 1.9612824077860473,
      "grad_norm": 12.740859031677246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26266151666641235,
      "logits/rejected": -0.38489973545074463,
      "logps/chosen": -600.7103881835938,
      "logps/rejected": -482.248779296875,
      "loss": 0.5407,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.436864972114563,
      "rewards/margins": 0.9271718263626099,
      "rewards/rejected": -2.364036798477173,
      "step": 14988
    },
    {
      "epoch": 1.9614132657233991,
      "grad_norm": 7.015012264251709,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06225524842739105,
      "logits/rejected": -0.1959969699382782,
      "logps/chosen": -510.9308166503906,
      "logps/rejected": -437.95001220703125,
      "loss": 0.261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3752161264419556,
      "rewards/margins": 2.4920713901519775,
      "rewards/rejected": -3.8672876358032227,
      "step": 14989
    },
    {
      "epoch": 1.961544123660751,
      "grad_norm": 10.106593132019043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4590100049972534,
      "logits/rejected": -0.4819713234901428,
      "logps/chosen": -747.9066772460938,
      "logps/rejected": -642.333740234375,
      "loss": 0.499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.333229422569275,
      "rewards/margins": 2.075089693069458,
      "rewards/rejected": -3.4083189964294434,
      "step": 14990
    },
    {
      "epoch": 1.9616749815981027,
      "grad_norm": 6.978499889373779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1043073758482933,
      "logits/rejected": -0.16994288563728333,
      "logps/chosen": -347.30572509765625,
      "logps/rejected": -289.49462890625,
      "loss": 0.386,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.593585729598999,
      "rewards/margins": 1.53157639503479,
      "rewards/rejected": -3.1251626014709473,
      "step": 14991
    },
    {
      "epoch": 1.9618058395354543,
      "grad_norm": 6.493090629577637,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09989924728870392,
      "logits/rejected": -0.46360182762145996,
      "logps/chosen": -470.9002685546875,
      "logps/rejected": -437.16851806640625,
      "loss": 0.2593,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9865607023239136,
      "rewards/margins": 1.9615402221679688,
      "rewards/rejected": -2.9481008052825928,
      "step": 14992
    },
    {
      "epoch": 1.961936697472806,
      "grad_norm": 14.143728256225586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.49176689982414246,
      "logits/rejected": -0.35215651988983154,
      "logps/chosen": -548.6407470703125,
      "logps/rejected": -565.5657958984375,
      "loss": 0.7672,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7585678100585938,
      "rewards/margins": 0.9446895718574524,
      "rewards/rejected": -2.7032575607299805,
      "step": 14993
    },
    {
      "epoch": 1.9620675554101579,
      "grad_norm": 13.525979042053223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.463615357875824,
      "logits/rejected": -0.4171069264411926,
      "logps/chosen": -345.7315368652344,
      "logps/rejected": -243.75196838378906,
      "loss": 0.7963,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4725747108459473,
      "rewards/margins": 0.39946889877319336,
      "rewards/rejected": -2.8720438480377197,
      "step": 14994
    },
    {
      "epoch": 1.9621984133475097,
      "grad_norm": 10.769881248474121,
      "learning_rate": 5e-06,
      "logits/chosen": 0.01211945153772831,
      "logits/rejected": -0.1569429486989975,
      "logps/chosen": -545.34326171875,
      "logps/rejected": -630.4832763671875,
      "loss": 0.5855,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2472989559173584,
      "rewards/margins": 1.168151617050171,
      "rewards/rejected": -2.4154508113861084,
      "step": 14995
    },
    {
      "epoch": 1.9623292712848612,
      "grad_norm": 10.535666465759277,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2712651789188385,
      "logits/rejected": -0.3705141544342041,
      "logps/chosen": -443.14422607421875,
      "logps/rejected": -488.41595458984375,
      "loss": 0.54,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.068636894226074,
      "rewards/margins": 0.8775978088378906,
      "rewards/rejected": -2.9462344646453857,
      "step": 14996
    },
    {
      "epoch": 1.962460129222213,
      "grad_norm": 12.698037147521973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3863402307033539,
      "logits/rejected": -0.41620010137557983,
      "logps/chosen": -584.3524169921875,
      "logps/rejected": -519.0960693359375,
      "loss": 0.463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.404127836227417,
      "rewards/margins": 0.7147483825683594,
      "rewards/rejected": -3.1188764572143555,
      "step": 14997
    },
    {
      "epoch": 1.9625909871595648,
      "grad_norm": 15.217077255249023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33287766575813293,
      "logits/rejected": -0.3549045920372009,
      "logps/chosen": -388.3372802734375,
      "logps/rejected": -346.022216796875,
      "loss": 0.6938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0948975086212158,
      "rewards/margins": 0.9347162842750549,
      "rewards/rejected": -2.029613733291626,
      "step": 14998
    },
    {
      "epoch": 1.9627218450969166,
      "grad_norm": 12.860862731933594,
      "learning_rate": 5e-06,
      "logits/chosen": -0.03453170880675316,
      "logits/rejected": -0.21241629123687744,
      "logps/chosen": -544.4273681640625,
      "logps/rejected": -507.552734375,
      "loss": 0.7096,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8816974759101868,
      "rewards/margins": 1.2127351760864258,
      "rewards/rejected": -2.0944325923919678,
      "step": 14999
    },
    {
      "epoch": 1.9628527030342684,
      "grad_norm": 14.201895713806152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.44100916385650635,
      "logits/rejected": -0.40852785110473633,
      "logps/chosen": -384.64886474609375,
      "logps/rejected": -319.42694091796875,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.365703821182251,
      "rewards/margins": 0.7149015069007874,
      "rewards/rejected": -3.0806052684783936,
      "step": 15000
    },
    {
      "epoch": 1.9629835609716202,
      "grad_norm": 11.869584083557129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14290212094783783,
      "logits/rejected": -0.11862382292747498,
      "logps/chosen": -673.87158203125,
      "logps/rejected": -495.9281311035156,
      "loss": 0.4914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.890160322189331,
      "rewards/margins": 2.0736069679260254,
      "rewards/rejected": -3.9637672901153564,
      "step": 15001
    },
    {
      "epoch": 1.963114418908972,
      "grad_norm": 14.398489952087402,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2812904715538025,
      "logits/rejected": -0.24680635333061218,
      "logps/chosen": -642.1939697265625,
      "logps/rejected": -652.13916015625,
      "loss": 0.5847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.446851968765259,
      "rewards/margins": 0.7696040868759155,
      "rewards/rejected": -3.2164559364318848,
      "step": 15002
    },
    {
      "epoch": 1.9632452768463238,
      "grad_norm": 5.3857316970825195,
      "learning_rate": 5e-06,
      "logits/chosen": -0.027814416214823723,
      "logits/rejected": -0.42427965998649597,
      "logps/chosen": -397.3316650390625,
      "logps/rejected": -559.7967529296875,
      "loss": 0.2595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.236889123916626,
      "rewards/margins": 2.3903183937072754,
      "rewards/rejected": -3.6272075176239014,
      "step": 15003
    },
    {
      "epoch": 1.9633761347836756,
      "grad_norm": 8.1136474609375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45045632123947144,
      "logits/rejected": -0.4457909166812897,
      "logps/chosen": -596.0272827148438,
      "logps/rejected": -513.15869140625,
      "loss": 0.304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0806045532226562,
      "rewards/margins": 1.7037934064865112,
      "rewards/rejected": -2.784398078918457,
      "step": 15004
    },
    {
      "epoch": 1.9635069927210274,
      "grad_norm": 9.995985984802246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21352815628051758,
      "logits/rejected": -0.6204708814620972,
      "logps/chosen": -488.56231689453125,
      "logps/rejected": -435.1561279296875,
      "loss": 0.4003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.022629976272583,
      "rewards/margins": 1.0288435220718384,
      "rewards/rejected": -2.051473379135132,
      "step": 15005
    },
    {
      "epoch": 1.9636378506583791,
      "grad_norm": 12.349151611328125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5189722776412964,
      "logits/rejected": -0.37128281593322754,
      "logps/chosen": -361.2916564941406,
      "logps/rejected": -403.78839111328125,
      "loss": 0.6711,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2380478382110596,
      "rewards/margins": 1.1594828367233276,
      "rewards/rejected": -3.3975305557250977,
      "step": 15006
    },
    {
      "epoch": 1.9637687085957307,
      "grad_norm": 8.85105037689209,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32847535610198975,
      "logits/rejected": -0.4942238926887512,
      "logps/chosen": -481.6539306640625,
      "logps/rejected": -433.3305358886719,
      "loss": 0.3874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3025081157684326,
      "rewards/margins": 1.7284736633300781,
      "rewards/rejected": -3.0309817790985107,
      "step": 15007
    },
    {
      "epoch": 1.9638995665330825,
      "grad_norm": 9.085575103759766,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2312236726284027,
      "logits/rejected": -0.2685447633266449,
      "logps/chosen": -471.71527099609375,
      "logps/rejected": -393.6214904785156,
      "loss": 0.4634,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7722436189651489,
      "rewards/margins": 1.5931470394134521,
      "rewards/rejected": -2.3653905391693115,
      "step": 15008
    },
    {
      "epoch": 1.9640304244704343,
      "grad_norm": 8.575249671936035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2222895473241806,
      "logits/rejected": -0.13845466077327728,
      "logps/chosen": -373.6573791503906,
      "logps/rejected": -461.86981201171875,
      "loss": 0.4761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8542481660842896,
      "rewards/margins": 1.0704147815704346,
      "rewards/rejected": -2.9246628284454346,
      "step": 15009
    },
    {
      "epoch": 1.964161282407786,
      "grad_norm": 7.311842441558838,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4221024513244629,
      "logits/rejected": -0.4842950701713562,
      "logps/chosen": -553.7684326171875,
      "logps/rejected": -382.10076904296875,
      "loss": 0.3809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2159143686294556,
      "rewards/margins": 1.3579769134521484,
      "rewards/rejected": -2.5738911628723145,
      "step": 15010
    },
    {
      "epoch": 1.9642921403451377,
      "grad_norm": 6.883581161499023,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3575204610824585,
      "logits/rejected": -0.33449625968933105,
      "logps/chosen": -441.3077392578125,
      "logps/rejected": -386.7793273925781,
      "loss": 0.3322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.232047438621521,
      "rewards/margins": 1.8480498790740967,
      "rewards/rejected": -3.080096960067749,
      "step": 15011
    },
    {
      "epoch": 1.9644229982824895,
      "grad_norm": 9.210970878601074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25288125872612,
      "logits/rejected": -0.4385918080806732,
      "logps/chosen": -499.9783935546875,
      "logps/rejected": -421.44476318359375,
      "loss": 0.4608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2060983180999756,
      "rewards/margins": 1.9066298007965088,
      "rewards/rejected": -3.1127281188964844,
      "step": 15012
    },
    {
      "epoch": 1.9645538562198412,
      "grad_norm": 8.115216255187988,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4568506181240082,
      "logits/rejected": -0.41866981983184814,
      "logps/chosen": -379.4975280761719,
      "logps/rejected": -434.22064208984375,
      "loss": 0.2855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8563809394836426,
      "rewards/margins": 1.4046705961227417,
      "rewards/rejected": -3.2610514163970947,
      "step": 15013
    },
    {
      "epoch": 1.964684714157193,
      "grad_norm": 13.39889144897461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.05915023013949394,
      "logits/rejected": 0.010876070708036423,
      "logps/chosen": -529.729736328125,
      "logps/rejected": -456.605224609375,
      "loss": 0.6211,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1261532306671143,
      "rewards/margins": 0.7916200757026672,
      "rewards/rejected": -2.9177732467651367,
      "step": 15014
    },
    {
      "epoch": 1.9648155720945448,
      "grad_norm": 3.5668818950653076,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34079861640930176,
      "logits/rejected": -0.5174286961555481,
      "logps/chosen": -432.5287170410156,
      "logps/rejected": -379.02325439453125,
      "loss": 0.2269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3281021118164062,
      "rewards/margins": 1.7149989604949951,
      "rewards/rejected": -3.0431008338928223,
      "step": 15015
    },
    {
      "epoch": 1.9649464300318966,
      "grad_norm": 4.633265972137451,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05975553020834923,
      "logits/rejected": 0.1067504733800888,
      "logps/chosen": -785.778564453125,
      "logps/rejected": -638.86572265625,
      "loss": 0.1949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1313953399658203,
      "rewards/margins": 2.023435592651367,
      "rewards/rejected": -3.1548311710357666,
      "step": 15016
    },
    {
      "epoch": 1.9650772879692484,
      "grad_norm": 10.521239280700684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7209806442260742,
      "logits/rejected": -0.6194707751274109,
      "logps/chosen": -341.8050231933594,
      "logps/rejected": -400.5262451171875,
      "loss": 0.3991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.610253095626831,
      "rewards/margins": 2.425947666168213,
      "rewards/rejected": -4.036201000213623,
      "step": 15017
    },
    {
      "epoch": 1.9652081459066002,
      "grad_norm": 5.535258769989014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3394938111305237,
      "logits/rejected": -0.43787437677383423,
      "logps/chosen": -394.5246887207031,
      "logps/rejected": -445.39666748046875,
      "loss": 0.2571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.229478359222412,
      "rewards/margins": 1.9021143913269043,
      "rewards/rejected": -3.1315927505493164,
      "step": 15018
    },
    {
      "epoch": 1.965339003843952,
      "grad_norm": 8.590351104736328,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07951658964157104,
      "logits/rejected": -0.19538350403308868,
      "logps/chosen": -510.381103515625,
      "logps/rejected": -426.3830871582031,
      "loss": 0.3164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6216285228729248,
      "rewards/margins": 1.701292634010315,
      "rewards/rejected": -3.3229212760925293,
      "step": 15019
    },
    {
      "epoch": 1.9654698617813038,
      "grad_norm": 13.145620346069336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24999012053012848,
      "logits/rejected": -0.2596273720264435,
      "logps/chosen": -415.286865234375,
      "logps/rejected": -495.3599548339844,
      "loss": 0.6868,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0350430011749268,
      "rewards/margins": 1.8033549785614014,
      "rewards/rejected": -3.838397979736328,
      "step": 15020
    },
    {
      "epoch": 1.9656007197186556,
      "grad_norm": 6.199182033538818,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32448360323905945,
      "logits/rejected": -0.4446376860141754,
      "logps/chosen": -336.5990905761719,
      "logps/rejected": -395.5557556152344,
      "loss": 0.2553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1550040245056152,
      "rewards/margins": 2.5520036220550537,
      "rewards/rejected": -3.70700740814209,
      "step": 15021
    },
    {
      "epoch": 1.9657315776560071,
      "grad_norm": 6.941874027252197,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33450111746788025,
      "logits/rejected": -0.287089079618454,
      "logps/chosen": -472.7327880859375,
      "logps/rejected": -487.43280029296875,
      "loss": 0.4532,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9539449214935303,
      "rewards/margins": 1.1399824619293213,
      "rewards/rejected": -3.0939273834228516,
      "step": 15022
    },
    {
      "epoch": 1.965862435593359,
      "grad_norm": 6.733322620391846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33286309242248535,
      "logits/rejected": -0.48186129331588745,
      "logps/chosen": -442.7729797363281,
      "logps/rejected": -334.2276916503906,
      "loss": 0.3088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4999336004257202,
      "rewards/margins": 1.1047817468643188,
      "rewards/rejected": -2.604715585708618,
      "step": 15023
    },
    {
      "epoch": 1.9659932935307107,
      "grad_norm": 19.68435287475586,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36062726378440857,
      "logits/rejected": -0.4282514452934265,
      "logps/chosen": -439.3852233886719,
      "logps/rejected": -422.49285888671875,
      "loss": 1.0132,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.605158567428589,
      "rewards/margins": 0.43198657035827637,
      "rewards/rejected": -3.0371451377868652,
      "step": 15024
    },
    {
      "epoch": 1.9661241514680625,
      "grad_norm": 6.122836112976074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0026548663154244423,
      "logits/rejected": -0.29258087277412415,
      "logps/chosen": -366.3094482421875,
      "logps/rejected": -354.51617431640625,
      "loss": 0.3397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.080104112625122,
      "rewards/margins": 1.6424142122268677,
      "rewards/rejected": -2.7225184440612793,
      "step": 15025
    },
    {
      "epoch": 1.966255009405414,
      "grad_norm": 8.390678405761719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2657264173030853,
      "logits/rejected": -0.2852749824523926,
      "logps/chosen": -483.46575927734375,
      "logps/rejected": -485.0506286621094,
      "loss": 0.4156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9284151792526245,
      "rewards/margins": 0.9097117185592651,
      "rewards/rejected": -1.8381268978118896,
      "step": 15026
    },
    {
      "epoch": 1.9663858673427659,
      "grad_norm": 8.547333717346191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35898011922836304,
      "logits/rejected": -0.35828280448913574,
      "logps/chosen": -454.6899108886719,
      "logps/rejected": -496.642578125,
      "loss": 0.525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1015560626983643,
      "rewards/margins": 1.8246105909347534,
      "rewards/rejected": -2.9261667728424072,
      "step": 15027
    },
    {
      "epoch": 1.9665167252801177,
      "grad_norm": 4.736108303070068,
      "learning_rate": 5e-06,
      "logits/chosen": -0.060820527374744415,
      "logits/rejected": -0.04020470380783081,
      "logps/chosen": -367.97735595703125,
      "logps/rejected": -392.6772155761719,
      "loss": 0.1905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0851263999938965,
      "rewards/margins": 2.579505681991577,
      "rewards/rejected": -3.6646318435668945,
      "step": 15028
    },
    {
      "epoch": 1.9666475832174695,
      "grad_norm": 8.90261173248291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3323914408683777,
      "logits/rejected": -0.18800583481788635,
      "logps/chosen": -348.92950439453125,
      "logps/rejected": -433.41253662109375,
      "loss": 0.5691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.155795693397522,
      "rewards/margins": 0.7965874075889587,
      "rewards/rejected": -1.9523831605911255,
      "step": 15029
    },
    {
      "epoch": 1.9667784411548213,
      "grad_norm": 8.268390655517578,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27885866165161133,
      "logits/rejected": -0.08421202003955841,
      "logps/chosen": -610.8518676757812,
      "logps/rejected": -835.2146606445312,
      "loss": 0.3178,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.651533842086792,
      "rewards/margins": 1.6273326873779297,
      "rewards/rejected": -3.278866767883301,
      "step": 15030
    },
    {
      "epoch": 1.966909299092173,
      "grad_norm": 11.392553329467773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10789045691490173,
      "logits/rejected": -0.07443810999393463,
      "logps/chosen": -509.2662353515625,
      "logps/rejected": -404.8165283203125,
      "loss": 0.4406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2976632118225098,
      "rewards/margins": 1.7342807054519653,
      "rewards/rejected": -3.0319437980651855,
      "step": 15031
    },
    {
      "epoch": 1.9670401570295248,
      "grad_norm": 4.383476734161377,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17522817850112915,
      "logits/rejected": -0.4273452162742615,
      "logps/chosen": -450.64208984375,
      "logps/rejected": -421.6114196777344,
      "loss": 0.1572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.199543833732605,
      "rewards/margins": 2.4822216033935547,
      "rewards/rejected": -3.681765556335449,
      "step": 15032
    },
    {
      "epoch": 1.9671710149668766,
      "grad_norm": 7.741801738739014,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06441006809473038,
      "logits/rejected": -0.02070339024066925,
      "logps/chosen": -483.1618957519531,
      "logps/rejected": -498.9482727050781,
      "loss": 0.4158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.053762435913086,
      "rewards/margins": 1.6939432621002197,
      "rewards/rejected": -2.7477059364318848,
      "step": 15033
    },
    {
      "epoch": 1.9673018729042284,
      "grad_norm": 6.6404500007629395,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4503944516181946,
      "logits/rejected": -0.47825315594673157,
      "logps/chosen": -565.3585815429688,
      "logps/rejected": -495.9862060546875,
      "loss": 0.1883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6433250308036804,
      "rewards/margins": 2.3019087314605713,
      "rewards/rejected": -2.9452333450317383,
      "step": 15034
    },
    {
      "epoch": 1.9674327308415802,
      "grad_norm": 9.056822776794434,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4195398688316345,
      "logits/rejected": -0.3238877058029175,
      "logps/chosen": -525.3045654296875,
      "logps/rejected": -568.8515625,
      "loss": 0.4739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.647156834602356,
      "rewards/margins": 1.0487816333770752,
      "rewards/rejected": -2.6959385871887207,
      "step": 15035
    },
    {
      "epoch": 1.967563588778932,
      "grad_norm": 6.302030563354492,
      "learning_rate": 5e-06,
      "logits/chosen": 0.002843886613845825,
      "logits/rejected": -0.15917882323265076,
      "logps/chosen": -346.7854309082031,
      "logps/rejected": -474.4503173828125,
      "loss": 0.2243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.552751898765564,
      "rewards/margins": 1.9677810668945312,
      "rewards/rejected": -3.5205330848693848,
      "step": 15036
    },
    {
      "epoch": 1.9676944467162838,
      "grad_norm": 4.652581691741943,
      "learning_rate": 5e-06,
      "logits/chosen": 0.17937569320201874,
      "logits/rejected": -0.1429465413093567,
      "logps/chosen": -361.6903991699219,
      "logps/rejected": -499.75299072265625,
      "loss": 0.2079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0558902025222778,
      "rewards/margins": 3.2319283485412598,
      "rewards/rejected": -4.28781795501709,
      "step": 15037
    },
    {
      "epoch": 1.9678253046536354,
      "grad_norm": 6.676459312438965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7519307136535645,
      "logits/rejected": -0.7612566947937012,
      "logps/chosen": -346.0383605957031,
      "logps/rejected": -383.105224609375,
      "loss": 0.3368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3284778594970703,
      "rewards/margins": 1.7364709377288818,
      "rewards/rejected": -3.0649490356445312,
      "step": 15038
    },
    {
      "epoch": 1.9679561625909872,
      "grad_norm": 10.090716361999512,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3423024117946625,
      "logits/rejected": -0.31764930486679077,
      "logps/chosen": -318.4892578125,
      "logps/rejected": -505.57550048828125,
      "loss": 0.4999,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8469130992889404,
      "rewards/margins": 1.1722393035888672,
      "rewards/rejected": -3.0191526412963867,
      "step": 15039
    },
    {
      "epoch": 1.968087020528339,
      "grad_norm": 10.299788475036621,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25608593225479126,
      "logits/rejected": -0.4136216640472412,
      "logps/chosen": -407.3606872558594,
      "logps/rejected": -342.47296142578125,
      "loss": 0.5064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8584650754928589,
      "rewards/margins": 1.574324607849121,
      "rewards/rejected": -3.4327893257141113,
      "step": 15040
    },
    {
      "epoch": 1.9682178784656907,
      "grad_norm": 9.595165252685547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26641154289245605,
      "logits/rejected": -0.15163764357566833,
      "logps/chosen": -427.1712341308594,
      "logps/rejected": -409.67291259765625,
      "loss": 0.5885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1847301721572876,
      "rewards/margins": 0.8891077041625977,
      "rewards/rejected": -2.073837995529175,
      "step": 15041
    },
    {
      "epoch": 1.9683487364030423,
      "grad_norm": 9.723590850830078,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3478092551231384,
      "logits/rejected": -0.6618890166282654,
      "logps/chosen": -539.3265380859375,
      "logps/rejected": -420.34393310546875,
      "loss": 0.4021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.173307418823242,
      "rewards/margins": 1.8221282958984375,
      "rewards/rejected": -3.995436191558838,
      "step": 15042
    },
    {
      "epoch": 1.968479594340394,
      "grad_norm": 10.716952323913574,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3447037637233734,
      "logits/rejected": -0.6211097240447998,
      "logps/chosen": -596.46826171875,
      "logps/rejected": -490.27423095703125,
      "loss": 0.6073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.784911036491394,
      "rewards/margins": 1.336477518081665,
      "rewards/rejected": -3.1213889122009277,
      "step": 15043
    },
    {
      "epoch": 1.968610452277746,
      "grad_norm": 9.473454475402832,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16374963521957397,
      "logits/rejected": -0.030507013201713562,
      "logps/chosen": -504.86395263671875,
      "logps/rejected": -449.8539733886719,
      "loss": 0.3229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.427216649055481,
      "rewards/margins": 1.5424703359603882,
      "rewards/rejected": -2.96968674659729,
      "step": 15044
    },
    {
      "epoch": 1.9687413102150977,
      "grad_norm": 10.47642707824707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3003084063529968,
      "logits/rejected": -0.21357563138008118,
      "logps/chosen": -401.12255859375,
      "logps/rejected": -395.5020446777344,
      "loss": 0.483,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4039762020111084,
      "rewards/margins": 1.6517307758331299,
      "rewards/rejected": -3.0557069778442383,
      "step": 15045
    },
    {
      "epoch": 1.9688721681524495,
      "grad_norm": 13.395855903625488,
      "learning_rate": 5e-06,
      "logits/chosen": 0.10560904443264008,
      "logits/rejected": -0.07488828897476196,
      "logps/chosen": -651.6539916992188,
      "logps/rejected": -573.275634765625,
      "loss": 0.8725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.677247405052185,
      "rewards/margins": 1.554642677307129,
      "rewards/rejected": -3.2318904399871826,
      "step": 15046
    },
    {
      "epoch": 1.9690030260898013,
      "grad_norm": 9.692972183227539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11550848931074142,
      "logits/rejected": -0.35540610551834106,
      "logps/chosen": -409.7517395019531,
      "logps/rejected": -343.82635498046875,
      "loss": 0.4188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6170142889022827,
      "rewards/margins": 1.2477593421936035,
      "rewards/rejected": -2.864773750305176,
      "step": 15047
    },
    {
      "epoch": 1.969133884027153,
      "grad_norm": 5.695035457611084,
      "learning_rate": 5e-06,
      "logits/chosen": -0.42692553997039795,
      "logits/rejected": -0.35702213644981384,
      "logps/chosen": -563.0316772460938,
      "logps/rejected": -386.2729187011719,
      "loss": 0.2308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9737032651901245,
      "rewards/margins": 1.8862308263778687,
      "rewards/rejected": -3.859934091567993,
      "step": 15048
    },
    {
      "epoch": 1.9692647419645049,
      "grad_norm": 13.560201644897461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07112019509077072,
      "logits/rejected": -0.3913760781288147,
      "logps/chosen": -660.2525634765625,
      "logps/rejected": -398.7520751953125,
      "loss": 0.6017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9394689798355103,
      "rewards/margins": 1.942474365234375,
      "rewards/rejected": -2.8819429874420166,
      "step": 15049
    },
    {
      "epoch": 1.9693955999018566,
      "grad_norm": 7.454804420471191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2586978077888489,
      "logits/rejected": -0.049869999289512634,
      "logps/chosen": -339.51116943359375,
      "logps/rejected": -472.1329040527344,
      "loss": 0.2713,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7073441743850708,
      "rewards/margins": 2.1349313259124756,
      "rewards/rejected": -3.842275619506836,
      "step": 15050
    },
    {
      "epoch": 1.9695264578392084,
      "grad_norm": 4.62084436416626,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09977494925260544,
      "logits/rejected": -0.42411673069000244,
      "logps/chosen": -398.81182861328125,
      "logps/rejected": -439.81964111328125,
      "loss": 0.2043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8718278408050537,
      "rewards/margins": 2.287606954574585,
      "rewards/rejected": -3.1594350337982178,
      "step": 15051
    },
    {
      "epoch": 1.9696573157765602,
      "grad_norm": 7.329216957092285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.176526740193367,
      "logits/rejected": -0.11381284147500992,
      "logps/chosen": -328.25860595703125,
      "logps/rejected": -325.50360107421875,
      "loss": 0.3358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1764570474624634,
      "rewards/margins": 1.3758774995803833,
      "rewards/rejected": -2.552334785461426,
      "step": 15052
    },
    {
      "epoch": 1.9697881737139118,
      "grad_norm": 12.199517250061035,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29293134808540344,
      "logits/rejected": -0.2733803987503052,
      "logps/chosen": -444.3673095703125,
      "logps/rejected": -478.3478088378906,
      "loss": 0.633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7588223218917847,
      "rewards/margins": 0.6833392381668091,
      "rewards/rejected": -2.4421615600585938,
      "step": 15053
    },
    {
      "epoch": 1.9699190316512636,
      "grad_norm": 4.389708518981934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3879268765449524,
      "logits/rejected": -0.33556780219078064,
      "logps/chosen": -288.0012512207031,
      "logps/rejected": -314.298583984375,
      "loss": 0.2078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3101792335510254,
      "rewards/margins": 2.631605386734009,
      "rewards/rejected": -3.941784620285034,
      "step": 15054
    },
    {
      "epoch": 1.9700498895886154,
      "grad_norm": 14.370927810668945,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6366320848464966,
      "logits/rejected": -0.4210277199745178,
      "logps/chosen": -391.6700439453125,
      "logps/rejected": -402.0772705078125,
      "loss": 0.7897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8344887495040894,
      "rewards/margins": 0.3662906587123871,
      "rewards/rejected": -2.200779438018799,
      "step": 15055
    },
    {
      "epoch": 1.9701807475259672,
      "grad_norm": 8.29493522644043,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17947539687156677,
      "logits/rejected": -0.39103350043296814,
      "logps/chosen": -458.1124267578125,
      "logps/rejected": -505.01177978515625,
      "loss": 0.3603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0676933526992798,
      "rewards/margins": 2.1966593265533447,
      "rewards/rejected": -3.264352798461914,
      "step": 15056
    },
    {
      "epoch": 1.9703116054633187,
      "grad_norm": 3.675185203552246,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043603312224149704,
      "logits/rejected": -0.2429550588130951,
      "logps/chosen": -467.9433898925781,
      "logps/rejected": -569.6728515625,
      "loss": 0.2028,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9398720264434814,
      "rewards/margins": 2.467036724090576,
      "rewards/rejected": -3.4069087505340576,
      "step": 15057
    },
    {
      "epoch": 1.9704424634006705,
      "grad_norm": 8.32642936706543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09454428404569626,
      "logits/rejected": -0.18465672433376312,
      "logps/chosen": -496.2402038574219,
      "logps/rejected": -445.2873840332031,
      "loss": 0.3346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.211093544960022,
      "rewards/margins": 1.5179469585418701,
      "rewards/rejected": -2.7290406227111816,
      "step": 15058
    },
    {
      "epoch": 1.9705733213380223,
      "grad_norm": 9.746182441711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3360147774219513,
      "logits/rejected": -0.30849409103393555,
      "logps/chosen": -467.7698974609375,
      "logps/rejected": -416.70880126953125,
      "loss": 0.4351,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3781075477600098,
      "rewards/margins": 1.2023404836654663,
      "rewards/rejected": -2.5804479122161865,
      "step": 15059
    },
    {
      "epoch": 1.9707041792753741,
      "grad_norm": 14.22360610961914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3672809302806854,
      "logits/rejected": -0.35162198543548584,
      "logps/chosen": -553.6949462890625,
      "logps/rejected": -441.1895446777344,
      "loss": 0.8758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.110722780227661,
      "rewards/margins": 0.0028583109378814697,
      "rewards/rejected": -2.1135811805725098,
      "step": 15060
    },
    {
      "epoch": 1.970835037212726,
      "grad_norm": 7.866913795471191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.38301870226860046,
      "logits/rejected": -0.17190569639205933,
      "logps/chosen": -389.9533996582031,
      "logps/rejected": -451.198974609375,
      "loss": 0.3551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1596096754074097,
      "rewards/margins": 1.3756088018417358,
      "rewards/rejected": -2.5352184772491455,
      "step": 15061
    },
    {
      "epoch": 1.9709658951500777,
      "grad_norm": 6.259496212005615,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3648948073387146,
      "logits/rejected": -0.2950606048107147,
      "logps/chosen": -506.473388671875,
      "logps/rejected": -480.35308837890625,
      "loss": 0.2104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1110528707504272,
      "rewards/margins": 1.6665470600128174,
      "rewards/rejected": -2.777600049972534,
      "step": 15062
    },
    {
      "epoch": 1.9710967530874295,
      "grad_norm": 12.463789939880371,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21759624779224396,
      "logits/rejected": -0.385995090007782,
      "logps/chosen": -471.23797607421875,
      "logps/rejected": -485.12469482421875,
      "loss": 0.6595,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5161631107330322,
      "rewards/margins": 0.9628482460975647,
      "rewards/rejected": -1.4790115356445312,
      "step": 15063
    },
    {
      "epoch": 1.9712276110247813,
      "grad_norm": 19.56159019470215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16690465807914734,
      "logits/rejected": -0.231347918510437,
      "logps/chosen": -429.494140625,
      "logps/rejected": -468.17529296875,
      "loss": 0.6739,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0038785934448242,
      "rewards/margins": 1.9450501203536987,
      "rewards/rejected": -2.9489288330078125,
      "step": 15064
    },
    {
      "epoch": 1.971358468962133,
      "grad_norm": 9.005623817443848,
      "learning_rate": 5e-06,
      "logits/chosen": 0.09788340330123901,
      "logits/rejected": 0.019945040345191956,
      "logps/chosen": -643.927734375,
      "logps/rejected": -491.3659362792969,
      "loss": 0.3172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2552731037139893,
      "rewards/margins": 1.78116774559021,
      "rewards/rejected": -4.036440372467041,
      "step": 15065
    },
    {
      "epoch": 1.9714893268994849,
      "grad_norm": 6.566150665283203,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08040536195039749,
      "logits/rejected": 0.055247969925403595,
      "logps/chosen": -589.5227661132812,
      "logps/rejected": -653.5468139648438,
      "loss": 0.2395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12998296320438385,
      "rewards/margins": 2.6328651905059814,
      "rewards/rejected": -2.762848377227783,
      "step": 15066
    },
    {
      "epoch": 1.9716201848368367,
      "grad_norm": 9.717198371887207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3352653980255127,
      "logits/rejected": -0.35566529631614685,
      "logps/chosen": -627.59765625,
      "logps/rejected": -640.2631225585938,
      "loss": 0.2741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8431000113487244,
      "rewards/margins": 2.4462239742279053,
      "rewards/rejected": -3.2893240451812744,
      "step": 15067
    },
    {
      "epoch": 1.9717510427741882,
      "grad_norm": 7.706337928771973,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12786661088466644,
      "logits/rejected": -0.15752868354320526,
      "logps/chosen": -423.2177429199219,
      "logps/rejected": -421.2794189453125,
      "loss": 0.4165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.068159818649292,
      "rewards/margins": 1.3830530643463135,
      "rewards/rejected": -3.4512128829956055,
      "step": 15068
    },
    {
      "epoch": 1.97188190071154,
      "grad_norm": 7.129379749298096,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5763910412788391,
      "logits/rejected": -0.6282821893692017,
      "logps/chosen": -384.4595947265625,
      "logps/rejected": -354.99658203125,
      "loss": 0.3044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5736500024795532,
      "rewards/margins": 2.131476402282715,
      "rewards/rejected": -3.7051262855529785,
      "step": 15069
    },
    {
      "epoch": 1.9720127586488918,
      "grad_norm": 16.456130981445312,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2876249849796295,
      "logits/rejected": -0.2133692055940628,
      "logps/chosen": -707.6835327148438,
      "logps/rejected": -637.9856567382812,
      "loss": 0.7664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0490269660949707,
      "rewards/margins": 0.548737645149231,
      "rewards/rejected": -2.597764730453491,
      "step": 15070
    },
    {
      "epoch": 1.9721436165862436,
      "grad_norm": 5.9740376472473145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23640888929367065,
      "logits/rejected": -0.26749223470687866,
      "logps/chosen": -599.100341796875,
      "logps/rejected": -402.558837890625,
      "loss": 0.2845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9348398447036743,
      "rewards/margins": 1.641674518585205,
      "rewards/rejected": -2.57651424407959,
      "step": 15071
    },
    {
      "epoch": 1.9722744745235952,
      "grad_norm": 11.944276809692383,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2827105224132538,
      "logits/rejected": -0.08526608347892761,
      "logps/chosen": -481.26336669921875,
      "logps/rejected": -449.1910400390625,
      "loss": 0.6252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8649078607559204,
      "rewards/margins": 0.5201930999755859,
      "rewards/rejected": -1.3851008415222168,
      "step": 15072
    },
    {
      "epoch": 1.972405332460947,
      "grad_norm": 11.958701133728027,
      "learning_rate": 5e-06,
      "logits/chosen": -0.02051587402820587,
      "logits/rejected": -0.09060385823249817,
      "logps/chosen": -517.4091186523438,
      "logps/rejected": -500.6258239746094,
      "loss": 0.7934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.75105220079422,
      "rewards/margins": 0.4331144690513611,
      "rewards/rejected": -1.184166669845581,
      "step": 15073
    },
    {
      "epoch": 1.9725361903982987,
      "grad_norm": 9.39509105682373,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18828827142715454,
      "logits/rejected": -0.03125850856304169,
      "logps/chosen": -568.2896728515625,
      "logps/rejected": -471.5779113769531,
      "loss": 0.4702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8777389526367188,
      "rewards/margins": 1.182135820388794,
      "rewards/rejected": -2.0598747730255127,
      "step": 15074
    },
    {
      "epoch": 1.9726670483356505,
      "grad_norm": 12.524658203125,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1142229288816452,
      "logits/rejected": 0.10063131153583527,
      "logps/chosen": -426.3658447265625,
      "logps/rejected": -376.7154235839844,
      "loss": 0.7474,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3858585357666016,
      "rewards/margins": 0.16127733886241913,
      "rewards/rejected": -1.5471359491348267,
      "step": 15075
    },
    {
      "epoch": 1.9727979062730023,
      "grad_norm": 14.497385025024414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40137413144111633,
      "logits/rejected": -0.46196141839027405,
      "logps/chosen": -521.05810546875,
      "logps/rejected": -426.0166015625,
      "loss": 0.7501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4059464931488037,
      "rewards/margins": 0.5587629675865173,
      "rewards/rejected": -2.9647092819213867,
      "step": 15076
    },
    {
      "epoch": 1.9729287642103541,
      "grad_norm": 10.092249870300293,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2323395311832428,
      "logits/rejected": -0.4021531939506531,
      "logps/chosen": -593.1021118164062,
      "logps/rejected": -464.9378662109375,
      "loss": 0.3504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4199870824813843,
      "rewards/margins": 1.9311649799346924,
      "rewards/rejected": -2.351151943206787,
      "step": 15077
    },
    {
      "epoch": 1.973059622147706,
      "grad_norm": 18.473989486694336,
      "learning_rate": 5e-06,
      "logits/chosen": -0.34965434670448303,
      "logits/rejected": -0.27472826838493347,
      "logps/chosen": -411.3421630859375,
      "logps/rejected": -398.0644836425781,
      "loss": 0.9728,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.076970338821411,
      "rewards/margins": 1.5586047172546387,
      "rewards/rejected": -3.6355748176574707,
      "step": 15078
    },
    {
      "epoch": 1.9731904800850577,
      "grad_norm": 10.6981782913208,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37724584341049194,
      "logits/rejected": -0.4317355155944824,
      "logps/chosen": -434.15283203125,
      "logps/rejected": -451.5604248046875,
      "loss": 0.3975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2614742517471313,
      "rewards/margins": 1.7415804862976074,
      "rewards/rejected": -3.0030548572540283,
      "step": 15079
    },
    {
      "epoch": 1.9733213380224095,
      "grad_norm": 12.017498970031738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.33305856585502625,
      "logits/rejected": -0.2940811812877655,
      "logps/chosen": -509.4266662597656,
      "logps/rejected": -477.13543701171875,
      "loss": 0.7049,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.908581018447876,
      "rewards/margins": 0.8918428421020508,
      "rewards/rejected": -1.8004238605499268,
      "step": 15080
    },
    {
      "epoch": 1.9734521959597613,
      "grad_norm": 12.008593559265137,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16937968134880066,
      "logits/rejected": -0.28353172540664673,
      "logps/chosen": -472.06640625,
      "logps/rejected": -347.8726806640625,
      "loss": 0.5277,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9222223162651062,
      "rewards/margins": 1.856439471244812,
      "rewards/rejected": -2.7786617279052734,
      "step": 15081
    },
    {
      "epoch": 1.973583053897113,
      "grad_norm": 10.024243354797363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3172522187232971,
      "logits/rejected": -0.27297353744506836,
      "logps/chosen": -441.1108703613281,
      "logps/rejected": -399.4661865234375,
      "loss": 0.6513,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2946226596832275,
      "rewards/margins": 1.1394511461257935,
      "rewards/rejected": -2.4340739250183105,
      "step": 15082
    },
    {
      "epoch": 1.9737139118344649,
      "grad_norm": 11.854406356811523,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6637029051780701,
      "logits/rejected": -0.44766855239868164,
      "logps/chosen": -378.466552734375,
      "logps/rejected": -462.00518798828125,
      "loss": 0.6011,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8264203667640686,
      "rewards/margins": 0.5822307467460632,
      "rewards/rejected": -1.4086511135101318,
      "step": 15083
    },
    {
      "epoch": 1.9738447697718164,
      "grad_norm": 9.956097602844238,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26078686118125916,
      "logits/rejected": -0.5283381938934326,
      "logps/chosen": -869.1904296875,
      "logps/rejected": -500.16680908203125,
      "loss": 0.3479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47894150018692017,
      "rewards/margins": 2.310582399368286,
      "rewards/rejected": -2.7895240783691406,
      "step": 15084
    },
    {
      "epoch": 1.9739756277091682,
      "grad_norm": 8.442280769348145,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3730586767196655,
      "logits/rejected": -0.4216982424259186,
      "logps/chosen": -590.8139038085938,
      "logps/rejected": -512.1222534179688,
      "loss": 0.4265,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5953048467636108,
      "rewards/margins": 1.9067103862762451,
      "rewards/rejected": -3.5020153522491455,
      "step": 15085
    },
    {
      "epoch": 1.97410648564652,
      "grad_norm": 9.291979789733887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2691865861415863,
      "logits/rejected": -0.2999309003353119,
      "logps/chosen": -411.95257568359375,
      "logps/rejected": -430.7072448730469,
      "loss": 0.5401,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7601966857910156,
      "rewards/margins": 1.9500422477722168,
      "rewards/rejected": -2.7102391719818115,
      "step": 15086
    },
    {
      "epoch": 1.9742373435838716,
      "grad_norm": 4.524418354034424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16817909479141235,
      "logits/rejected": -0.37853917479515076,
      "logps/chosen": -526.6134643554688,
      "logps/rejected": -410.288330078125,
      "loss": 0.2066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6101252436637878,
      "rewards/margins": 2.553344964981079,
      "rewards/rejected": -3.1634702682495117,
      "step": 15087
    },
    {
      "epoch": 1.9743682015212234,
      "grad_norm": 4.722489356994629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2974786162376404,
      "logits/rejected": -0.2632826864719391,
      "logps/chosen": -553.97900390625,
      "logps/rejected": -492.7840881347656,
      "loss": 0.1927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1666850745677948,
      "rewards/margins": 2.1372008323669434,
      "rewards/rejected": -2.3038859367370605,
      "step": 15088
    },
    {
      "epoch": 1.9744990594585752,
      "grad_norm": 6.428377151489258,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7189263701438904,
      "logits/rejected": -0.6786010265350342,
      "logps/chosen": -298.8432922363281,
      "logps/rejected": -404.4100036621094,
      "loss": 0.2231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2763731479644775,
      "rewards/margins": 3.2122013568878174,
      "rewards/rejected": -4.488574504852295,
      "step": 15089
    },
    {
      "epoch": 1.974629917395927,
      "grad_norm": 7.051309585571289,
      "learning_rate": 5e-06,
      "logits/chosen": 0.20193016529083252,
      "logits/rejected": -0.25394976139068604,
      "logps/chosen": -530.4718017578125,
      "logps/rejected": -570.40185546875,
      "loss": 0.2312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.246251106262207,
      "rewards/margins": 1.887235403060913,
      "rewards/rejected": -3.13348650932312,
      "step": 15090
    },
    {
      "epoch": 1.9747607753332788,
      "grad_norm": 10.869645118713379,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39679646492004395,
      "logits/rejected": -0.4392486810684204,
      "logps/chosen": -471.46844482421875,
      "logps/rejected": -264.33929443359375,
      "loss": 0.8562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.2055561542510986,
      "rewards/margins": 0.2823878526687622,
      "rewards/rejected": -2.4879438877105713,
      "step": 15091
    },
    {
      "epoch": 1.9748916332706306,
      "grad_norm": 12.103466033935547,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3068498969078064,
      "logits/rejected": -0.3624746799468994,
      "logps/chosen": -540.211181640625,
      "logps/rejected": -432.02978515625,
      "loss": 0.5557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6456226110458374,
      "rewards/margins": 1.08524489402771,
      "rewards/rejected": -1.7308675050735474,
      "step": 15092
    },
    {
      "epoch": 1.9750224912079823,
      "grad_norm": 8.544577598571777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5127246379852295,
      "logits/rejected": -0.5651035308837891,
      "logps/chosen": -417.60614013671875,
      "logps/rejected": -474.00531005859375,
      "loss": 0.4752,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2586098909378052,
      "rewards/margins": 1.0070325136184692,
      "rewards/rejected": -2.2656424045562744,
      "step": 15093
    },
    {
      "epoch": 1.9751533491453341,
      "grad_norm": 7.714633464813232,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0975557193160057,
      "logits/rejected": -0.15013131499290466,
      "logps/chosen": -328.3999328613281,
      "logps/rejected": -447.37933349609375,
      "loss": 0.3558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8416049480438232,
      "rewards/margins": 1.186122179031372,
      "rewards/rejected": -3.0277271270751953,
      "step": 15094
    },
    {
      "epoch": 1.975284207082686,
      "grad_norm": 10.244085311889648,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20539365708827972,
      "logits/rejected": -0.39802852272987366,
      "logps/chosen": -508.6849060058594,
      "logps/rejected": -377.2830810546875,
      "loss": 0.5102,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.959628701210022,
      "rewards/margins": 1.4366998672485352,
      "rewards/rejected": -2.3963286876678467,
      "step": 15095
    },
    {
      "epoch": 1.9754150650200377,
      "grad_norm": 7.825865268707275,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11416938900947571,
      "logits/rejected": -0.1318800002336502,
      "logps/chosen": -472.0672607421875,
      "logps/rejected": -374.86566162109375,
      "loss": 0.4509,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7756307721138,
      "rewards/margins": 1.6377677917480469,
      "rewards/rejected": -2.413398504257202,
      "step": 15096
    },
    {
      "epoch": 1.9755459229573895,
      "grad_norm": 10.290489196777344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0633254200220108,
      "logits/rejected": -0.21507564187049866,
      "logps/chosen": -724.9888916015625,
      "logps/rejected": -473.6136474609375,
      "loss": 0.4376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2934356927871704,
      "rewards/margins": 1.6914172172546387,
      "rewards/rejected": -2.9848530292510986,
      "step": 15097
    },
    {
      "epoch": 1.9756767808947413,
      "grad_norm": 7.558089733123779,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24436205625534058,
      "logits/rejected": -0.2620537579059601,
      "logps/chosen": -416.44189453125,
      "logps/rejected": -428.1051330566406,
      "loss": 0.4626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10854281485080719,
      "rewards/margins": 1.9207055568695068,
      "rewards/rejected": -1.8121627569198608,
      "step": 15098
    },
    {
      "epoch": 1.9758076388320929,
      "grad_norm": 10.729103088378906,
      "learning_rate": 5e-06,
      "logits/chosen": -0.37777143716812134,
      "logits/rejected": -0.5038474202156067,
      "logps/chosen": -386.9023742675781,
      "logps/rejected": -421.22265625,
      "loss": 0.7546,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0512306690216064,
      "rewards/margins": 0.1907798796892166,
      "rewards/rejected": -2.2420105934143066,
      "step": 15099
    },
    {
      "epoch": 1.9759384967694447,
      "grad_norm": 11.598992347717285,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12032948434352875,
      "logits/rejected": -0.3194081783294678,
      "logps/chosen": -515.8126220703125,
      "logps/rejected": -523.0970458984375,
      "loss": 0.6356,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8145031332969666,
      "rewards/margins": 0.7383049130439758,
      "rewards/rejected": -1.5528080463409424,
      "step": 15100
    },
    {
      "epoch": 1.9760693547067965,
      "grad_norm": 7.402973175048828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0977756679058075,
      "logits/rejected": -0.2880776822566986,
      "logps/chosen": -717.1091918945312,
      "logps/rejected": -504.70709228515625,
      "loss": 0.307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.060390979051589966,
      "rewards/margins": 1.8641151189804077,
      "rewards/rejected": -1.8037244081497192,
      "step": 15101
    },
    {
      "epoch": 1.9762002126441482,
      "grad_norm": 13.405777931213379,
      "learning_rate": 5e-06,
      "logits/chosen": 0.03156164288520813,
      "logits/rejected": -0.12001745402812958,
      "logps/chosen": -518.0172119140625,
      "logps/rejected": -403.42401123046875,
      "loss": 0.7683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3015846014022827,
      "rewards/margins": 0.5480421185493469,
      "rewards/rejected": -1.8496267795562744,
      "step": 15102
    },
    {
      "epoch": 1.9763310705814998,
      "grad_norm": 10.269686698913574,
      "learning_rate": 5e-06,
      "logits/chosen": 0.0029492639005184174,
      "logits/rejected": -0.3833867311477661,
      "logps/chosen": -414.11065673828125,
      "logps/rejected": -321.1412048339844,
      "loss": 0.5211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.038029193878174,
      "rewards/margins": 0.7428900003433228,
      "rewards/rejected": -2.780919313430786,
      "step": 15103
    },
    {
      "epoch": 1.9764619285188516,
      "grad_norm": 6.846656799316406,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1394970715045929,
      "logits/rejected": 0.05844016745686531,
      "logps/chosen": -483.8237609863281,
      "logps/rejected": -532.8480224609375,
      "loss": 0.2824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07239814847707748,
      "rewards/margins": 1.638225793838501,
      "rewards/rejected": -1.7106237411499023,
      "step": 15104
    },
    {
      "epoch": 1.9765927864562034,
      "grad_norm": 4.584986686706543,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20890748500823975,
      "logits/rejected": -0.0613495409488678,
      "logps/chosen": -295.0949401855469,
      "logps/rejected": -384.4985656738281,
      "loss": 0.2069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1564464420080185,
      "rewards/margins": 2.0300445556640625,
      "rewards/rejected": -2.1864912509918213,
      "step": 15105
    },
    {
      "epoch": 1.9767236443935552,
      "grad_norm": 6.643810272216797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39514079689979553,
      "logits/rejected": -0.25694742798805237,
      "logps/chosen": -373.6010437011719,
      "logps/rejected": -294.2391357421875,
      "loss": 0.2535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06736192107200623,
      "rewards/margins": 2.043928623199463,
      "rewards/rejected": -2.111290693283081,
      "step": 15106
    },
    {
      "epoch": 1.976854502330907,
      "grad_norm": 17.402936935424805,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10197749733924866,
      "logits/rejected": -0.19770824909210205,
      "logps/chosen": -605.7671508789062,
      "logps/rejected": -674.34375,
      "loss": 0.7955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.910529375076294,
      "rewards/margins": 0.7769725918769836,
      "rewards/rejected": -2.687501907348633,
      "step": 15107
    },
    {
      "epoch": 1.9769853602682588,
      "grad_norm": 7.09954309463501,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20914074778556824,
      "logits/rejected": -0.2672140598297119,
      "logps/chosen": -346.5487976074219,
      "logps/rejected": -379.3385009765625,
      "loss": 0.3128,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9524770975112915,
      "rewards/margins": 1.673843502998352,
      "rewards/rejected": -2.6263206005096436,
      "step": 15108
    },
    {
      "epoch": 1.9771162182056106,
      "grad_norm": 8.295266151428223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3955311179161072,
      "logits/rejected": -0.5069475173950195,
      "logps/chosen": -365.2857360839844,
      "logps/rejected": -699.8700561523438,
      "loss": 0.2743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7497228384017944,
      "rewards/margins": 1.8543119430541992,
      "rewards/rejected": -2.604034900665283,
      "step": 15109
    },
    {
      "epoch": 1.9772470761429624,
      "grad_norm": 5.925879955291748,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2058815062046051,
      "logits/rejected": -0.4357565641403198,
      "logps/chosen": -589.3426513671875,
      "logps/rejected": -457.8756408691406,
      "loss": 0.2886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4936324954032898,
      "rewards/margins": 2.3232078552246094,
      "rewards/rejected": -2.816840410232544,
      "step": 15110
    },
    {
      "epoch": 1.9773779340803141,
      "grad_norm": 4.713373184204102,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2704278826713562,
      "logits/rejected": -0.2061547487974167,
      "logps/chosen": -455.58599853515625,
      "logps/rejected": -454.95709228515625,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11826041340827942,
      "rewards/margins": 1.1984350681304932,
      "rewards/rejected": -1.0801746845245361,
      "step": 15111
    },
    {
      "epoch": 1.977508792017666,
      "grad_norm": 7.419996738433838,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45830291509628296,
      "logits/rejected": -0.3027418851852417,
      "logps/chosen": -434.91802978515625,
      "logps/rejected": -414.1582946777344,
      "loss": 0.32,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6680935621261597,
      "rewards/margins": 1.492991328239441,
      "rewards/rejected": -2.1610846519470215,
      "step": 15112
    },
    {
      "epoch": 1.9776396499550177,
      "grad_norm": 9.28899097442627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6938391923904419,
      "logits/rejected": -0.4832557737827301,
      "logps/chosen": -650.1914672851562,
      "logps/rejected": -774.3450317382812,
      "loss": 0.4108,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2686344385147095,
      "rewards/margins": 1.5602868795394897,
      "rewards/rejected": -1.8289211988449097,
      "step": 15113
    },
    {
      "epoch": 1.9777705078923693,
      "grad_norm": 10.489073753356934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25091752409935,
      "logits/rejected": -0.3793368935585022,
      "logps/chosen": -435.68603515625,
      "logps/rejected": -384.36163330078125,
      "loss": 0.4367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7008880972862244,
      "rewards/margins": 1.2567232847213745,
      "rewards/rejected": -1.957611322402954,
      "step": 15114
    },
    {
      "epoch": 1.977901365829721,
      "grad_norm": 8.72066879272461,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5673706531524658,
      "logits/rejected": -0.2580929696559906,
      "logps/chosen": -508.9746398925781,
      "logps/rejected": -442.513427734375,
      "loss": 0.375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27315181493759155,
      "rewards/margins": 1.883885145187378,
      "rewards/rejected": -2.1570370197296143,
      "step": 15115
    },
    {
      "epoch": 1.9780322237670729,
      "grad_norm": 5.309328556060791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.14823530614376068,
      "logits/rejected": -0.2995103895664215,
      "logps/chosen": -441.0354309082031,
      "logps/rejected": -370.81756591796875,
      "loss": 0.2673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1289758682250977,
      "rewards/margins": 1.9930791854858398,
      "rewards/rejected": -3.1220550537109375,
      "step": 15116
    },
    {
      "epoch": 1.9781630817044247,
      "grad_norm": 12.1701021194458,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12487448751926422,
      "logits/rejected": -0.12799718976020813,
      "logps/chosen": -548.2071533203125,
      "logps/rejected": -484.83721923828125,
      "loss": 0.4883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5067955255508423,
      "rewards/margins": 0.9429324865341187,
      "rewards/rejected": -2.449728012084961,
      "step": 15117
    },
    {
      "epoch": 1.9782939396417762,
      "grad_norm": 5.446597099304199,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3537651002407074,
      "logits/rejected": -0.13544167578220367,
      "logps/chosen": -321.1359558105469,
      "logps/rejected": -443.75628662109375,
      "loss": 0.2971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4289669096469879,
      "rewards/margins": 1.3247809410095215,
      "rewards/rejected": -1.7537477016448975,
      "step": 15118
    },
    {
      "epoch": 1.978424797579128,
      "grad_norm": 12.794876098632812,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35031044483184814,
      "logits/rejected": -0.5383659601211548,
      "logps/chosen": -616.7677612304688,
      "logps/rejected": -742.7401123046875,
      "loss": 0.6888,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6111693382263184,
      "rewards/margins": 1.1055182218551636,
      "rewards/rejected": -2.7166876792907715,
      "step": 15119
    },
    {
      "epoch": 1.9785556555164798,
      "grad_norm": 8.243380546569824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15973566472530365,
      "logits/rejected": -0.2164686769247055,
      "logps/chosen": -389.4155578613281,
      "logps/rejected": -417.1671447753906,
      "loss": 0.4385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5091817378997803,
      "rewards/margins": 1.6459378004074097,
      "rewards/rejected": -2.1551196575164795,
      "step": 15120
    },
    {
      "epoch": 1.9786865134538316,
      "grad_norm": 6.667873382568359,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10894910246133804,
      "logits/rejected": -0.18572218716144562,
      "logps/chosen": -740.1651000976562,
      "logps/rejected": -487.1257629394531,
      "loss": 0.1985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3513997793197632,
      "rewards/margins": 2.8574154376983643,
      "rewards/rejected": -3.208815097808838,
      "step": 15121
    },
    {
      "epoch": 1.9788173713911834,
      "grad_norm": 9.595954895019531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3985394239425659,
      "logits/rejected": -0.43616271018981934,
      "logps/chosen": -344.1475830078125,
      "logps/rejected": -402.7303771972656,
      "loss": 0.4974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1410694122314453,
      "rewards/margins": 1.0982743501663208,
      "rewards/rejected": -2.2393438816070557,
      "step": 15122
    },
    {
      "epoch": 1.9789482293285352,
      "grad_norm": 15.54648208618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06976519525051117,
      "logits/rejected": -0.2635667324066162,
      "logps/chosen": -524.2216796875,
      "logps/rejected": -434.9561462402344,
      "loss": 1.066,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.52901691198349,
      "rewards/margins": 1.1059552431106567,
      "rewards/rejected": -1.6349722146987915,
      "step": 15123
    },
    {
      "epoch": 1.979079087265887,
      "grad_norm": 8.78850269317627,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3287350833415985,
      "logits/rejected": -0.44514358043670654,
      "logps/chosen": -450.3027648925781,
      "logps/rejected": -535.24609375,
      "loss": 0.5624,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7390764951705933,
      "rewards/margins": 1.3751813173294067,
      "rewards/rejected": -2.1142578125,
      "step": 15124
    },
    {
      "epoch": 1.9792099452032388,
      "grad_norm": 6.542456150054932,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16056522727012634,
      "logits/rejected": -0.1175604909658432,
      "logps/chosen": -403.4913635253906,
      "logps/rejected": -512.2808837890625,
      "loss": 0.2786,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04006289690732956,
      "rewards/margins": 1.8025799989700317,
      "rewards/rejected": -1.762516975402832,
      "step": 15125
    },
    {
      "epoch": 1.9793408031405906,
      "grad_norm": 7.118399620056152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.00684814527630806,
      "logits/rejected": -0.11191628873348236,
      "logps/chosen": -421.53192138671875,
      "logps/rejected": -443.19580078125,
      "loss": 0.3122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.007690668106079,
      "rewards/margins": 1.545856237411499,
      "rewards/rejected": -2.553546905517578,
      "step": 15126
    },
    {
      "epoch": 1.9794716610779424,
      "grad_norm": 5.44471549987793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11608827114105225,
      "logits/rejected": -0.3079693615436554,
      "logps/chosen": -604.7664184570312,
      "logps/rejected": -612.7125854492188,
      "loss": 0.1494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14136943221092224,
      "rewards/margins": 3.2981061935424805,
      "rewards/rejected": -3.4394755363464355,
      "step": 15127
    },
    {
      "epoch": 1.9796025190152942,
      "grad_norm": 10.210297584533691,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3811824917793274,
      "logits/rejected": -0.30616647005081177,
      "logps/chosen": -294.1031494140625,
      "logps/rejected": -241.53042602539062,
      "loss": 0.7154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7949705123901367,
      "rewards/margins": 0.5793720483779907,
      "rewards/rejected": -2.374342441558838,
      "step": 15128
    },
    {
      "epoch": 1.9797333769526457,
      "grad_norm": 10.843867301940918,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3887183368206024,
      "logits/rejected": -0.28334054350852966,
      "logps/chosen": -471.43109130859375,
      "logps/rejected": -443.473388671875,
      "loss": 0.4235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8274284601211548,
      "rewards/margins": 1.4734535217285156,
      "rewards/rejected": -3.30088210105896,
      "step": 15129
    },
    {
      "epoch": 1.9798642348899975,
      "grad_norm": 8.180621147155762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3305344879627228,
      "logits/rejected": -0.3886094391345978,
      "logps/chosen": -479.16168212890625,
      "logps/rejected": -416.3088073730469,
      "loss": 0.4006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3962986171245575,
      "rewards/margins": 1.0579609870910645,
      "rewards/rejected": -1.4542593955993652,
      "step": 15130
    },
    {
      "epoch": 1.9799950928273493,
      "grad_norm": 13.38559341430664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.005975538399070501,
      "logits/rejected": -0.22071431577205658,
      "logps/chosen": -555.8153686523438,
      "logps/rejected": -507.4226989746094,
      "loss": 0.6899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3177399635314941,
      "rewards/margins": 0.3712049126625061,
      "rewards/rejected": -1.6889448165893555,
      "step": 15131
    },
    {
      "epoch": 1.980125950764701,
      "grad_norm": 8.49790096282959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4574030637741089,
      "logits/rejected": -0.3504343330860138,
      "logps/chosen": -353.61572265625,
      "logps/rejected": -332.32843017578125,
      "loss": 0.4026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2973250150680542,
      "rewards/margins": 1.5553462505340576,
      "rewards/rejected": -2.8526713848114014,
      "step": 15132
    },
    {
      "epoch": 1.9802568087020527,
      "grad_norm": 7.610535144805908,
      "learning_rate": 5e-06,
      "logits/chosen": -0.070151187479496,
      "logits/rejected": -0.06903770565986633,
      "logps/chosen": -421.59405517578125,
      "logps/rejected": -420.6774597167969,
      "loss": 0.4114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.365283727645874,
      "rewards/margins": 1.194671630859375,
      "rewards/rejected": -1.5599554777145386,
      "step": 15133
    },
    {
      "epoch": 1.9803876666394045,
      "grad_norm": 10.750375747680664,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3917221128940582,
      "logits/rejected": -0.19594454765319824,
      "logps/chosen": -355.14825439453125,
      "logps/rejected": -436.50482177734375,
      "loss": 0.6917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1197227239608765,
      "rewards/margins": 0.1464233696460724,
      "rewards/rejected": -1.2661460638046265,
      "step": 15134
    },
    {
      "epoch": 1.9805185245767563,
      "grad_norm": 15.55014705657959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.257109671831131,
      "logits/rejected": -0.09217971563339233,
      "logps/chosen": -532.2656860351562,
      "logps/rejected": -533.6159057617188,
      "loss": 1.0717,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2557182312011719,
      "rewards/margins": -0.14163419604301453,
      "rewards/rejected": -1.1140841245651245,
      "step": 15135
    },
    {
      "epoch": 1.980649382514108,
      "grad_norm": 12.484435081481934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2044733762741089,
      "logits/rejected": -0.22214125096797943,
      "logps/chosen": -427.00921630859375,
      "logps/rejected": -294.337158203125,
      "loss": 0.5177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9672129154205322,
      "rewards/margins": 0.8580926656723022,
      "rewards/rejected": -1.825305700302124,
      "step": 15136
    },
    {
      "epoch": 1.9807802404514598,
      "grad_norm": 8.12629508972168,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3304281234741211,
      "logits/rejected": -0.1909673660993576,
      "logps/chosen": -430.03582763671875,
      "logps/rejected": -518.8557739257812,
      "loss": 0.428,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8012475967407227,
      "rewards/margins": 2.506826400756836,
      "rewards/rejected": -3.3080735206604004,
      "step": 15137
    },
    {
      "epoch": 1.9809110983888116,
      "grad_norm": 3.6745529174804688,
      "learning_rate": 5e-06,
      "logits/chosen": 0.19603893160820007,
      "logits/rejected": -0.06434997916221619,
      "logps/chosen": -466.6585693359375,
      "logps/rejected": -479.9673767089844,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19852828979492188,
      "rewards/margins": 2.43094801902771,
      "rewards/rejected": -2.629476308822632,
      "step": 15138
    },
    {
      "epoch": 1.9810419563261634,
      "grad_norm": 6.396395206451416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.053412362933158875,
      "logits/rejected": -0.16979923844337463,
      "logps/chosen": -478.9524841308594,
      "logps/rejected": -449.62109375,
      "loss": 0.3057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23840181529521942,
      "rewards/margins": 1.8688910007476807,
      "rewards/rejected": -2.107292890548706,
      "step": 15139
    },
    {
      "epoch": 1.9811728142635152,
      "grad_norm": 8.28161334991455,
      "learning_rate": 5e-06,
      "logits/chosen": 0.015171598643064499,
      "logits/rejected": -0.2594068944454193,
      "logps/chosen": -394.6964111328125,
      "logps/rejected": -366.6665954589844,
      "loss": 0.5832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7212910056114197,
      "rewards/margins": 0.897919237613678,
      "rewards/rejected": -1.6192103624343872,
      "step": 15140
    },
    {
      "epoch": 1.981303672200867,
      "grad_norm": 4.1841583251953125,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41242295503616333,
      "logits/rejected": -0.3588383197784424,
      "logps/chosen": -587.361083984375,
      "logps/rejected": -425.4057922363281,
      "loss": 0.1937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10487422347068787,
      "rewards/margins": 2.9344589710235596,
      "rewards/rejected": -3.0393331050872803,
      "step": 15141
    },
    {
      "epoch": 1.9814345301382188,
      "grad_norm": 10.11219596862793,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1268223077058792,
      "logits/rejected": -0.0005918368697166443,
      "logps/chosen": -308.9012451171875,
      "logps/rejected": -320.5735778808594,
      "loss": 0.4235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6040668487548828,
      "rewards/margins": 1.8239059448242188,
      "rewards/rejected": -2.4279730319976807,
      "step": 15142
    },
    {
      "epoch": 1.9815653880755706,
      "grad_norm": 7.020894527435303,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09932485222816467,
      "logits/rejected": -0.13593998551368713,
      "logps/chosen": -632.7725830078125,
      "logps/rejected": -431.5401306152344,
      "loss": 0.3433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15547654032707214,
      "rewards/margins": 1.8883512020111084,
      "rewards/rejected": -2.043827772140503,
      "step": 15143
    },
    {
      "epoch": 1.9816962460129224,
      "grad_norm": 8.585853576660156,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3003940284252167,
      "logits/rejected": -0.21215534210205078,
      "logps/chosen": -305.9607849121094,
      "logps/rejected": -436.85357666015625,
      "loss": 0.4569,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8802503943443298,
      "rewards/margins": 1.2508013248443604,
      "rewards/rejected": -2.131051778793335,
      "step": 15144
    },
    {
      "epoch": 1.981827103950274,
      "grad_norm": 8.050761222839355,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29962342977523804,
      "logits/rejected": -0.572013258934021,
      "logps/chosen": -467.1396179199219,
      "logps/rejected": -554.9567260742188,
      "loss": 0.3973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9782871007919312,
      "rewards/margins": 2.261686325073242,
      "rewards/rejected": -3.2399730682373047,
      "step": 15145
    },
    {
      "epoch": 1.9819579618876257,
      "grad_norm": 5.471523284912109,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10606878250837326,
      "logits/rejected": -0.32674580812454224,
      "logps/chosen": -486.8666687011719,
      "logps/rejected": -402.3382568359375,
      "loss": 0.2155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29944488406181335,
      "rewards/margins": 2.049456834793091,
      "rewards/rejected": -2.3489017486572266,
      "step": 15146
    },
    {
      "epoch": 1.9820888198249775,
      "grad_norm": 11.924201965332031,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3682367205619812,
      "logits/rejected": -0.33167535066604614,
      "logps/chosen": -518.8765869140625,
      "logps/rejected": -497.2878723144531,
      "loss": 0.6077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.622058093547821,
      "rewards/margins": 0.4983903765678406,
      "rewards/rejected": -1.1204485893249512,
      "step": 15147
    },
    {
      "epoch": 1.9822196777623293,
      "grad_norm": 10.019792556762695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.39404186606407166,
      "logits/rejected": -0.3270779550075531,
      "logps/chosen": -393.11822509765625,
      "logps/rejected": -367.4466247558594,
      "loss": 0.4742,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4480559825897217,
      "rewards/margins": 1.550870656967163,
      "rewards/rejected": -2.9989266395568848,
      "step": 15148
    },
    {
      "epoch": 1.982350535699681,
      "grad_norm": 10.158517837524414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.09189081937074661,
      "logits/rejected": -0.16865089535713196,
      "logps/chosen": -384.8541259765625,
      "logps/rejected": -547.2422485351562,
      "loss": 0.4674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3795424699783325,
      "rewards/margins": 1.06363844871521,
      "rewards/rejected": -2.443181037902832,
      "step": 15149
    },
    {
      "epoch": 1.9824813936370327,
      "grad_norm": 9.199437141418457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2070642113685608,
      "logits/rejected": -0.23613396286964417,
      "logps/chosen": -451.0462951660156,
      "logps/rejected": -367.82208251953125,
      "loss": 0.449,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11572440713644028,
      "rewards/margins": 1.3739522695541382,
      "rewards/rejected": -1.489676594734192,
      "step": 15150
    },
    {
      "epoch": 1.9826122515743845,
      "grad_norm": 9.267829895019531,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23265479505062103,
      "logits/rejected": -0.1671275794506073,
      "logps/chosen": -350.1228332519531,
      "logps/rejected": -341.6195983886719,
      "loss": 0.5488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9683777093887329,
      "rewards/margins": 1.2051148414611816,
      "rewards/rejected": -2.173492670059204,
      "step": 15151
    },
    {
      "epoch": 1.9827431095117363,
      "grad_norm": 9.831141471862793,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11093481630086899,
      "logits/rejected": -0.1013047993183136,
      "logps/chosen": -377.2496643066406,
      "logps/rejected": -408.96148681640625,
      "loss": 0.6652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2426611185073853,
      "rewards/margins": 0.7177463173866272,
      "rewards/rejected": -1.9604074954986572,
      "step": 15152
    },
    {
      "epoch": 1.982873967449088,
      "grad_norm": 10.59335708618164,
      "learning_rate": 5e-06,
      "logits/chosen": -0.24539479613304138,
      "logits/rejected": -0.2991349697113037,
      "logps/chosen": -348.6519470214844,
      "logps/rejected": -475.9970397949219,
      "loss": 0.5423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6763094663619995,
      "rewards/margins": 0.6727196574211121,
      "rewards/rejected": -2.349029064178467,
      "step": 15153
    },
    {
      "epoch": 1.9830048253864399,
      "grad_norm": 4.237133979797363,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2760987877845764,
      "logits/rejected": -0.46221116185188293,
      "logps/chosen": -287.83441162109375,
      "logps/rejected": -310.3845520019531,
      "loss": 0.2881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04386778175830841,
      "rewards/margins": 3.341371774673462,
      "rewards/rejected": -3.29750394821167,
      "step": 15154
    },
    {
      "epoch": 1.9831356833237916,
      "grad_norm": 11.177309036254883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2659223675727844,
      "logits/rejected": -0.4923533797264099,
      "logps/chosen": -387.7378234863281,
      "logps/rejected": -403.66845703125,
      "loss": 0.6517,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.182489037513733,
      "rewards/margins": 0.7011497616767883,
      "rewards/rejected": -1.8836387395858765,
      "step": 15155
    },
    {
      "epoch": 1.9832665412611434,
      "grad_norm": 6.743587493896484,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4547737240791321,
      "logits/rejected": -0.7637938857078552,
      "logps/chosen": -591.474853515625,
      "logps/rejected": -592.9000244140625,
      "loss": 0.3104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9664360284805298,
      "rewards/margins": 2.0957350730895996,
      "rewards/rejected": -3.062171220779419,
      "step": 15156
    },
    {
      "epoch": 1.9833973991984952,
      "grad_norm": 13.626741409301758,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20508040487766266,
      "logits/rejected": -0.3306789696216583,
      "logps/chosen": -504.7156677246094,
      "logps/rejected": -394.65264892578125,
      "loss": 0.3829,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5988854169845581,
      "rewards/margins": 1.831519365310669,
      "rewards/rejected": -2.4304046630859375,
      "step": 15157
    },
    {
      "epoch": 1.983528257135847,
      "grad_norm": 8.49129581451416,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4521907567977905,
      "logits/rejected": -0.46864834427833557,
      "logps/chosen": -250.2518310546875,
      "logps/rejected": -234.6078338623047,
      "loss": 0.4385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1114181280136108,
      "rewards/margins": 1.4930311441421509,
      "rewards/rejected": -2.604449510574341,
      "step": 15158
    },
    {
      "epoch": 1.9836591150731988,
      "grad_norm": 11.758551597595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1451253592967987,
      "logits/rejected": -0.02117607370018959,
      "logps/chosen": -392.6346740722656,
      "logps/rejected": -503.9628601074219,
      "loss": 0.667,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.1364476680755615,
      "rewards/margins": 0.9548595547676086,
      "rewards/rejected": -3.0913071632385254,
      "step": 15159
    },
    {
      "epoch": 1.9837899730105504,
      "grad_norm": 7.918618679046631,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16069062054157257,
      "logits/rejected": -0.2713560163974762,
      "logps/chosen": -559.8284912109375,
      "logps/rejected": -480.17144775390625,
      "loss": 0.3645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3403475284576416,
      "rewards/margins": 1.329451322555542,
      "rewards/rejected": -2.6697988510131836,
      "step": 15160
    },
    {
      "epoch": 1.9839208309479022,
      "grad_norm": 8.285102844238281,
      "learning_rate": 5e-06,
      "logits/chosen": -0.23323805630207062,
      "logits/rejected": -0.19626247882843018,
      "logps/chosen": -483.2313232421875,
      "logps/rejected": -494.11669921875,
      "loss": 0.4648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.595039427280426,
      "rewards/margins": 0.991604208946228,
      "rewards/rejected": -1.5866436958312988,
      "step": 15161
    },
    {
      "epoch": 1.984051688885254,
      "grad_norm": 2.3898754119873047,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6231057643890381,
      "logits/rejected": -0.5075817704200745,
      "logps/chosen": -419.4736328125,
      "logps/rejected": -422.72015380859375,
      "loss": 0.0767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.46172967553138733,
      "rewards/margins": 2.8863589763641357,
      "rewards/rejected": -3.348088264465332,
      "step": 15162
    },
    {
      "epoch": 1.9841825468226058,
      "grad_norm": 11.424487113952637,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07361825555562973,
      "logits/rejected": -0.16271090507507324,
      "logps/chosen": -470.5357666015625,
      "logps/rejected": -531.84130859375,
      "loss": 0.5032,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9645438194274902,
      "rewards/margins": 1.49199640750885,
      "rewards/rejected": -3.456540107727051,
      "step": 15163
    },
    {
      "epoch": 1.9843134047599573,
      "grad_norm": 5.542855739593506,
      "learning_rate": 5e-06,
      "logits/chosen": 0.025722801685333252,
      "logits/rejected": 0.006258603185415268,
      "logps/chosen": -412.2606201171875,
      "logps/rejected": -366.9647521972656,
      "loss": 0.2534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.01290244609117508,
      "rewards/margins": 1.3033515214920044,
      "rewards/rejected": -1.2904491424560547,
      "step": 15164
    },
    {
      "epoch": 1.9844442626973091,
      "grad_norm": 9.89854621887207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16388261318206787,
      "logits/rejected": -0.12533876299858093,
      "logps/chosen": -416.14398193359375,
      "logps/rejected": -328.6173095703125,
      "loss": 0.4801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32653024792671204,
      "rewards/margins": 1.6472935676574707,
      "rewards/rejected": -1.9738237857818604,
      "step": 15165
    },
    {
      "epoch": 1.984575120634661,
      "grad_norm": 9.044722557067871,
      "learning_rate": 5e-06,
      "logits/chosen": -0.40857547521591187,
      "logits/rejected": -0.11479975283145905,
      "logps/chosen": -480.27850341796875,
      "logps/rejected": -629.2322387695312,
      "loss": 0.5312,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5310260653495789,
      "rewards/margins": 0.6547680497169495,
      "rewards/rejected": -1.1857941150665283,
      "step": 15166
    },
    {
      "epoch": 1.9847059785720127,
      "grad_norm": 16.525434494018555,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08088397979736328,
      "logits/rejected": -0.14918000996112823,
      "logps/chosen": -437.910888671875,
      "logps/rejected": -483.7732849121094,
      "loss": 0.9625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1309477090835571,
      "rewards/margins": 0.2981145679950714,
      "rewards/rejected": -1.4290622472763062,
      "step": 15167
    },
    {
      "epoch": 1.9848368365093645,
      "grad_norm": 11.41831111907959,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26071733236312866,
      "logits/rejected": -0.39173564314842224,
      "logps/chosen": -516.1873168945312,
      "logps/rejected": -544.169677734375,
      "loss": 0.5009,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.966378390789032,
      "rewards/margins": 1.0451993942260742,
      "rewards/rejected": -2.011577606201172,
      "step": 15168
    },
    {
      "epoch": 1.9849676944467163,
      "grad_norm": 9.73852825164795,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17528343200683594,
      "logits/rejected": -0.2340991199016571,
      "logps/chosen": -454.2593994140625,
      "logps/rejected": -348.4197998046875,
      "loss": 0.4228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36965543031692505,
      "rewards/margins": 2.1182870864868164,
      "rewards/rejected": -2.487942695617676,
      "step": 15169
    },
    {
      "epoch": 1.985098552384068,
      "grad_norm": 7.685892105102539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18747222423553467,
      "logits/rejected": -0.36752384901046753,
      "logps/chosen": -397.41400146484375,
      "logps/rejected": -426.03173828125,
      "loss": 0.3802,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.981270968914032,
      "rewards/margins": 2.556561231613159,
      "rewards/rejected": -3.537832260131836,
      "step": 15170
    },
    {
      "epoch": 1.9852294103214199,
      "grad_norm": 10.13090705871582,
      "learning_rate": 5e-06,
      "logits/chosen": -0.07655437290668488,
      "logits/rejected": -0.24435409903526306,
      "logps/chosen": -450.5726623535156,
      "logps/rejected": -368.9820861816406,
      "loss": 0.6243,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2844535708427429,
      "rewards/margins": 0.7648577690124512,
      "rewards/rejected": -1.0493113994598389,
      "step": 15171
    },
    {
      "epoch": 1.9853602682587717,
      "grad_norm": 11.72913646697998,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16185706853866577,
      "logits/rejected": -0.07653877139091492,
      "logps/chosen": -486.5826721191406,
      "logps/rejected": -592.2158813476562,
      "loss": 0.5364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.829420804977417,
      "rewards/margins": 0.5932842493057251,
      "rewards/rejected": -2.4227051734924316,
      "step": 15172
    },
    {
      "epoch": 1.9854911261961234,
      "grad_norm": 10.129785537719727,
      "learning_rate": 5e-06,
      "logits/chosen": 0.209488183259964,
      "logits/rejected": -0.006739277392625809,
      "logps/chosen": -593.052978515625,
      "logps/rejected": -561.3954467773438,
      "loss": 0.4227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0749918222427368,
      "rewards/margins": 1.6071867942810059,
      "rewards/rejected": -2.6821787357330322,
      "step": 15173
    },
    {
      "epoch": 1.9856219841334752,
      "grad_norm": 6.872685432434082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25840461254119873,
      "logits/rejected": -0.35658422112464905,
      "logps/chosen": -495.95928955078125,
      "logps/rejected": -415.88885498046875,
      "loss": 0.236,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2579065263271332,
      "rewards/margins": 2.377669334411621,
      "rewards/rejected": -2.635575771331787,
      "step": 15174
    },
    {
      "epoch": 1.9857528420708268,
      "grad_norm": 14.993109703063965,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10186935216188431,
      "logits/rejected": -0.13419854640960693,
      "logps/chosen": -329.7137451171875,
      "logps/rejected": -379.5269470214844,
      "loss": 0.5978,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1194058656692505,
      "rewards/margins": 1.1451845169067383,
      "rewards/rejected": -2.264590263366699,
      "step": 15175
    },
    {
      "epoch": 1.9858837000081786,
      "grad_norm": 9.73797607421875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45511722564697266,
      "logits/rejected": -0.46903908252716064,
      "logps/chosen": -440.1962890625,
      "logps/rejected": -512.8809204101562,
      "loss": 0.3479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2043874263763428,
      "rewards/margins": 1.9585928916931152,
      "rewards/rejected": -3.162980556488037,
      "step": 15176
    },
    {
      "epoch": 1.9860145579455304,
      "grad_norm": 20.056734085083008,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10258246213197708,
      "logits/rejected": -0.08246119320392609,
      "logps/chosen": -513.15234375,
      "logps/rejected": -477.70379638671875,
      "loss": 1.2739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1202759742736816,
      "rewards/margins": 0.25862014293670654,
      "rewards/rejected": -2.3788962364196777,
      "step": 15177
    },
    {
      "epoch": 1.9861454158828822,
      "grad_norm": 12.43035888671875,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13720591366291046,
      "logits/rejected": -0.19623242318630219,
      "logps/chosen": -545.9982299804688,
      "logps/rejected": -481.4818420410156,
      "loss": 0.3576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.224290132522583,
      "rewards/margins": 1.6077945232391357,
      "rewards/rejected": -2.8320846557617188,
      "step": 15178
    },
    {
      "epoch": 1.9862762738202338,
      "grad_norm": 6.033432483673096,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0527942068874836,
      "logits/rejected": -0.023057321086525917,
      "logps/chosen": -338.49365234375,
      "logps/rejected": -463.5749816894531,
      "loss": 0.2728,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5313614010810852,
      "rewards/margins": 1.739471197128296,
      "rewards/rejected": -2.2708323001861572,
      "step": 15179
    },
    {
      "epoch": 1.9864071317575855,
      "grad_norm": 16.242271423339844,
      "learning_rate": 5e-06,
      "logits/chosen": -0.45275089144706726,
      "logits/rejected": -0.47312262654304504,
      "logps/chosen": -497.8811950683594,
      "logps/rejected": -527.801513671875,
      "loss": 0.694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.014589548110962,
      "rewards/margins": 1.2175565958023071,
      "rewards/rejected": -2.2321462631225586,
      "step": 15180
    },
    {
      "epoch": 1.9865379896949373,
      "grad_norm": 6.196639060974121,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4760926365852356,
      "logits/rejected": -0.454379141330719,
      "logps/chosen": -357.15338134765625,
      "logps/rejected": -431.5751953125,
      "loss": 0.3391,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9485616683959961,
      "rewards/margins": 1.674670696258545,
      "rewards/rejected": -2.623232126235962,
      "step": 15181
    },
    {
      "epoch": 1.9866688476322891,
      "grad_norm": 7.5740227699279785,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3070034980773926,
      "logits/rejected": -0.27577126026153564,
      "logps/chosen": -711.0944213867188,
      "logps/rejected": -527.4777221679688,
      "loss": 0.3061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3822067081928253,
      "rewards/margins": 2.0316338539123535,
      "rewards/rejected": -2.4138405323028564,
      "step": 15182
    },
    {
      "epoch": 1.986799705569641,
      "grad_norm": 10.87320613861084,
      "learning_rate": 5e-06,
      "logits/chosen": 0.16923551261425018,
      "logits/rejected": -0.08628973364830017,
      "logps/chosen": -380.2357177734375,
      "logps/rejected": -387.29547119140625,
      "loss": 0.5266,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17618422210216522,
      "rewards/margins": 1.6091418266296387,
      "rewards/rejected": -1.7853260040283203,
      "step": 15183
    },
    {
      "epoch": 1.9869305635069927,
      "grad_norm": 10.80807876586914,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2613877058029175,
      "logits/rejected": -0.33643653988838196,
      "logps/chosen": -362.2911376953125,
      "logps/rejected": -448.94757080078125,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1493744850158691,
      "rewards/margins": 0.8258052468299866,
      "rewards/rejected": -1.975179672241211,
      "step": 15184
    },
    {
      "epoch": 1.9870614214443445,
      "grad_norm": 5.448503494262695,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12264198064804077,
      "logits/rejected": -0.31737852096557617,
      "logps/chosen": -411.0120849609375,
      "logps/rejected": -481.63970947265625,
      "loss": 0.4076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36015188694000244,
      "rewards/margins": 2.7394232749938965,
      "rewards/rejected": -3.0995750427246094,
      "step": 15185
    },
    {
      "epoch": 1.9871922793816963,
      "grad_norm": 8.482684135437012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22479988634586334,
      "logits/rejected": -0.37231364846229553,
      "logps/chosen": -419.16314697265625,
      "logps/rejected": -294.2369079589844,
      "loss": 0.3761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3414206504821777,
      "rewards/margins": 1.6348246335983276,
      "rewards/rejected": -2.976245164871216,
      "step": 15186
    },
    {
      "epoch": 1.987323137319048,
      "grad_norm": 4.432736396789551,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2301861196756363,
      "logits/rejected": -0.028941530734300613,
      "logps/chosen": -385.22711181640625,
      "logps/rejected": -423.43060302734375,
      "loss": 0.1395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.029558934271335602,
      "rewards/margins": 3.1199891567230225,
      "rewards/rejected": -3.149548053741455,
      "step": 15187
    },
    {
      "epoch": 1.9874539952563999,
      "grad_norm": 6.866717338562012,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0910472571849823,
      "logits/rejected": -0.41397204995155334,
      "logps/chosen": -493.4350280761719,
      "logps/rejected": -440.1577453613281,
      "loss": 0.3539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4046827554702759,
      "rewards/margins": 1.5174390077590942,
      "rewards/rejected": -2.922122001647949,
      "step": 15188
    },
    {
      "epoch": 1.9875848531937517,
      "grad_norm": 9.689739227294922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4735446572303772,
      "logits/rejected": -0.5227063298225403,
      "logps/chosen": -539.0703125,
      "logps/rejected": -498.27325439453125,
      "loss": 0.3965,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5593565702438354,
      "rewards/margins": 1.847301959991455,
      "rewards/rejected": -3.406658411026001,
      "step": 15189
    },
    {
      "epoch": 1.9877157111311035,
      "grad_norm": 8.808826446533203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41843748092651367,
      "logits/rejected": -0.463894784450531,
      "logps/chosen": -422.0736083984375,
      "logps/rejected": -384.49908447265625,
      "loss": 0.4079,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4372351169586182,
      "rewards/margins": 1.2815812826156616,
      "rewards/rejected": -2.718816041946411,
      "step": 15190
    },
    {
      "epoch": 1.987846569068455,
      "grad_norm": 3.3415634632110596,
      "learning_rate": 5e-06,
      "logits/chosen": -0.16784802079200745,
      "logits/rejected": -0.27332377433776855,
      "logps/chosen": -559.3978271484375,
      "logps/rejected": -415.0168151855469,
      "loss": 0.1669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16828158497810364,
      "rewards/margins": 2.3030242919921875,
      "rewards/rejected": -2.471306085586548,
      "step": 15191
    },
    {
      "epoch": 1.9879774270058068,
      "grad_norm": 7.508224964141846,
      "learning_rate": 5e-06,
      "logits/chosen": -0.043073877692222595,
      "logits/rejected": -0.19687548279762268,
      "logps/chosen": -439.2031555175781,
      "logps/rejected": -470.2198181152344,
      "loss": 0.5283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.074526309967041,
      "rewards/margins": 1.5151249170303345,
      "rewards/rejected": -2.589651107788086,
      "step": 15192
    },
    {
      "epoch": 1.9881082849431586,
      "grad_norm": 4.119952201843262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1066252589225769,
      "logits/rejected": -0.08041426539421082,
      "logps/chosen": -319.3043212890625,
      "logps/rejected": -401.095703125,
      "loss": 0.1038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0049148797988892,
      "rewards/margins": 2.849581718444824,
      "rewards/rejected": -3.854496479034424,
      "step": 15193
    },
    {
      "epoch": 1.9882391428805102,
      "grad_norm": 7.928440570831299,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2398758828639984,
      "logits/rejected": -0.3943381607532501,
      "logps/chosen": -525.3701171875,
      "logps/rejected": -699.1901245117188,
      "loss": 0.3679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4947332441806793,
      "rewards/margins": 1.352730631828308,
      "rewards/rejected": -1.8474640846252441,
      "step": 15194
    },
    {
      "epoch": 1.988370000817862,
      "grad_norm": 8.400287628173828,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35378772020339966,
      "logits/rejected": -0.2940937876701355,
      "logps/chosen": -514.00732421875,
      "logps/rejected": -431.8757019042969,
      "loss": 0.4237,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7656924724578857,
      "rewards/margins": 1.5160901546478271,
      "rewards/rejected": -2.281782627105713,
      "step": 15195
    },
    {
      "epoch": 1.9885008587552138,
      "grad_norm": 4.734719276428223,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3339499235153198,
      "logits/rejected": -0.4027688503265381,
      "logps/chosen": -279.86383056640625,
      "logps/rejected": -433.1477355957031,
      "loss": 0.3026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05999806523323059,
      "rewards/margins": 2.6560752391815186,
      "rewards/rejected": -2.7160727977752686,
      "step": 15196
    },
    {
      "epoch": 1.9886317166925656,
      "grad_norm": 10.275534629821777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.7751309871673584,
      "logits/rejected": -0.5729920864105225,
      "logps/chosen": -232.5657196044922,
      "logps/rejected": -383.30206298828125,
      "loss": 0.8555,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4779764413833618,
      "rewards/margins": 1.0834519863128662,
      "rewards/rejected": -2.5614283084869385,
      "step": 15197
    },
    {
      "epoch": 1.9887625746299173,
      "grad_norm": 4.482484817504883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27777746319770813,
      "logits/rejected": -0.3839426636695862,
      "logps/chosen": -318.0042419433594,
      "logps/rejected": -345.1531066894531,
      "loss": 0.2959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10183297097682953,
      "rewards/margins": 1.9885103702545166,
      "rewards/rejected": -2.090343475341797,
      "step": 15198
    },
    {
      "epoch": 1.9888934325672691,
      "grad_norm": 3.9823498725891113,
      "learning_rate": 5e-06,
      "logits/chosen": -0.20324665307998657,
      "logits/rejected": -0.18891337513923645,
      "logps/chosen": -524.21240234375,
      "logps/rejected": -450.458251953125,
      "loss": 0.1332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.399173617362976,
      "rewards/margins": 2.465610980987549,
      "rewards/rejected": -3.8647844791412354,
      "step": 15199
    },
    {
      "epoch": 1.989024290504621,
      "grad_norm": 10.476841926574707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1552017331123352,
      "logits/rejected": -0.25755777955055237,
      "logps/chosen": -552.8314819335938,
      "logps/rejected": -501.427490234375,
      "loss": 0.5642,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.315317153930664,
      "rewards/margins": 0.9260052442550659,
      "rewards/rejected": -2.2413222789764404,
      "step": 15200
    },
    {
      "epoch": 1.9891551484419727,
      "grad_norm": 8.283774375915527,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5564006567001343,
      "logits/rejected": -0.8595980405807495,
      "logps/chosen": -443.8547058105469,
      "logps/rejected": -379.9109191894531,
      "loss": 0.3531,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3021247386932373,
      "rewards/margins": 2.7167093753814697,
      "rewards/rejected": -4.018833637237549,
      "step": 15201
    },
    {
      "epoch": 1.9892860063793245,
      "grad_norm": 10.421555519104004,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22609487175941467,
      "logits/rejected": -0.4282296299934387,
      "logps/chosen": -384.9578552246094,
      "logps/rejected": -426.6763000488281,
      "loss": 0.5845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.597491443157196,
      "rewards/margins": 1.4452834129333496,
      "rewards/rejected": -2.0427746772766113,
      "step": 15202
    },
    {
      "epoch": 1.9894168643166763,
      "grad_norm": 5.693448066711426,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5911428928375244,
      "logits/rejected": -0.5582836270332336,
      "logps/chosen": -500.4314270019531,
      "logps/rejected": -609.9005737304688,
      "loss": 0.1446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9031313061714172,
      "rewards/margins": 3.734630823135376,
      "rewards/rejected": -4.637762069702148,
      "step": 15203
    },
    {
      "epoch": 1.989547722254028,
      "grad_norm": 9.869470596313477,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35027146339416504,
      "logits/rejected": -0.17052611708641052,
      "logps/chosen": -428.1046447753906,
      "logps/rejected": -434.9046630859375,
      "loss": 0.5888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3580287992954254,
      "rewards/margins": 1.1261664628982544,
      "rewards/rejected": -1.4841952323913574,
      "step": 15204
    },
    {
      "epoch": 1.9896785801913799,
      "grad_norm": 21.221174240112305,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3487035036087036,
      "logits/rejected": -0.32441872358322144,
      "logps/chosen": -441.77642822265625,
      "logps/rejected": -382.96026611328125,
      "loss": 1.1117,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.7711316347122192,
      "rewards/margins": 0.20037437975406647,
      "rewards/rejected": -1.9715057611465454,
      "step": 15205
    },
    {
      "epoch": 1.9898094381287315,
      "grad_norm": 12.598106384277344,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25653183460235596,
      "logits/rejected": -0.25732421875,
      "logps/chosen": -292.5039367675781,
      "logps/rejected": -416.42474365234375,
      "loss": 0.7462,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.521504521369934,
      "rewards/margins": 1.6302366256713867,
      "rewards/rejected": -3.1517410278320312,
      "step": 15206
    },
    {
      "epoch": 1.9899402960660832,
      "grad_norm": 9.100377082824707,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12554322183132172,
      "logits/rejected": -0.09525865316390991,
      "logps/chosen": -501.94110107421875,
      "logps/rejected": -480.0965270996094,
      "loss": 0.4506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0540339946746826,
      "rewards/margins": 1.3663700819015503,
      "rewards/rejected": -2.4204039573669434,
      "step": 15207
    },
    {
      "epoch": 1.990071154003435,
      "grad_norm": 7.9962310791015625,
      "learning_rate": 5e-06,
      "logits/chosen": 0.07929325103759766,
      "logits/rejected": -0.18456026911735535,
      "logps/chosen": -382.5384521484375,
      "logps/rejected": -413.12799072265625,
      "loss": 0.3404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2359325885772705,
      "rewards/margins": 1.9511830806732178,
      "rewards/rejected": -3.1871156692504883,
      "step": 15208
    },
    {
      "epoch": 1.9902020119407868,
      "grad_norm": 2.5443105697631836,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3635634779930115,
      "logits/rejected": -0.4725850224494934,
      "logps/chosen": -458.6839294433594,
      "logps/rejected": -460.22509765625,
      "loss": 0.0791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5550474524497986,
      "rewards/margins": 3.1471312046051025,
      "rewards/rejected": -3.702178716659546,
      "step": 15209
    },
    {
      "epoch": 1.9903328698781384,
      "grad_norm": 15.320976257324219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3892333507537842,
      "logits/rejected": -0.5106076598167419,
      "logps/chosen": -543.9506225585938,
      "logps/rejected": -461.64727783203125,
      "loss": 0.6277,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5578771829605103,
      "rewards/margins": 0.9931737184524536,
      "rewards/rejected": -1.5510507822036743,
      "step": 15210
    },
    {
      "epoch": 1.9904637278154902,
      "grad_norm": 9.757298469543457,
      "learning_rate": 5e-06,
      "logits/chosen": -0.036744892597198486,
      "logits/rejected": -0.23698864877223969,
      "logps/chosen": -580.5374755859375,
      "logps/rejected": -594.8587646484375,
      "loss": 0.5086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2787843942642212,
      "rewards/margins": 1.2894537448883057,
      "rewards/rejected": -2.568237781524658,
      "step": 15211
    },
    {
      "epoch": 1.990594585752842,
      "grad_norm": 5.614014148712158,
      "learning_rate": 5e-06,
      "logits/chosen": -0.030105680227279663,
      "logits/rejected": -0.3765227496623993,
      "logps/chosen": -533.0084838867188,
      "logps/rejected": -436.33123779296875,
      "loss": 0.2506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17545375227928162,
      "rewards/margins": 2.4145679473876953,
      "rewards/rejected": -2.590022087097168,
      "step": 15212
    },
    {
      "epoch": 1.9907254436901938,
      "grad_norm": 7.713871479034424,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3421090543270111,
      "logits/rejected": -0.4393381178379059,
      "logps/chosen": -394.9347839355469,
      "logps/rejected": -429.5417175292969,
      "loss": 0.4793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.668984591960907,
      "rewards/margins": 1.712679386138916,
      "rewards/rejected": -2.3816637992858887,
      "step": 15213
    },
    {
      "epoch": 1.9908563016275456,
      "grad_norm": 7.325855731964111,
      "learning_rate": 5e-06,
      "logits/chosen": -0.149631068110466,
      "logits/rejected": -0.1563672572374344,
      "logps/chosen": -537.8533935546875,
      "logps/rejected": -391.62823486328125,
      "loss": 0.3029,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9432117342948914,
      "rewards/margins": 1.5012502670288086,
      "rewards/rejected": -2.4444620609283447,
      "step": 15214
    },
    {
      "epoch": 1.9909871595648974,
      "grad_norm": 11.857484817504883,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5262439846992493,
      "logits/rejected": -0.4394891560077667,
      "logps/chosen": -280.8935241699219,
      "logps/rejected": -345.2205505371094,
      "loss": 0.5915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6403778791427612,
      "rewards/margins": 0.48213210701942444,
      "rewards/rejected": -2.1225099563598633,
      "step": 15215
    },
    {
      "epoch": 1.9911180175022491,
      "grad_norm": 10.421381950378418,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3000335097312927,
      "logits/rejected": -0.49847280979156494,
      "logps/chosen": -528.9913940429688,
      "logps/rejected": -543.4533081054688,
      "loss": 0.4658,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8126782774925232,
      "rewards/margins": 2.418294668197632,
      "rewards/rejected": -3.2309727668762207,
      "step": 15216
    },
    {
      "epoch": 1.991248875439601,
      "grad_norm": 8.19942569732666,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2050372064113617,
      "logits/rejected": -0.018277201801538467,
      "logps/chosen": -779.8690185546875,
      "logps/rejected": -557.7471923828125,
      "loss": 0.3368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3355255424976349,
      "rewards/margins": 1.30478036403656,
      "rewards/rejected": -1.640305757522583,
      "step": 15217
    },
    {
      "epoch": 1.9913797333769527,
      "grad_norm": 10.891098022460938,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15284967422485352,
      "logits/rejected": -0.3246106207370758,
      "logps/chosen": -545.9962158203125,
      "logps/rejected": -490.72381591796875,
      "loss": 0.4641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8962650299072266,
      "rewards/margins": 1.3486323356628418,
      "rewards/rejected": -2.2448973655700684,
      "step": 15218
    },
    {
      "epoch": 1.9915105913143045,
      "grad_norm": 17.11812973022461,
      "learning_rate": 5e-06,
      "logits/chosen": 0.08824749290943146,
      "logits/rejected": 0.049120381474494934,
      "logps/chosen": -464.8737487792969,
      "logps/rejected": -600.7659301757812,
      "loss": 0.987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9885931015014648,
      "rewards/margins": -0.14372001588344574,
      "rewards/rejected": -0.8448730111122131,
      "step": 15219
    },
    {
      "epoch": 1.9916414492516563,
      "grad_norm": 7.050236225128174,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2061484009027481,
      "logits/rejected": -0.2778981626033783,
      "logps/chosen": -439.6249084472656,
      "logps/rejected": -425.16156005859375,
      "loss": 0.2304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47334200143814087,
      "rewards/margins": 2.546718120574951,
      "rewards/rejected": -3.0200600624084473,
      "step": 15220
    },
    {
      "epoch": 1.9917723071890079,
      "grad_norm": 13.419787406921387,
      "learning_rate": 5e-06,
      "logits/chosen": -0.05903375521302223,
      "logits/rejected": 0.0385284386575222,
      "logps/chosen": -479.88836669921875,
      "logps/rejected": -459.5293273925781,
      "loss": 0.7583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5957458019256592,
      "rewards/margins": 1.1041265726089478,
      "rewards/rejected": -2.6998722553253174,
      "step": 15221
    },
    {
      "epoch": 1.9919031651263597,
      "grad_norm": 9.082245826721191,
      "learning_rate": 5e-06,
      "logits/chosen": -0.27220216393470764,
      "logits/rejected": -0.4126233458518982,
      "logps/chosen": -361.1296081542969,
      "logps/rejected": -340.73516845703125,
      "loss": 0.5586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.561355710029602,
      "rewards/margins": 1.761765480041504,
      "rewards/rejected": -3.3231208324432373,
      "step": 15222
    },
    {
      "epoch": 1.9920340230637115,
      "grad_norm": 10.0358247756958,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2823833227157593,
      "logits/rejected": -0.28841835260391235,
      "logps/chosen": -400.28057861328125,
      "logps/rejected": -403.6840515136719,
      "loss": 0.2894,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9952833652496338,
      "rewards/margins": 2.3894312381744385,
      "rewards/rejected": -3.384714365005493,
      "step": 15223
    },
    {
      "epoch": 1.9921648810010633,
      "grad_norm": 9.772464752197266,
      "learning_rate": 5e-06,
      "logits/chosen": -0.8042858839035034,
      "logits/rejected": -0.6463947892189026,
      "logps/chosen": -425.1689758300781,
      "logps/rejected": -386.03204345703125,
      "loss": 0.424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2505966424942017,
      "rewards/margins": 1.5934733152389526,
      "rewards/rejected": -2.844069719314575,
      "step": 15224
    },
    {
      "epoch": 1.9922957389384148,
      "grad_norm": 12.034337043762207,
      "learning_rate": 5e-06,
      "logits/chosen": -0.35071542859077454,
      "logits/rejected": -0.5541513562202454,
      "logps/chosen": -458.5888366699219,
      "logps/rejected": -613.7001953125,
      "loss": 0.5621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.45385971665382385,
      "rewards/margins": 0.7749710083007812,
      "rewards/rejected": -1.2288306951522827,
      "step": 15225
    },
    {
      "epoch": 1.9924265968757666,
      "grad_norm": 10.802008628845215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4423161745071411,
      "logits/rejected": -0.44783473014831543,
      "logps/chosen": -406.1870422363281,
      "logps/rejected": -503.9293212890625,
      "loss": 0.5454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1379855871200562,
      "rewards/margins": 0.579354465007782,
      "rewards/rejected": -1.7173399925231934,
      "step": 15226
    },
    {
      "epoch": 1.9925574548131184,
      "grad_norm": 15.513318061828613,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3043171167373657,
      "logits/rejected": -0.34101077914237976,
      "logps/chosen": -342.29937744140625,
      "logps/rejected": -381.12799072265625,
      "loss": 1.1454,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8949624300003052,
      "rewards/margins": -0.4268089532852173,
      "rewards/rejected": -1.4681535959243774,
      "step": 15227
    },
    {
      "epoch": 1.9926883127504702,
      "grad_norm": 7.9290618896484375,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4946284890174866,
      "logits/rejected": -0.49895453453063965,
      "logps/chosen": -360.0521240234375,
      "logps/rejected": -631.6162719726562,
      "loss": 0.291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6521318554878235,
      "rewards/margins": 2.625023603439331,
      "rewards/rejected": -3.2771553993225098,
      "step": 15228
    },
    {
      "epoch": 1.992819170687822,
      "grad_norm": 13.268839836120605,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11634999513626099,
      "logits/rejected": -0.21873608231544495,
      "logps/chosen": -684.4618530273438,
      "logps/rejected": -590.4649047851562,
      "loss": 0.6244,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6536121368408203,
      "rewards/margins": 0.9884335994720459,
      "rewards/rejected": -1.6420456171035767,
      "step": 15229
    },
    {
      "epoch": 1.9929500286251738,
      "grad_norm": 9.49323844909668,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10169555246829987,
      "logits/rejected": -0.11014871299266815,
      "logps/chosen": -586.8394165039062,
      "logps/rejected": -596.8497314453125,
      "loss": 0.3785,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4788970947265625,
      "rewards/margins": 1.459510087966919,
      "rewards/rejected": -1.9384071826934814,
      "step": 15230
    },
    {
      "epoch": 1.9930808865625256,
      "grad_norm": 9.746365547180176,
      "learning_rate": 5e-06,
      "logits/chosen": 0.1207486093044281,
      "logits/rejected": -0.07107749581336975,
      "logps/chosen": -349.4953918457031,
      "logps/rejected": -481.71295166015625,
      "loss": 0.3567,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2788647413253784,
      "rewards/margins": 2.1007938385009766,
      "rewards/rejected": -3.3796584606170654,
      "step": 15231
    },
    {
      "epoch": 1.9932117444998774,
      "grad_norm": 17.588123321533203,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5211277604103088,
      "logits/rejected": -0.6506325006484985,
      "logps/chosen": -679.2805786132812,
      "logps/rejected": -570.661865234375,
      "loss": 0.5374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2562698125839233,
      "rewards/margins": 0.7952260971069336,
      "rewards/rejected": -2.0514960289001465,
      "step": 15232
    },
    {
      "epoch": 1.9933426024372292,
      "grad_norm": 10.746679306030273,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12609295547008514,
      "logits/rejected": -0.24743226170539856,
      "logps/chosen": -300.3788757324219,
      "logps/rejected": -349.98370361328125,
      "loss": 0.6134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9347671866416931,
      "rewards/margins": 1.5214366912841797,
      "rewards/rejected": -2.4562039375305176,
      "step": 15233
    },
    {
      "epoch": 1.993473460374581,
      "grad_norm": 12.549437522888184,
      "learning_rate": 5e-06,
      "logits/chosen": -0.12347054481506348,
      "logits/rejected": -0.14774084091186523,
      "logps/chosen": -550.1640014648438,
      "logps/rejected": -518.5349731445312,
      "loss": 0.5436,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7808048725128174,
      "rewards/margins": 1.5513582229614258,
      "rewards/rejected": -3.332162857055664,
      "step": 15234
    },
    {
      "epoch": 1.9936043183119327,
      "grad_norm": 6.535891056060791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.08856633305549622,
      "logits/rejected": -0.3399377465248108,
      "logps/chosen": -422.5164794921875,
      "logps/rejected": -396.2254638671875,
      "loss": 0.3394,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.572001039981842,
      "rewards/margins": 1.775618314743042,
      "rewards/rejected": -2.3476195335388184,
      "step": 15235
    },
    {
      "epoch": 1.9937351762492845,
      "grad_norm": 11.921489715576172,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3340945839881897,
      "logits/rejected": -0.7221401929855347,
      "logps/chosen": -625.1060791015625,
      "logps/rejected": -594.0194091796875,
      "loss": 0.5684,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40931108593940735,
      "rewards/margins": 0.9247472882270813,
      "rewards/rejected": -1.334058403968811,
      "step": 15236
    },
    {
      "epoch": 1.993866034186636,
      "grad_norm": 16.591045379638672,
      "learning_rate": 5e-06,
      "logits/chosen": -0.0943836122751236,
      "logits/rejected": -0.22158798575401306,
      "logps/chosen": -441.26190185546875,
      "logps/rejected": -400.27532958984375,
      "loss": 0.9358,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.355563759803772,
      "rewards/margins": 0.2266739010810852,
      "rewards/rejected": -1.582237720489502,
      "step": 15237
    },
    {
      "epoch": 1.993996892123988,
      "grad_norm": 11.787064552307129,
      "learning_rate": 5e-06,
      "logits/chosen": -0.04711627587676048,
      "logits/rejected": -0.012988328002393246,
      "logps/chosen": -322.46343994140625,
      "logps/rejected": -443.9838562011719,
      "loss": 0.8575,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3318020105361938,
      "rewards/margins": 0.4709895849227905,
      "rewards/rejected": -1.8027915954589844,
      "step": 15238
    },
    {
      "epoch": 1.9941277500613397,
      "grad_norm": 11.832046508789062,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5209618806838989,
      "logits/rejected": -0.2714337110519409,
      "logps/chosen": -296.8037414550781,
      "logps/rejected": -362.3331298828125,
      "loss": 0.779,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.673748254776001,
      "rewards/margins": 0.0051629915833473206,
      "rewards/rejected": -1.6789112091064453,
      "step": 15239
    },
    {
      "epoch": 1.9942586079986913,
      "grad_norm": 7.533806324005127,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3996875584125519,
      "logits/rejected": -0.3326535224914551,
      "logps/chosen": -395.57391357421875,
      "logps/rejected": -357.7179870605469,
      "loss": 0.4491,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5627168416976929,
      "rewards/margins": 0.8202144503593445,
      "rewards/rejected": -1.3829312324523926,
      "step": 15240
    },
    {
      "epoch": 1.994389465936043,
      "grad_norm": 16.09769058227539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.36381229758262634,
      "logits/rejected": -0.22379198670387268,
      "logps/chosen": -603.922607421875,
      "logps/rejected": -472.7071533203125,
      "loss": 0.8127,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6950252056121826,
      "rewards/margins": 0.08922594785690308,
      "rewards/rejected": -1.7842512130737305,
      "step": 15241
    },
    {
      "epoch": 1.9945203238733948,
      "grad_norm": 8.167956352233887,
      "learning_rate": 5e-06,
      "logits/chosen": -0.17545726895332336,
      "logits/rejected": -0.472947895526886,
      "logps/chosen": -375.1906433105469,
      "logps/rejected": -347.7846374511719,
      "loss": 0.554,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5226521492004395,
      "rewards/margins": 1.0981402397155762,
      "rewards/rejected": -2.6207921504974365,
      "step": 15242
    },
    {
      "epoch": 1.9946511818107466,
      "grad_norm": 10.129889488220215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21492497622966766,
      "logits/rejected": -0.3652906119823456,
      "logps/chosen": -400.27716064453125,
      "logps/rejected": -386.3912353515625,
      "loss": 0.4476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.40886062383651733,
      "rewards/margins": 1.8994072675704956,
      "rewards/rejected": -2.308267831802368,
      "step": 15243
    },
    {
      "epoch": 1.9947820397480984,
      "grad_norm": 5.551716327667236,
      "learning_rate": 5e-06,
      "logits/chosen": -0.648162841796875,
      "logits/rejected": -0.5058361887931824,
      "logps/chosen": -317.90191650390625,
      "logps/rejected": -459.6032409667969,
      "loss": 0.2302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.516007423400879,
      "rewards/margins": 1.9232988357543945,
      "rewards/rejected": -3.4393064975738525,
      "step": 15244
    },
    {
      "epoch": 1.9949128976854502,
      "grad_norm": 9.138673782348633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11646237969398499,
      "logits/rejected": -0.13321854174137115,
      "logps/chosen": -403.67974853515625,
      "logps/rejected": -442.0791931152344,
      "loss": 0.6098,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8293335437774658,
      "rewards/margins": 2.0244925022125244,
      "rewards/rejected": -2.8538260459899902,
      "step": 15245
    },
    {
      "epoch": 1.995043755622802,
      "grad_norm": 6.412639617919922,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3720009922981262,
      "logits/rejected": -0.509125828742981,
      "logps/chosen": -271.0491638183594,
      "logps/rejected": -354.8476867675781,
      "loss": 0.3462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0913187265396118,
      "rewards/margins": 2.4213690757751465,
      "rewards/rejected": -3.512688159942627,
      "step": 15246
    },
    {
      "epoch": 1.9951746135601538,
      "grad_norm": 9.927665710449219,
      "learning_rate": 5e-06,
      "logits/chosen": -0.21777977049350739,
      "logits/rejected": -0.1441994607448578,
      "logps/chosen": -342.2095947265625,
      "logps/rejected": -412.49200439453125,
      "loss": 0.5463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8430103063583374,
      "rewards/margins": 1.2188392877578735,
      "rewards/rejected": -2.061849594116211,
      "step": 15247
    },
    {
      "epoch": 1.9953054714975056,
      "grad_norm": 8.272677421569824,
      "learning_rate": 5e-06,
      "logits/chosen": -0.007562246173620224,
      "logits/rejected": -0.5178720951080322,
      "logps/chosen": -484.4520263671875,
      "logps/rejected": -478.0149230957031,
      "loss": 0.3856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5144989490509033,
      "rewards/margins": 2.5523576736450195,
      "rewards/rejected": -3.066856622695923,
      "step": 15248
    },
    {
      "epoch": 1.9954363294348574,
      "grad_norm": 7.547232627868652,
      "learning_rate": 5e-06,
      "logits/chosen": -0.13109901547431946,
      "logits/rejected": -0.394725501537323,
      "logps/chosen": -591.3184204101562,
      "logps/rejected": -449.6072998046875,
      "loss": 0.2531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7961870431900024,
      "rewards/margins": 2.090864896774292,
      "rewards/rejected": -2.887052059173584,
      "step": 15249
    },
    {
      "epoch": 1.9955671873722092,
      "grad_norm": 11.17682933807373,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1737300008535385,
      "logits/rejected": -0.21358132362365723,
      "logps/chosen": -539.869384765625,
      "logps/rejected": -500.558837890625,
      "loss": 0.6343,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29129713773727417,
      "rewards/margins": 0.6074056625366211,
      "rewards/rejected": -0.8987029194831848,
      "step": 15250
    },
    {
      "epoch": 1.995698045309561,
      "grad_norm": 5.88359260559082,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2683350741863251,
      "logits/rejected": -0.377087265253067,
      "logps/chosen": -472.5968933105469,
      "logps/rejected": -474.20538330078125,
      "loss": 0.1824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8986605405807495,
      "rewards/margins": 2.4277820587158203,
      "rewards/rejected": -3.3264427185058594,
      "step": 15251
    },
    {
      "epoch": 1.9958289032469125,
      "grad_norm": 8.138140678405762,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3582547605037689,
      "logits/rejected": -0.25316789746284485,
      "logps/chosen": -235.4408721923828,
      "logps/rejected": -350.9393310546875,
      "loss": 0.4237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3550885915756226,
      "rewards/margins": 1.428314208984375,
      "rewards/rejected": -2.783402919769287,
      "step": 15252
    },
    {
      "epoch": 1.9959597611842643,
      "grad_norm": 11.311270713806152,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3758071959018707,
      "logits/rejected": -0.22160294651985168,
      "logps/chosen": -534.4432983398438,
      "logps/rejected": -401.28985595703125,
      "loss": 0.5295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2804591655731201,
      "rewards/margins": 0.9994664192199707,
      "rewards/rejected": -2.279925584793091,
      "step": 15253
    },
    {
      "epoch": 1.9960906191216161,
      "grad_norm": 12.88967227935791,
      "learning_rate": 5e-06,
      "logits/chosen": -0.25291907787323,
      "logits/rejected": -0.19572904706001282,
      "logps/chosen": -413.7684631347656,
      "logps/rejected": -486.2127990722656,
      "loss": 0.8377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.117640733718872,
      "rewards/margins": 0.8238883018493652,
      "rewards/rejected": -1.9415290355682373,
      "step": 15254
    },
    {
      "epoch": 1.996221477058968,
      "grad_norm": 7.467904090881348,
      "learning_rate": 5e-06,
      "logits/chosen": -0.22057557106018066,
      "logits/rejected": -0.3875577449798584,
      "logps/chosen": -494.2397155761719,
      "logps/rejected": -445.3633117675781,
      "loss": 0.3789,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3478800058364868,
      "rewards/margins": 1.3185489177703857,
      "rewards/rejected": -1.666429042816162,
      "step": 15255
    },
    {
      "epoch": 1.9963523349963195,
      "grad_norm": 9.507079124450684,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2867368757724762,
      "logits/rejected": -0.2823627293109894,
      "logps/chosen": -337.6416320800781,
      "logps/rejected": -406.8974304199219,
      "loss": 0.5715,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0974655151367188,
      "rewards/margins": 0.5561847686767578,
      "rewards/rejected": -1.653650164604187,
      "step": 15256
    },
    {
      "epoch": 1.9964831929336713,
      "grad_norm": 9.287985801696777,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3946678042411804,
      "logits/rejected": -0.5826238989830017,
      "logps/chosen": -455.41998291015625,
      "logps/rejected": -501.22576904296875,
      "loss": 0.5144,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21863508224487305,
      "rewards/margins": 1.4533082246780396,
      "rewards/rejected": -1.671943187713623,
      "step": 15257
    },
    {
      "epoch": 1.996614050871023,
      "grad_norm": 11.809048652648926,
      "learning_rate": 5e-06,
      "logits/chosen": -0.18940593302249908,
      "logits/rejected": -0.348728746175766,
      "logps/chosen": -435.4468688964844,
      "logps/rejected": -473.9776306152344,
      "loss": 0.4364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8319088220596313,
      "rewards/margins": 1.701546549797058,
      "rewards/rejected": -2.5334553718566895,
      "step": 15258
    },
    {
      "epoch": 1.9967449088083749,
      "grad_norm": 9.051276206970215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4976009726524353,
      "logits/rejected": -0.5994961857795715,
      "logps/chosen": -229.91552734375,
      "logps/rejected": -240.54620361328125,
      "loss": 0.5423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0951467752456665,
      "rewards/margins": 0.4394814968109131,
      "rewards/rejected": -1.5346283912658691,
      "step": 15259
    },
    {
      "epoch": 1.9968757667457266,
      "grad_norm": 6.871519088745117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.009360335767269135,
      "logits/rejected": -0.17408548295497894,
      "logps/chosen": -615.5283203125,
      "logps/rejected": -480.07489013671875,
      "loss": 0.227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25431978702545166,
      "rewards/margins": 1.889484167098999,
      "rewards/rejected": -2.1438040733337402,
      "step": 15260
    },
    {
      "epoch": 1.9970066246830784,
      "grad_norm": 5.5686445236206055,
      "learning_rate": 5e-06,
      "logits/chosen": -0.11973357945680618,
      "logits/rejected": -0.3773265779018402,
      "logps/chosen": -443.5455627441406,
      "logps/rejected": -461.96624755859375,
      "loss": 0.2697,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21065998077392578,
      "rewards/margins": 2.0074572563171387,
      "rewards/rejected": -2.2181172370910645,
      "step": 15261
    },
    {
      "epoch": 1.9971374826204302,
      "grad_norm": 13.578540802001953,
      "learning_rate": 5e-06,
      "logits/chosen": -0.15168505907058716,
      "logits/rejected": -0.28226470947265625,
      "logps/chosen": -540.9375,
      "logps/rejected": -484.8878173828125,
      "loss": 0.5483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6591442227363586,
      "rewards/margins": 1.067453384399414,
      "rewards/rejected": -1.7265976667404175,
      "step": 15262
    },
    {
      "epoch": 1.997268340557782,
      "grad_norm": 11.678526878356934,
      "learning_rate": 5e-06,
      "logits/chosen": -0.31040915846824646,
      "logits/rejected": -0.18807606399059296,
      "logps/chosen": -392.92010498046875,
      "logps/rejected": -551.2528076171875,
      "loss": 0.3942,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6492887735366821,
      "rewards/margins": 1.7032043933868408,
      "rewards/rejected": -2.3524932861328125,
      "step": 15263
    },
    {
      "epoch": 1.9973991984951338,
      "grad_norm": 12.419927597045898,
      "learning_rate": 5e-06,
      "logits/chosen": -0.5738377571105957,
      "logits/rejected": -0.4473208487033844,
      "logps/chosen": -371.56707763671875,
      "logps/rejected": -500.44940185546875,
      "loss": 0.6541,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2166025638580322,
      "rewards/margins": 0.5153179168701172,
      "rewards/rejected": -1.7319204807281494,
      "step": 15264
    },
    {
      "epoch": 1.9975300564324856,
      "grad_norm": 19.258068084716797,
      "learning_rate": 5e-06,
      "logits/chosen": -0.6324262619018555,
      "logits/rejected": -0.2084963321685791,
      "logps/chosen": -434.12506103515625,
      "logps/rejected": -467.0174255371094,
      "loss": 1.4099,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.0745773315429688,
      "rewards/margins": -0.7358517646789551,
      "rewards/rejected": -1.3387253284454346,
      "step": 15265
    },
    {
      "epoch": 1.9976609143698374,
      "grad_norm": 8.37185001373291,
      "learning_rate": 5e-06,
      "logits/chosen": -0.43657150864601135,
      "logits/rejected": -0.5748564004898071,
      "logps/chosen": -446.5041198730469,
      "logps/rejected": -456.115966796875,
      "loss": 0.3997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6735716462135315,
      "rewards/margins": 1.140064001083374,
      "rewards/rejected": -1.8136357069015503,
      "step": 15266
    },
    {
      "epoch": 1.997791772307189,
      "grad_norm": 6.318161487579346,
      "learning_rate": 5e-06,
      "logits/chosen": -0.26645103096961975,
      "logits/rejected": -0.3699934184551239,
      "logps/chosen": -263.43170166015625,
      "logps/rejected": -347.5202941894531,
      "loss": 0.4082,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0547891855239868,
      "rewards/margins": 1.0319362878799438,
      "rewards/rejected": -2.0867254734039307,
      "step": 15267
    },
    {
      "epoch": 1.9979226302445408,
      "grad_norm": 9.453306198120117,
      "learning_rate": 5e-06,
      "logits/chosen": -0.4926828444004059,
      "logits/rejected": -0.4401487708091736,
      "logps/chosen": -782.5724487304688,
      "logps/rejected": -430.2913513183594,
      "loss": 0.383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6909351348876953,
      "rewards/margins": 1.3556865453720093,
      "rewards/rejected": -2.046621799468994,
      "step": 15268
    },
    {
      "epoch": 1.9980534881818925,
      "grad_norm": 16.89090919494629,
      "learning_rate": 5e-06,
      "logits/chosen": -0.41615986824035645,
      "logits/rejected": -0.21832790970802307,
      "logps/chosen": -468.1082763671875,
      "logps/rejected": -452.5426025390625,
      "loss": 0.9906,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.451702356338501,
      "rewards/margins": 0.2956654131412506,
      "rewards/rejected": -1.7473678588867188,
      "step": 15269
    },
    {
      "epoch": 1.9981843461192443,
      "grad_norm": 8.691571235656738,
      "learning_rate": 5e-06,
      "logits/chosen": -0.1543244570493698,
      "logits/rejected": -0.34944409132003784,
      "logps/chosen": -618.7984619140625,
      "logps/rejected": -506.9132385253906,
      "loss": 0.4419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.028636176139116287,
      "rewards/margins": 1.2748323678970337,
      "rewards/rejected": -1.3034685850143433,
      "step": 15270
    },
    {
      "epoch": 1.998315204056596,
      "grad_norm": 6.8369879722595215,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3761846423149109,
      "logits/rejected": -0.4392351508140564,
      "logps/chosen": -540.5628051757812,
      "logps/rejected": -462.9077453613281,
      "loss": 0.3839,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5397636890411377,
      "rewards/margins": 1.1699259281158447,
      "rewards/rejected": -1.7096894979476929,
      "step": 15271
    },
    {
      "epoch": 1.9984460619939477,
      "grad_norm": 9.219073295593262,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3660738468170166,
      "logits/rejected": -0.4726349115371704,
      "logps/chosen": -410.0455627441406,
      "logps/rejected": -463.1375732421875,
      "loss": 0.475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5644534826278687,
      "rewards/margins": 0.6479135751724243,
      "rewards/rejected": -2.212367296218872,
      "step": 15272
    },
    {
      "epoch": 1.9985769199312995,
      "grad_norm": 12.651369094848633,
      "learning_rate": 5e-06,
      "logits/chosen": -0.32429417967796326,
      "logits/rejected": -0.43623870611190796,
      "logps/chosen": -632.2826538085938,
      "logps/rejected": -514.5816650390625,
      "loss": 0.7715,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.4681358337402344,
      "rewards/margins": 0.04520248621702194,
      "rewards/rejected": -1.513338327407837,
      "step": 15273
    },
    {
      "epoch": 1.9987077778686513,
      "grad_norm": 11.261301040649414,
      "learning_rate": 5e-06,
      "logits/chosen": -0.29066818952560425,
      "logits/rejected": -0.28537189960479736,
      "logps/chosen": -598.7221069335938,
      "logps/rejected": -448.2549133300781,
      "loss": 0.5687,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1832228899002075,
      "rewards/margins": 0.9803519248962402,
      "rewards/rejected": -2.163574695587158,
      "step": 15274
    },
    {
      "epoch": 1.998838635806003,
      "grad_norm": 4.793325901031494,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3871147632598877,
      "logits/rejected": -0.30724021792411804,
      "logps/chosen": -486.7569885253906,
      "logps/rejected": -516.4998168945312,
      "loss": 0.2294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36562538146972656,
      "rewards/margins": 2.2012877464294434,
      "rewards/rejected": -2.566912889480591,
      "step": 15275
    },
    {
      "epoch": 1.9989694937433549,
      "grad_norm": 12.943148612976074,
      "learning_rate": 5e-06,
      "logits/chosen": -0.3120726943016052,
      "logits/rejected": -0.4020148515701294,
      "logps/chosen": -329.3615417480469,
      "logps/rejected": -303.66607666015625,
      "loss": 0.5284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0223021507263184,
      "rewards/margins": 0.8472356796264648,
      "rewards/rejected": -1.8695377111434937,
      "step": 15276
    },
    {
      "epoch": 1.9991003516807067,
      "grad_norm": 3.896649122238159,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2666616141796112,
      "logits/rejected": -0.11911290138959885,
      "logps/chosen": -324.8679504394531,
      "logps/rejected": -417.9793395996094,
      "loss": 0.162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.038308534771203995,
      "rewards/margins": 2.050483226776123,
      "rewards/rejected": -2.012174606323242,
      "step": 15277
    },
    {
      "epoch": 1.9992312096180584,
      "grad_norm": 9.020072937011719,
      "learning_rate": 5e-06,
      "logits/chosen": -0.2600083649158478,
      "logits/rejected": -0.27891257405281067,
      "logps/chosen": -357.53985595703125,
      "logps/rejected": -608.772705078125,
      "loss": 0.4416,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3242745399475098,
      "rewards/margins": 1.7339556217193604,
      "rewards/rejected": -3.05823016166687,
      "step": 15278
    },
    {
      "epoch": 1.9993620675554102,
      "grad_norm": 8.35317325592041,
      "learning_rate": 5e-06,
      "logits/chosen": -0.06573984026908875,
      "logits/rejected": 0.013545658439397812,
      "logps/chosen": -596.7703857421875,
      "logps/rejected": -586.2481689453125,
      "loss": 0.2591,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9919145107269287,
      "rewards/margins": 1.8531218767166138,
      "rewards/rejected": -2.845036268234253,
      "step": 15279
    },
    {
      "epoch": 1.999492925492762,
      "grad_norm": 3.4144461154937744,
      "learning_rate": 5e-06,
      "logits/chosen": -0.8491511940956116,
      "logits/rejected": -0.8509823083877563,
      "logps/chosen": -521.3489379882812,
      "logps/rejected": -442.3291015625,
      "loss": 0.1149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5482017993927002,
      "rewards/margins": 3.0153255462646484,
      "rewards/rejected": -3.5635273456573486,
      "step": 15280
    },
    {
      "epoch": 1.9996237834301138,
      "grad_norm": 8.914037704467773,
      "learning_rate": 5e-06,
      "logits/chosen": -0.004085220396518707,
      "logits/rejected": -0.08905425667762756,
      "logps/chosen": -470.0291748046875,
      "logps/rejected": -480.880859375,
      "loss": 0.3871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1900403499603271,
      "rewards/margins": 1.2786060571670532,
      "rewards/rejected": -2.46864652633667,
      "step": 15281
    },
    {
      "epoch": 1.9997546413674654,
      "grad_norm": 6.816290378570557,
      "learning_rate": 5e-06,
      "logits/chosen": 0.18855157494544983,
      "logits/rejected": 0.304248183965683,
      "logps/chosen": -479.5544738769531,
      "logps/rejected": -447.69964599609375,
      "loss": 0.278,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0514884889125824,
      "rewards/margins": 2.154282569885254,
      "rewards/rejected": -2.1027936935424805,
      "step": 15282
    },
    {
      "epoch": 1.9998854993048172,
      "grad_norm": 6.374922275543213,
      "learning_rate": 5e-06,
      "logits/chosen": -0.10420191287994385,
      "logits/rejected": -0.06555766612291336,
      "logps/chosen": -390.5780944824219,
      "logps/rejected": -354.4221496582031,
      "loss": 0.3315,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.28540533781051636,
      "rewards/margins": 1.684490442276001,
      "rewards/rejected": -1.9698957204818726,
      "step": 15283
    },
    {
      "epoch": 2.0,
      "grad_norm": 8.06497573852539,
      "learning_rate": 5e-06,
      "logits/chosen": -0.029900865629315376,
      "logits/rejected": -0.051966458559036255,
      "logps/chosen": -304.9049377441406,
      "logps/rejected": -354.7724609375,
      "loss": 0.3122,
      "rewards/accuracies": 0.8571428656578064,
      "rewards/chosen": -0.5283486247062683,
      "rewards/margins": 1.563572883605957,
      "rewards/rejected": -2.091921329498291,
      "step": 15284
    },
    {
      "epoch": 2.0,
      "step": 15284,
      "total_flos": 0.0,
      "train_loss": 0.504847784901515,
      "train_runtime": 66682.8955,
      "train_samples_per_second": 1.834,
      "train_steps_per_second": 0.229
    }
  ],
  "logging_steps": 1,
  "max_steps": 15284,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}